1 /*
   2  * Copyright (c) 2008, 2016, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #include "precompiled.hpp"
  26 #include "c1/c1_Compilation.hpp"
  27 #include "c1/c1_LIRAssembler.hpp"
  28 #include "c1/c1_MacroAssembler.hpp"
  29 #include "c1/c1_Runtime1.hpp"
  30 #include "c1/c1_ValueStack.hpp"
  31 #include "ci/ciArrayKlass.hpp"
  32 #include "ci/ciInstance.hpp"
  33 #include "gc/shared/barrierSet.hpp"
  34 #include "gc/shared/cardTableModRefBS.hpp"
  35 #include "gc/shared/collectedHeap.hpp"
  36 #include "nativeInst_arm.hpp"
  37 #include "oops/objArrayKlass.hpp"
  38 #include "runtime/sharedRuntime.hpp"
  39 #include "vmreg_arm.inline.hpp"
  40 
  41 #define __ _masm->
  42 
  43 // Note: Rtemp usage is this file should not impact C2 and should be
  44 // correct as long as it is not implicitly used in lower layers (the
  45 // arm [macro]assembler) and used with care in the other C1 specific
  46 // files.
  47 
  48 bool LIR_Assembler::is_small_constant(LIR_Opr opr) {
  49   ShouldNotCallThis(); // Not used on ARM
  50   return false;
  51 }
  52 
  53 
  54 LIR_Opr LIR_Assembler::receiverOpr() {
  55   // The first register in Java calling conventions
  56   return FrameMap::R0_oop_opr;
  57 }
  58 
  59 LIR_Opr LIR_Assembler::osrBufferPointer() {
  60   return FrameMap::as_pointer_opr(R0);
  61 }
  62 
  63 #ifndef PRODUCT
  64 void LIR_Assembler::verify_reserved_argument_area_size(int args_count) {
  65   assert(args_count * wordSize <= frame_map()->reserved_argument_area_size(), "not enough space for arguments");
  66 }
  67 #endif // !PRODUCT
  68 
  69 void LIR_Assembler::store_parameter(jint c, int offset_from_sp_in_words) {
  70   assert(offset_from_sp_in_words >= 0, "invalid offset from sp");
  71   int offset_from_sp_in_bytes = offset_from_sp_in_words * BytesPerWord;
  72   assert(offset_from_sp_in_bytes < frame_map()->reserved_argument_area_size(), "not enough space");
  73   __ mov_slow(Rtemp, c);
  74   __ str(Rtemp, Address(SP, offset_from_sp_in_bytes));
  75 }
  76 
  77 void LIR_Assembler::store_parameter(Metadata* m, int offset_from_sp_in_words) {
  78   assert(offset_from_sp_in_words >= 0, "invalid offset from sp");
  79   int offset_from_sp_in_bytes = offset_from_sp_in_words * BytesPerWord;
  80   assert(offset_from_sp_in_bytes < frame_map()->reserved_argument_area_size(), "not enough space");
  81   __ mov_metadata(Rtemp, m);
  82   __ str(Rtemp, Address(SP, offset_from_sp_in_bytes));
  83 }
  84 
  85 //--------------fpu register translations-----------------------
  86 
  87 
  88 void LIR_Assembler::set_24bit_FPU() {
  89   ShouldNotReachHere();
  90 }
  91 
  92 void LIR_Assembler::reset_FPU() {
  93   ShouldNotReachHere();
  94 }
  95 
  96 void LIR_Assembler::fpop() {
  97   Unimplemented();
  98 }
  99 
 100 void LIR_Assembler::fxch(int i) {
 101   Unimplemented();
 102 }
 103 
 104 void LIR_Assembler::fld(int i) {
 105   Unimplemented();
 106 }
 107 
 108 void LIR_Assembler::ffree(int i) {
 109   Unimplemented();
 110 }
 111 
 112 void LIR_Assembler::breakpoint() {
 113   __ breakpoint();
 114 }
 115 
 116 void LIR_Assembler::push(LIR_Opr opr) {
 117   Unimplemented();
 118 }
 119 
 120 void LIR_Assembler::pop(LIR_Opr opr) {
 121   Unimplemented();
 122 }
 123 
 124 //-------------------------------------------
 125 Address LIR_Assembler::as_Address(LIR_Address* addr) {
 126   Register base = addr->base()->as_pointer_register();
 127 
 128 #ifdef AARCH64
 129   int align = exact_log2(type2aelembytes(addr->type(), true));
 130 #endif
 131 
 132   if (addr->index()->is_illegal() || addr->index()->is_constant()) {
 133     int offset = addr->disp();
 134     if (addr->index()->is_constant()) {
 135       offset += addr->index()->as_constant_ptr()->as_jint() << addr->scale();
 136     }
 137 
 138 #ifdef AARCH64
 139     if (!Assembler::is_unsigned_imm_in_range(offset, 12, align) && !Assembler::is_imm_in_range(offset, 9, 0)) {
 140       BAILOUT_("offset not in range", Address(base));
 141     }
 142     assert(UseUnalignedAccesses || (offset & right_n_bits(align)) == 0, "offset should be aligned");
 143 #else
 144     if ((offset <= -4096) || (offset >= 4096)) {
 145       BAILOUT_("offset not in range", Address(base));
 146     }
 147 #endif // AARCH64
 148 
 149     return Address(base, offset);
 150 
 151   } else {
 152     assert(addr->disp() == 0, "can't have both");
 153     int scale = addr->scale();
 154 
 155 #ifdef AARCH64
 156     assert((scale == 0) || (scale == align), "scale should be zero or equal to embedded shift");
 157 
 158     bool is_index_extended = (addr->index()->type() == T_INT);
 159     if (is_index_extended) {
 160       assert(addr->index()->is_single_cpu(), "should be");
 161       return Address(base, addr->index()->as_register(), ex_sxtw, scale);
 162     } else {
 163       assert(addr->index()->is_double_cpu(), "should be");
 164       return Address(base, addr->index()->as_register_lo(), ex_lsl, scale);
 165     }
 166 #else
 167     assert(addr->index()->is_single_cpu(), "should be");
 168     return scale >= 0 ? Address(base, addr->index()->as_register(), lsl, scale) :
 169                         Address(base, addr->index()->as_register(), lsr, -scale);
 170 #endif // AARCH64
 171   }
 172 }
 173 
 174 Address LIR_Assembler::as_Address_hi(LIR_Address* addr) {
 175 #ifdef AARCH64
 176   ShouldNotCallThis(); // Not used on AArch64
 177   return Address();
 178 #else
 179   Address base = as_Address(addr);
 180   assert(base.index() == noreg, "must be");
 181   if (base.disp() + BytesPerWord >= 4096) { BAILOUT_("offset not in range", Address(base.base(),0)); }
 182   return Address(base.base(), base.disp() + BytesPerWord);
 183 #endif // AARCH64
 184 }
 185 
 186 Address LIR_Assembler::as_Address_lo(LIR_Address* addr) {
 187 #ifdef AARCH64
 188   ShouldNotCallThis(); // Not used on AArch64
 189   return Address();
 190 #else
 191   return as_Address(addr);
 192 #endif // AARCH64
 193 }
 194 
 195 
 196 void LIR_Assembler::osr_entry() {
 197   offsets()->set_value(CodeOffsets::OSR_Entry, code_offset());
 198   BlockBegin* osr_entry = compilation()->hir()->osr_entry();
 199   ValueStack* entry_state = osr_entry->end()->state();
 200   int number_of_locks = entry_state->locks_size();
 201 
 202   __ build_frame(initial_frame_size_in_bytes(), bang_size_in_bytes());
 203   Register OSR_buf = osrBufferPointer()->as_pointer_register();
 204 
 205   assert(frame::interpreter_frame_monitor_size() == BasicObjectLock::size(), "adjust code below");
 206   int monitor_offset = (method()->max_locals() + 2 * (number_of_locks - 1)) * BytesPerWord;
 207   for (int i = 0; i < number_of_locks; i++) {
 208     int slot_offset = monitor_offset - (i * 2 * BytesPerWord);
 209     __ ldr(R1, Address(OSR_buf, slot_offset + 0*BytesPerWord));
 210     __ ldr(R2, Address(OSR_buf, slot_offset + 1*BytesPerWord));
 211     __ str(R1, frame_map()->address_for_monitor_lock(i));
 212     __ str(R2, frame_map()->address_for_monitor_object(i));
 213   }
 214 }
 215 
 216 
 217 int LIR_Assembler::check_icache() {
 218   Register receiver = LIR_Assembler::receiverOpr()->as_register();
 219   int offset = __ offset();
 220   __ inline_cache_check(receiver, Ricklass);
 221   return offset;
 222 }
 223 
 224 
 225 void LIR_Assembler::jobject2reg_with_patching(Register reg, CodeEmitInfo* info) {
 226   jobject o = (jobject)Universe::non_oop_word();
 227   int index = __ oop_recorder()->allocate_oop_index(o);
 228 
 229   PatchingStub* patch = new PatchingStub(_masm, patching_id(info), index);
 230 
 231   __ patchable_mov_oop(reg, o, index);
 232   patching_epilog(patch, lir_patch_normal, reg, info);
 233 }
 234 
 235 
 236 void LIR_Assembler::klass2reg_with_patching(Register reg, CodeEmitInfo* info) {
 237   Metadata* o = (Metadata*)Universe::non_oop_word();
 238   int index = __ oop_recorder()->allocate_metadata_index(o);
 239   PatchingStub* patch = new PatchingStub(_masm, PatchingStub::load_klass_id, index);
 240 
 241   __ patchable_mov_metadata(reg, o, index);
 242   patching_epilog(patch, lir_patch_normal, reg, info);
 243 }
 244 
 245 
 246 int LIR_Assembler::initial_frame_size_in_bytes() const {
 247   // Subtracts two words to account for return address and link
 248   return frame_map()->framesize()*VMRegImpl::stack_slot_size - 2*wordSize;
 249 }
 250 
 251 
 252 int LIR_Assembler::emit_exception_handler() {
 253   // TODO: ARM
 254   __ nop(); // See comments in other ports
 255 
 256   address handler_base = __ start_a_stub(exception_handler_size());
 257   if (handler_base == NULL) {
 258     bailout("exception handler overflow");
 259     return -1;
 260   }
 261 
 262   int offset = code_offset();
 263 
 264   // check that there is really an exception
 265   __ verify_not_null_oop(Rexception_obj);
 266 
 267   __ call(Runtime1::entry_for(Runtime1::handle_exception_from_callee_id), relocInfo::runtime_call_type);
 268   __ should_not_reach_here();
 269 
 270   assert(code_offset() - offset <= exception_handler_size(), "overflow");
 271   __ end_a_stub();
 272 
 273   return offset;
 274 }
 275 
 276 // Emit the code to remove the frame from the stack in the exception
 277 // unwind path.
 278 int LIR_Assembler::emit_unwind_handler() {
 279 #ifndef PRODUCT
 280   if (CommentedAssembly) {
 281     _masm->block_comment("Unwind handler");
 282   }
 283 #endif
 284 
 285   int offset = code_offset();
 286 
 287   // Fetch the exception from TLS and clear out exception related thread state
 288   Register zero = __ zero_register(Rtemp);
 289   __ ldr(Rexception_obj, Address(Rthread, JavaThread::exception_oop_offset()));
 290   __ str(zero, Address(Rthread, JavaThread::exception_oop_offset()));
 291   __ str(zero, Address(Rthread, JavaThread::exception_pc_offset()));
 292 
 293   __ bind(_unwind_handler_entry);
 294   __ verify_not_null_oop(Rexception_obj);
 295 
 296   // Preform needed unlocking
 297   MonitorExitStub* stub = NULL;
 298   if (method()->is_synchronized()) {
 299     monitor_address(0, FrameMap::R0_opr);
 300     stub = new MonitorExitStub(FrameMap::R0_opr, true, 0);
 301     __ unlock_object(R2, R1, R0, Rtemp, *stub->entry());
 302     __ bind(*stub->continuation());
 303   }
 304 
 305   // remove the activation and dispatch to the unwind handler
 306   __ remove_frame(initial_frame_size_in_bytes()); // restores FP and LR
 307   __ jump(Runtime1::entry_for(Runtime1::unwind_exception_id), relocInfo::runtime_call_type, Rtemp);
 308 
 309   // Emit the slow path assembly
 310   if (stub != NULL) {
 311     stub->emit_code(this);
 312   }
 313 
 314   return offset;
 315 }
 316 
 317 
 318 int LIR_Assembler::emit_deopt_handler() {
 319   address handler_base = __ start_a_stub(deopt_handler_size());
 320   if (handler_base == NULL) {
 321     bailout("deopt handler overflow");
 322     return -1;
 323   }
 324 
 325   int offset = code_offset();
 326 
 327   __ mov_relative_address(LR, __ pc());
 328 #ifdef AARCH64
 329   __ raw_push(LR, LR);
 330   __ jump(SharedRuntime::deopt_blob()->unpack(), relocInfo::runtime_call_type, Rtemp);
 331 #else
 332   __ push(LR); // stub expects LR to be saved
 333   __ jump(SharedRuntime::deopt_blob()->unpack(), relocInfo::runtime_call_type, noreg);
 334 #endif // AARCH64
 335 
 336   assert(code_offset() - offset <= deopt_handler_size(), "overflow");
 337   __ end_a_stub();
 338 
 339   return offset;
 340 }
 341 
 342 
 343 void LIR_Assembler::return_op(LIR_Opr result) {
 344   // Pop the frame before safepoint polling
 345   __ remove_frame(initial_frame_size_in_bytes());
 346 
 347   // mov_slow here is usually one or two instruction
 348   // TODO-AARCH64 3 instructions on AArch64, so try to load polling page by ldr_literal
 349   __ mov_address(Rtemp, os::get_polling_page(), symbolic_Relocation::polling_page_reference);
 350   __ relocate(relocInfo::poll_return_type);
 351   __ ldr(Rtemp, Address(Rtemp));
 352   __ ret();
 353 }
 354 
 355 
 356 int LIR_Assembler::safepoint_poll(LIR_Opr tmp, CodeEmitInfo* info) {
 357   __ mov_address(Rtemp, os::get_polling_page(), symbolic_Relocation::polling_page_reference);
 358   if (info != NULL) {
 359     add_debug_info_for_branch(info);
 360   }
 361   int offset = __ offset();
 362   __ relocate(relocInfo::poll_type);
 363   __ ldr(Rtemp, Address(Rtemp));
 364   return offset;
 365 }
 366 
 367 
 368 void LIR_Assembler::move_regs(Register from_reg, Register to_reg) {
 369   if (from_reg != to_reg) {
 370     __ mov(to_reg, from_reg);
 371   }
 372 }
 373 
 374 void LIR_Assembler::const2reg(LIR_Opr src, LIR_Opr dest, LIR_PatchCode patch_code, CodeEmitInfo* info) {
 375   assert(src->is_constant() && dest->is_register(), "must be");
 376   LIR_Const* c = src->as_constant_ptr();
 377 
 378   switch (c->type()) {
 379     case T_ADDRESS:
 380     case T_INT:
 381       assert(patch_code == lir_patch_none, "no patching handled here");
 382       __ mov_slow(dest->as_register(), c->as_jint());
 383       break;
 384 
 385     case T_LONG:
 386       assert(patch_code == lir_patch_none, "no patching handled here");
 387 #ifdef AARCH64
 388       __ mov_slow(dest->as_pointer_register(), (intptr_t)c->as_jlong());
 389 #else
 390       __ mov_slow(dest->as_register_lo(), c->as_jint_lo());
 391       __ mov_slow(dest->as_register_hi(), c->as_jint_hi());
 392 #endif // AARCH64
 393       break;
 394 
 395     case T_OBJECT:
 396       if (patch_code == lir_patch_none) {
 397         __ mov_oop(dest->as_register(), c->as_jobject());
 398       } else {
 399         jobject2reg_with_patching(dest->as_register(), info);
 400       }
 401       break;
 402 
 403     case T_METADATA:
 404       if (patch_code == lir_patch_none) {
 405         __ mov_metadata(dest->as_register(), c->as_metadata());
 406       } else {
 407         klass2reg_with_patching(dest->as_register(), info);
 408       }
 409       break;
 410 
 411     case T_FLOAT:
 412       if (dest->is_single_fpu()) {
 413         __ mov_float(dest->as_float_reg(), c->as_jfloat());
 414       } else {
 415 #ifdef AARCH64
 416         ShouldNotReachHere();
 417 #else
 418         // Simple getters can return float constant directly into r0
 419         __ mov_slow(dest->as_register(), c->as_jint_bits());
 420 #endif // AARCH64
 421       }
 422       break;
 423 
 424     case T_DOUBLE:
 425       if (dest->is_double_fpu()) {
 426         __ mov_double(dest->as_double_reg(), c->as_jdouble());
 427       } else {
 428 #ifdef AARCH64
 429         ShouldNotReachHere();
 430 #else
 431         // Simple getters can return double constant directly into r1r0
 432         __ mov_slow(dest->as_register_lo(), c->as_jint_lo_bits());
 433         __ mov_slow(dest->as_register_hi(), c->as_jint_hi_bits());
 434 #endif // AARCH64
 435       }
 436       break;
 437 
 438     default:
 439       ShouldNotReachHere();
 440   }
 441 }
 442 
 443 void LIR_Assembler::const2stack(LIR_Opr src, LIR_Opr dest) {
 444   assert(src->is_constant(), "must be");
 445   assert(dest->is_stack(), "must be");
 446   LIR_Const* c = src->as_constant_ptr();
 447 
 448   switch (c->type()) {
 449     case T_INT:  // fall through
 450     case T_FLOAT:
 451       __ mov_slow(Rtemp, c->as_jint_bits());
 452       __ str_32(Rtemp, frame_map()->address_for_slot(dest->single_stack_ix()));
 453       break;
 454 
 455     case T_ADDRESS:
 456       __ mov_slow(Rtemp, c->as_jint());
 457       __ str(Rtemp, frame_map()->address_for_slot(dest->single_stack_ix()));
 458       break;
 459 
 460     case T_OBJECT:
 461       __ mov_oop(Rtemp, c->as_jobject());
 462       __ str(Rtemp, frame_map()->address_for_slot(dest->single_stack_ix()));
 463       break;
 464 
 465     case T_LONG:  // fall through
 466     case T_DOUBLE:
 467 #ifdef AARCH64
 468       __ mov_slow(Rtemp, c->as_jlong_bits());
 469       __ str(Rtemp, frame_map()->address_for_slot(dest->double_stack_ix()));
 470 #else
 471       __ mov_slow(Rtemp, c->as_jint_lo_bits());
 472       __ str(Rtemp, frame_map()->address_for_slot(dest->double_stack_ix(), lo_word_offset_in_bytes));
 473       if (c->as_jint_hi_bits() != c->as_jint_lo_bits()) {
 474         __ mov_slow(Rtemp, c->as_jint_hi_bits());
 475       }
 476       __ str(Rtemp, frame_map()->address_for_slot(dest->double_stack_ix(), hi_word_offset_in_bytes));
 477 #endif // AARCH64
 478       break;
 479 
 480     default:
 481       ShouldNotReachHere();
 482   }
 483 }
 484 
 485 void LIR_Assembler::const2mem(LIR_Opr src, LIR_Opr dest, BasicType type,
 486                               CodeEmitInfo* info, bool wide) {
 487 #ifdef AARCH64
 488   assert((src->as_constant_ptr()->type() == T_OBJECT && src->as_constant_ptr()->as_jobject() == NULL) ||
 489          (src->as_constant_ptr()->type() == T_INT && src->as_constant_ptr()->as_jint() == 0) ||
 490          (src->as_constant_ptr()->type() == T_LONG && src->as_constant_ptr()->as_jlong() == 0) ||
 491          (src->as_constant_ptr()->type() == T_FLOAT && src->as_constant_ptr()->as_jint_bits() == 0) ||
 492          (src->as_constant_ptr()->type() == T_DOUBLE && src->as_constant_ptr()->as_jlong_bits() == 0),
 493         "cannot handle otherwise");
 494   assert(dest->as_address_ptr()->type() == type, "should be");
 495 
 496   Address addr = as_Address(dest->as_address_ptr());
 497   int null_check_offset = code_offset();
 498   switch (type) {
 499     case T_OBJECT:  // fall through
 500     case T_ARRAY:
 501         if (UseCompressedOops && !wide) {
 502           __ str_w(ZR, addr);
 503         } else {
 504           __ str(ZR, addr);
 505         }
 506         break;
 507     case T_ADDRESS: // fall through
 508     case T_DOUBLE:  // fall through
 509     case T_LONG:    __ str(ZR, addr);   break;
 510     case T_FLOAT:   // fall through
 511     case T_INT:     __ str_w(ZR, addr); break;
 512     case T_BOOLEAN: // fall through
 513     case T_BYTE:    __ strb(ZR, addr);  break;
 514     case T_CHAR:    // fall through
 515     case T_SHORT:   __ strh(ZR, addr);  break;
 516     default: ShouldNotReachHere();
 517   }
 518 #else
 519   assert((src->as_constant_ptr()->type() == T_OBJECT && src->as_constant_ptr()->as_jobject() == NULL),"cannot handle otherwise");
 520   __ mov(Rtemp, 0);
 521 
 522   int null_check_offset = code_offset();
 523   __ str(Rtemp, as_Address(dest->as_address_ptr()));
 524 #endif // AARCH64
 525 
 526   if (info != NULL) {
 527 #ifndef AARCH64
 528     assert(false, "arm32 didn't support this before, investigate if bug");
 529 #endif
 530     add_debug_info_for_null_check(null_check_offset, info);
 531   }
 532 }
 533 
 534 void LIR_Assembler::reg2reg(LIR_Opr src, LIR_Opr dest) {
 535   assert(src->is_register() && dest->is_register(), "must be");
 536 
 537   if (src->is_single_cpu()) {
 538     if (dest->is_single_cpu()) {
 539       move_regs(src->as_register(), dest->as_register());
 540 #ifdef AARCH64
 541     } else if (dest->is_double_cpu()) {
 542       assert ((src->type() == T_OBJECT) || (src->type() == T_ARRAY) || (src->type() == T_ADDRESS), "invalid src type");
 543       move_regs(src->as_register(), dest->as_register_lo());
 544 #else
 545     } else if (dest->is_single_fpu()) {
 546       __ fmsr(dest->as_float_reg(), src->as_register());
 547 #endif // AARCH64
 548     } else {
 549       ShouldNotReachHere();
 550     }
 551   } else if (src->is_double_cpu()) {
 552 #ifdef AARCH64
 553     move_regs(src->as_register_lo(), dest->as_register_lo());
 554 #else
 555     if (dest->is_double_cpu()) {
 556       __ long_move(dest->as_register_lo(), dest->as_register_hi(), src->as_register_lo(), src->as_register_hi());
 557     } else {
 558       __ fmdrr(dest->as_double_reg(), src->as_register_lo(), src->as_register_hi());
 559     }
 560 #endif // AARCH64
 561   } else if (src->is_single_fpu()) {
 562     if (dest->is_single_fpu()) {
 563       __ mov_float(dest->as_float_reg(), src->as_float_reg());
 564     } else if (dest->is_single_cpu()) {
 565       __ mov_fpr2gpr_float(dest->as_register(), src->as_float_reg());
 566     } else {
 567       ShouldNotReachHere();
 568     }
 569   } else if (src->is_double_fpu()) {
 570     if (dest->is_double_fpu()) {
 571       __ mov_double(dest->as_double_reg(), src->as_double_reg());
 572     } else if (dest->is_double_cpu()) {
 573 #ifdef AARCH64
 574       __ fmov_xd(dest->as_register_lo(), src->as_double_reg());
 575 #else
 576       __ fmrrd(dest->as_register_lo(), dest->as_register_hi(), src->as_double_reg());
 577 #endif // AARCH64
 578     } else {
 579       ShouldNotReachHere();
 580     }
 581   } else {
 582     ShouldNotReachHere();
 583   }
 584 }
 585 
 586 void LIR_Assembler::reg2stack(LIR_Opr src, LIR_Opr dest, BasicType type, bool pop_fpu_stack) {
 587   assert(src->is_register(), "should not call otherwise");
 588   assert(dest->is_stack(), "should not call otherwise");
 589 
 590   Address addr = dest->is_single_word() ?
 591     frame_map()->address_for_slot(dest->single_stack_ix()) :
 592     frame_map()->address_for_slot(dest->double_stack_ix());
 593 
 594 #ifndef AARCH64
 595   assert(lo_word_offset_in_bytes == 0 && hi_word_offset_in_bytes == 4, "little ending");
 596   if (src->is_single_fpu() || src->is_double_fpu()) {
 597     if (addr.disp() >= 1024) { BAILOUT("Too exotic case to handle here"); }
 598   }
 599 #endif // !AARCH64
 600 
 601   if (src->is_single_cpu()) {
 602     switch (type) {
 603       case T_OBJECT:
 604       case T_ARRAY:    __ verify_oop(src->as_register());   // fall through
 605       case T_ADDRESS:
 606       case T_METADATA: __ str(src->as_register(), addr);    break;
 607       case T_FLOAT:    // used in intBitsToFloat intrinsic implementation, fall through
 608       case T_INT:      __ str_32(src->as_register(), addr); break;
 609       default:
 610         ShouldNotReachHere();
 611     }
 612   } else if (src->is_double_cpu()) {
 613     __ str(src->as_register_lo(), addr);
 614 #ifndef AARCH64
 615     __ str(src->as_register_hi(), frame_map()->address_for_slot(dest->double_stack_ix(), hi_word_offset_in_bytes));
 616 #endif // !AARCH64
 617   } else if (src->is_single_fpu()) {
 618     __ str_float(src->as_float_reg(), addr);
 619   } else if (src->is_double_fpu()) {
 620     __ str_double(src->as_double_reg(), addr);
 621   } else {
 622     ShouldNotReachHere();
 623   }
 624 }
 625 
 626 
 627 void LIR_Assembler::reg2mem(LIR_Opr src, LIR_Opr dest, BasicType type,
 628                             LIR_PatchCode patch_code, CodeEmitInfo* info,
 629                             bool pop_fpu_stack, bool wide,
 630                             bool unaligned) {
 631   LIR_Address* to_addr = dest->as_address_ptr();
 632   Register base_reg = to_addr->base()->as_pointer_register();
 633   const bool needs_patching = (patch_code != lir_patch_none);
 634 
 635   PatchingStub* patch = NULL;
 636   if (needs_patching) {
 637 #ifdef AARCH64
 638     // Same alignment of reg2mem code and PatchingStub code. Required to make copied bind_literal() code properly aligned.
 639     __ align(wordSize);
 640 #endif
 641     patch = new PatchingStub(_masm, PatchingStub::access_field_id);
 642 #ifdef AARCH64
 643     // Extra nop for MT safe patching
 644     __ nop();
 645 #endif // AARCH64
 646   }
 647 
 648   int null_check_offset = code_offset();
 649 
 650   switch (type) {
 651     case T_ARRAY:
 652     case T_OBJECT:
 653       if (UseCompressedOops && !wide) {
 654 #ifdef AARCH64
 655         const Register temp_src = Rtemp;
 656         assert_different_registers(temp_src, src->as_register());
 657         __ encode_heap_oop(temp_src, src->as_register());
 658         null_check_offset = code_offset();
 659         __ str_32(temp_src, as_Address(to_addr));
 660 #else
 661         ShouldNotReachHere();
 662 #endif // AARCH64
 663       } else {
 664         __ str(src->as_register(), as_Address(to_addr));
 665       }
 666       break;
 667 
 668     case T_ADDRESS:
 669 #ifdef AARCH64
 670     case T_LONG:
 671 #endif // AARCH64
 672       __ str(src->as_pointer_register(), as_Address(to_addr));
 673       break;
 674 
 675     case T_BYTE:
 676     case T_BOOLEAN:
 677       __ strb(src->as_register(), as_Address(to_addr));
 678       break;
 679 
 680     case T_CHAR:
 681     case T_SHORT:
 682       __ strh(src->as_register(), as_Address(to_addr));
 683       break;
 684 
 685     case T_INT:
 686 #ifdef __SOFTFP__
 687     case T_FLOAT:
 688 #endif // __SOFTFP__
 689       __ str_32(src->as_register(), as_Address(to_addr));
 690       break;
 691 
 692 #ifdef AARCH64
 693 
 694     case T_FLOAT:
 695       __ str_s(src->as_float_reg(), as_Address(to_addr));
 696       break;
 697 
 698     case T_DOUBLE:
 699       __ str_d(src->as_double_reg(), as_Address(to_addr));
 700       break;
 701 
 702 #else // AARCH64
 703 
 704 #ifdef __SOFTFP__
 705     case T_DOUBLE:
 706 #endif // __SOFTFP__
 707     case T_LONG: {
 708       Register from_lo = src->as_register_lo();
 709       Register from_hi = src->as_register_hi();
 710       if (to_addr->index()->is_register()) {
 711         assert(to_addr->scale() == LIR_Address::times_1,"Unexpected scaled register");
 712         assert(to_addr->disp() == 0, "Not yet supporting both");
 713         __ add(Rtemp, base_reg, to_addr->index()->as_register());
 714         base_reg = Rtemp;
 715         __ str(from_lo, Address(Rtemp));
 716         if (patch != NULL) {
 717           patching_epilog(patch, lir_patch_low, base_reg, info);
 718           patch = new PatchingStub(_masm, PatchingStub::access_field_id);
 719           patch_code = lir_patch_high;
 720         }
 721         __ str(from_hi, Address(Rtemp, BytesPerWord));
 722       } else if (base_reg == from_lo) {
 723         __ str(from_hi, as_Address_hi(to_addr));
 724         if (patch != NULL) {
 725           patching_epilog(patch, lir_patch_high, base_reg, info);
 726           patch = new PatchingStub(_masm, PatchingStub::access_field_id);
 727           patch_code = lir_patch_low;
 728         }
 729         __ str(from_lo, as_Address_lo(to_addr));
 730       } else {
 731         __ str(from_lo, as_Address_lo(to_addr));
 732         if (patch != NULL) {
 733           patching_epilog(patch, lir_patch_low, base_reg, info);
 734           patch = new PatchingStub(_masm, PatchingStub::access_field_id);
 735           patch_code = lir_patch_high;
 736         }
 737         __ str(from_hi, as_Address_hi(to_addr));
 738       }
 739       break;
 740     }
 741 
 742 #ifndef __SOFTFP__
 743     case T_FLOAT:
 744       if (to_addr->index()->is_register()) {
 745         assert(to_addr->scale() == LIR_Address::times_1,"Unexpected scaled register");
 746         __ add(Rtemp, base_reg, to_addr->index()->as_register());
 747         if ((to_addr->disp() <= -4096) || (to_addr->disp() >= 4096)) { BAILOUT("offset not in range"); }
 748         __ fsts(src->as_float_reg(), Address(Rtemp, to_addr->disp()));
 749       } else {
 750         __ fsts(src->as_float_reg(), as_Address(to_addr));
 751       }
 752       break;
 753 
 754     case T_DOUBLE:
 755       if (to_addr->index()->is_register()) {
 756         assert(to_addr->scale() == LIR_Address::times_1,"Unexpected scaled register");
 757         __ add(Rtemp, base_reg, to_addr->index()->as_register());
 758         if ((to_addr->disp() <= -4096) || (to_addr->disp() >= 4096)) { BAILOUT("offset not in range"); }
 759         __ fstd(src->as_double_reg(), Address(Rtemp, to_addr->disp()));
 760       } else {
 761         __ fstd(src->as_double_reg(), as_Address(to_addr));
 762       }
 763       break;
 764 #endif // __SOFTFP__
 765 
 766 #endif // AARCH64
 767 
 768     default:
 769       ShouldNotReachHere();
 770   }
 771 
 772   if (info != NULL) {
 773     add_debug_info_for_null_check(null_check_offset, info);
 774   }
 775 
 776   if (patch != NULL) {
 777     // Offset embeedded into LDR/STR instruction may appear not enough
 778     // to address a field. So, provide a space for one more instruction
 779     // that will deal with larger offsets.
 780     __ nop();
 781     patching_epilog(patch, patch_code, base_reg, info);
 782   }
 783 }
 784 
 785 
 786 void LIR_Assembler::stack2reg(LIR_Opr src, LIR_Opr dest, BasicType type) {
 787   assert(src->is_stack(), "should not call otherwise");
 788   assert(dest->is_register(), "should not call otherwise");
 789 
 790   Address addr = src->is_single_word() ?
 791     frame_map()->address_for_slot(src->single_stack_ix()) :
 792     frame_map()->address_for_slot(src->double_stack_ix());
 793 
 794 #ifndef AARCH64
 795   assert(lo_word_offset_in_bytes == 0 && hi_word_offset_in_bytes == 4, "little ending");
 796   if (dest->is_single_fpu() || dest->is_double_fpu()) {
 797     if (addr.disp() >= 1024) { BAILOUT("Too exotic case to handle here"); }
 798   }
 799 #endif // !AARCH64
 800 
 801   if (dest->is_single_cpu()) {
 802     switch (type) {
 803       case T_OBJECT:
 804       case T_ARRAY:
 805       case T_ADDRESS:
 806       case T_METADATA: __ ldr(dest->as_register(), addr); break;
 807       case T_FLOAT:    // used in floatToRawIntBits intrinsic implemenation
 808       case T_INT:      __ ldr_u32(dest->as_register(), addr); break;
 809       default:
 810         ShouldNotReachHere();
 811     }
 812     if ((type == T_OBJECT) || (type == T_ARRAY)) {
 813       __ verify_oop(dest->as_register());
 814     }
 815   } else if (dest->is_double_cpu()) {
 816     __ ldr(dest->as_register_lo(), addr);
 817 #ifndef AARCH64
 818     __ ldr(dest->as_register_hi(), frame_map()->address_for_slot(src->double_stack_ix(), hi_word_offset_in_bytes));
 819 #endif // !AARCH64
 820   } else if (dest->is_single_fpu()) {
 821     __ ldr_float(dest->as_float_reg(), addr);
 822   } else if (dest->is_double_fpu()) {
 823     __ ldr_double(dest->as_double_reg(), addr);
 824   } else {
 825     ShouldNotReachHere();
 826   }
 827 }
 828 
 829 
 830 void LIR_Assembler::stack2stack(LIR_Opr src, LIR_Opr dest, BasicType type) {
 831   if (src->is_single_stack()) {
 832     switch (src->type()) {
 833       case T_OBJECT:
 834       case T_ARRAY:
 835       case T_ADDRESS:
 836       case T_METADATA:
 837         __ ldr(Rtemp, frame_map()->address_for_slot(src->single_stack_ix()));
 838         __ str(Rtemp, frame_map()->address_for_slot(dest->single_stack_ix()));
 839         break;
 840 
 841       case T_INT:
 842       case T_FLOAT:
 843         __ ldr_u32(Rtemp, frame_map()->address_for_slot(src->single_stack_ix()));
 844         __ str_32(Rtemp, frame_map()->address_for_slot(dest->single_stack_ix()));
 845         break;
 846 
 847       default:
 848         ShouldNotReachHere();
 849     }
 850   } else {
 851     assert(src->is_double_stack(), "must be");
 852     __ ldr(Rtemp, frame_map()->address_for_slot(src->double_stack_ix(), lo_word_offset_in_bytes));
 853     __ str(Rtemp, frame_map()->address_for_slot(dest->double_stack_ix(), lo_word_offset_in_bytes));
 854 #ifdef AARCH64
 855     assert(lo_word_offset_in_bytes == 0, "adjust this code");
 856 #else
 857     __ ldr(Rtemp, frame_map()->address_for_slot(src->double_stack_ix(), hi_word_offset_in_bytes));
 858     __ str(Rtemp, frame_map()->address_for_slot(dest->double_stack_ix(), hi_word_offset_in_bytes));
 859 #endif // AARCH64
 860   }
 861 }
 862 
 863 
 864 void LIR_Assembler::mem2reg(LIR_Opr src, LIR_Opr dest, BasicType type,
 865                             LIR_PatchCode patch_code, CodeEmitInfo* info,
 866                             bool wide, bool unaligned) {
 867   assert(src->is_address(), "should not call otherwise");
 868   assert(dest->is_register(), "should not call otherwise");
 869   LIR_Address* addr = src->as_address_ptr();
 870 
 871   Register base_reg = addr->base()->as_pointer_register();
 872 
 873   PatchingStub* patch = NULL;
 874   if (patch_code != lir_patch_none) {
 875     patch = new PatchingStub(_masm, PatchingStub::access_field_id);
 876 #ifdef AARCH64
 877     // Extra nop for MT safe patching
 878     __ nop();
 879 #endif // AARCH64
 880   }
 881   if (info != NULL) {
 882     add_debug_info_for_null_check_here(info);
 883   }
 884 
 885   switch (type) {
 886     case T_OBJECT:  // fall through
 887     case T_ARRAY:
 888       if (UseCompressedOops && !wide) {
 889         __ ldr_u32(dest->as_register(), as_Address(addr));
 890       } else {
 891         __ ldr(dest->as_register(), as_Address(addr));
 892       }
 893       break;
 894 
 895     case T_ADDRESS:
 896       if (UseCompressedClassPointers && addr->disp() == oopDesc::klass_offset_in_bytes()) {
 897         __ ldr_u32(dest->as_pointer_register(), as_Address(addr));
 898       } else {
 899         __ ldr(dest->as_pointer_register(), as_Address(addr));
 900       }
 901       break;
 902 
 903 #ifdef AARCH64
 904     case T_LONG:
 905 #else
 906     case T_INT:
 907 #ifdef __SOFTFP__
 908     case T_FLOAT:
 909 #endif // __SOFTFP__
 910 #endif // AARCH64
 911       __ ldr(dest->as_pointer_register(), as_Address(addr));
 912       break;
 913 
 914     case T_BOOLEAN:
 915       __ ldrb(dest->as_register(), as_Address(addr));
 916       break;
 917 
 918     case T_BYTE:
 919       __ ldrsb(dest->as_register(), as_Address(addr));
 920       break;
 921 
 922     case T_CHAR:
 923       __ ldrh(dest->as_register(), as_Address(addr));
 924       break;
 925 
 926     case T_SHORT:
 927       __ ldrsh(dest->as_register(), as_Address(addr));
 928       break;
 929 
 930 #ifdef AARCH64
 931 
 932     case T_INT:
 933       __ ldr_w(dest->as_register(), as_Address(addr));
 934       break;
 935 
 936     case T_FLOAT:
 937       __ ldr_s(dest->as_float_reg(), as_Address(addr));
 938       break;
 939 
 940     case T_DOUBLE:
 941       __ ldr_d(dest->as_double_reg(), as_Address(addr));
 942       break;
 943 
 944 #else // AARCH64
 945 
 946 #ifdef __SOFTFP__
 947     case T_DOUBLE:
 948 #endif // __SOFTFP__
 949     case T_LONG: {
 950       Register to_lo = dest->as_register_lo();
 951       Register to_hi = dest->as_register_hi();
 952       if (addr->index()->is_register()) {
 953         assert(addr->scale() == LIR_Address::times_1,"Unexpected scaled register");
 954         assert(addr->disp() == 0, "Not yet supporting both");
 955         __ add(Rtemp, base_reg, addr->index()->as_register());
 956         base_reg = Rtemp;
 957         __ ldr(to_lo, Address(Rtemp));
 958         if (patch != NULL) {
 959           patching_epilog(patch, lir_patch_low, base_reg, info);
 960           patch = new PatchingStub(_masm, PatchingStub::access_field_id);
 961           patch_code = lir_patch_high;
 962         }
 963         __ ldr(to_hi, Address(Rtemp, BytesPerWord));
 964       } else if (base_reg == to_lo) {
 965         __ ldr(to_hi, as_Address_hi(addr));
 966         if (patch != NULL) {
 967           patching_epilog(patch, lir_patch_high, base_reg, info);
 968           patch = new PatchingStub(_masm, PatchingStub::access_field_id);
 969           patch_code = lir_patch_low;
 970         }
 971         __ ldr(to_lo, as_Address_lo(addr));
 972       } else {
 973         __ ldr(to_lo, as_Address_lo(addr));
 974         if (patch != NULL) {
 975           patching_epilog(patch, lir_patch_low, base_reg, info);
 976           patch = new PatchingStub(_masm, PatchingStub::access_field_id);
 977           patch_code = lir_patch_high;
 978         }
 979         __ ldr(to_hi, as_Address_hi(addr));
 980       }
 981       break;
 982     }
 983 
 984 #ifndef __SOFTFP__
 985     case T_FLOAT:
 986       if (addr->index()->is_register()) {
 987         assert(addr->scale() == LIR_Address::times_1,"Unexpected scaled register");
 988         __ add(Rtemp, base_reg, addr->index()->as_register());
 989         if ((addr->disp() <= -4096) || (addr->disp() >= 4096)) { BAILOUT("offset not in range"); }
 990         __ flds(dest->as_float_reg(), Address(Rtemp, addr->disp()));
 991       } else {
 992         __ flds(dest->as_float_reg(), as_Address(addr));
 993       }
 994       break;
 995 
 996     case T_DOUBLE:
 997       if (addr->index()->is_register()) {
 998         assert(addr->scale() == LIR_Address::times_1,"Unexpected scaled register");
 999         __ add(Rtemp, base_reg, addr->index()->as_register());
1000         if ((addr->disp() <= -4096) || (addr->disp() >= 4096)) { BAILOUT("offset not in range"); }
1001         __ fldd(dest->as_double_reg(), Address(Rtemp, addr->disp()));
1002       } else {
1003         __ fldd(dest->as_double_reg(), as_Address(addr));
1004       }
1005       break;
1006 #endif // __SOFTFP__
1007 
1008 #endif // AARCH64
1009 
1010     default:
1011       ShouldNotReachHere();
1012   }
1013 
1014   if (patch != NULL) {
1015     // Offset embeedded into LDR/STR instruction may appear not enough
1016     // to address a field. So, provide a space for one more instruction
1017     // that will deal with larger offsets.
1018     __ nop();
1019     patching_epilog(patch, patch_code, base_reg, info);
1020   }
1021 
1022 #ifdef AARCH64
1023   switch (type) {
1024     case T_ARRAY:
1025     case T_OBJECT:
1026       if (UseCompressedOops && !wide) {
1027         __ decode_heap_oop(dest->as_register());
1028       }
1029       __ verify_oop(dest->as_register());
1030       break;
1031 
1032     case T_ADDRESS:
1033       if (UseCompressedClassPointers && addr->disp() == oopDesc::klass_offset_in_bytes()) {
1034         __ decode_klass_not_null(dest->as_register());
1035       }
1036       break;
1037   }
1038 #endif // AARCH64
1039 }
1040 
1041 
1042 void LIR_Assembler::emit_op3(LIR_Op3* op) {
1043   bool is_32 = op->result_opr()->is_single_cpu();
1044 
1045   if (op->code() == lir_idiv && op->in_opr2()->is_constant() && is_32) {
1046     int c = op->in_opr2()->as_constant_ptr()->as_jint();
1047     assert(is_power_of_2(c), "non power-of-2 constant should be put in a register");
1048 
1049     Register left = op->in_opr1()->as_register();
1050     Register dest = op->result_opr()->as_register();
1051     if (c == 1) {
1052       __ mov(dest, left);
1053     } else if (c == 2) {
1054       __ add_32(dest, left, AsmOperand(left, lsr, 31));
1055       __ asr_32(dest, dest, 1);
1056     } else if (c != (int) 0x80000000) {
1057       int power = log2_intptr(c);
1058       __ asr_32(Rtemp, left, 31);
1059       __ add_32(dest, left, AsmOperand(Rtemp, lsr, 32-power)); // dest = left + (left < 0 ? 2^power - 1 : 0);
1060       __ asr_32(dest, dest, power);                            // dest = dest >>> power;
1061     } else {
1062       // x/0x80000000 is a special case, since dividend is a power of two, but is negative.
1063       // The only possible result values are 0 and 1, with 1 only for dividend == divisor == 0x80000000.
1064       __ cmp_32(left, c);
1065 #ifdef AARCH64
1066       __ cset(dest, eq);
1067 #else
1068       __ mov(dest, 0, ne);
1069       __ mov(dest, 1, eq);
1070 #endif // AARCH64
1071     }
1072   } else {
1073 #ifdef AARCH64
1074     Register left  = op->in_opr1()->as_pointer_register();
1075     Register right = op->in_opr2()->as_pointer_register();
1076     Register dest  = op->result_opr()->as_pointer_register();
1077 
1078     switch (op->code()) {
1079       case lir_idiv:
1080         if (is_32) {
1081           __ sdiv_w(dest, left, right);
1082         } else {
1083           __ sdiv(dest, left, right);
1084         }
1085         break;
1086       case lir_irem: {
1087         Register tmp = op->in_opr3()->as_pointer_register();
1088         assert_different_registers(left, tmp);
1089         assert_different_registers(right, tmp);
1090         if (is_32) {
1091           __ sdiv_w(tmp, left, right);
1092           __ msub_w(dest, right, tmp, left);
1093         } else {
1094           __ sdiv(tmp, left, right);
1095           __ msub(dest, right, tmp, left);
1096         }
1097         break;
1098       }
1099       default:
1100         ShouldNotReachHere();
1101     }
1102 #else
1103     assert(op->code() == lir_idiv || op->code() == lir_irem, "unexpected op3");
1104     __ call(StubRoutines::Arm::idiv_irem_entry(), relocInfo::runtime_call_type);
1105     add_debug_info_for_div0_here(op->info());
1106 #endif // AARCH64
1107   }
1108 }
1109 
1110 
1111 void LIR_Assembler::emit_opBranch(LIR_OpBranch* op) {
1112 #ifdef ASSERT
1113   assert(op->block() == NULL || op->block()->label() == op->label(), "wrong label");
1114   if (op->block() != NULL)  _branch_target_blocks.append(op->block());
1115   if (op->ublock() != NULL) _branch_target_blocks.append(op->ublock());
1116   assert(op->info() == NULL, "CodeEmitInfo?");
1117 #endif // ASSERT
1118 
1119 #ifdef __SOFTFP__
1120   assert (op->code() != lir_cond_float_branch, "this should be impossible");
1121 #else
1122   if (op->code() == lir_cond_float_branch) {
1123 #ifndef AARCH64
1124     __ fmstat();
1125 #endif // !AARCH64
1126     __ b(*(op->ublock()->label()), vs);
1127   }
1128 #endif // __SOFTFP__
1129 
1130   AsmCondition acond = al;
1131   switch (op->cond()) {
1132     case lir_cond_equal:        acond = eq; break;
1133     case lir_cond_notEqual:     acond = ne; break;
1134     case lir_cond_less:         acond = lt; break;
1135     case lir_cond_lessEqual:    acond = le; break;
1136     case lir_cond_greaterEqual: acond = ge; break;
1137     case lir_cond_greater:      acond = gt; break;
1138     case lir_cond_aboveEqual:   acond = hs; break;
1139     case lir_cond_belowEqual:   acond = ls; break;
1140     default: assert(op->cond() == lir_cond_always, "must be");
1141   }
1142   __ b(*(op->label()), acond);
1143 }
1144 
1145 
1146 void LIR_Assembler::emit_opConvert(LIR_OpConvert* op) {
1147   LIR_Opr src  = op->in_opr();
1148   LIR_Opr dest = op->result_opr();
1149 
1150   switch (op->bytecode()) {
1151     case Bytecodes::_i2l:
1152 #ifdef AARCH64
1153       __ sign_extend(dest->as_register_lo(), src->as_register(), 32);
1154 #else
1155       move_regs(src->as_register(), dest->as_register_lo());
1156       __ mov(dest->as_register_hi(), AsmOperand(src->as_register(), asr, 31));
1157 #endif // AARCH64
1158       break;
1159     case Bytecodes::_l2i:
1160       move_regs(src->as_register_lo(), dest->as_register());
1161       break;
1162     case Bytecodes::_i2b:
1163       __ sign_extend(dest->as_register(), src->as_register(), 8);
1164       break;
1165     case Bytecodes::_i2s:
1166       __ sign_extend(dest->as_register(), src->as_register(), 16);
1167       break;
1168     case Bytecodes::_i2c:
1169       __ zero_extend(dest->as_register(), src->as_register(), 16);
1170       break;
1171     case Bytecodes::_f2d:
1172       __ convert_f2d(dest->as_double_reg(), src->as_float_reg());
1173       break;
1174     case Bytecodes::_d2f:
1175       __ convert_d2f(dest->as_float_reg(), src->as_double_reg());
1176       break;
1177     case Bytecodes::_i2f:
1178 #ifdef AARCH64
1179       __ scvtf_sw(dest->as_float_reg(), src->as_register());
1180 #else
1181       __ fmsr(Stemp, src->as_register());
1182       __ fsitos(dest->as_float_reg(), Stemp);
1183 #endif // AARCH64
1184       break;
1185     case Bytecodes::_i2d:
1186 #ifdef AARCH64
1187       __ scvtf_dw(dest->as_double_reg(), src->as_register());
1188 #else
1189       __ fmsr(Stemp, src->as_register());
1190       __ fsitod(dest->as_double_reg(), Stemp);
1191 #endif // AARCH64
1192       break;
1193     case Bytecodes::_f2i:
1194 #ifdef AARCH64
1195       __ fcvtzs_ws(dest->as_register(), src->as_float_reg());
1196 #else
1197       __ ftosizs(Stemp, src->as_float_reg());
1198       __ fmrs(dest->as_register(), Stemp);
1199 #endif // AARCH64
1200       break;
1201     case Bytecodes::_d2i:
1202 #ifdef AARCH64
1203       __ fcvtzs_wd(dest->as_register(), src->as_double_reg());
1204 #else
1205       __ ftosizd(Stemp, src->as_double_reg());
1206       __ fmrs(dest->as_register(), Stemp);
1207 #endif // AARCH64
1208       break;
1209 #ifdef AARCH64
1210     case Bytecodes::_l2f:
1211       __ scvtf_sx(dest->as_float_reg(), src->as_register_lo());
1212       break;
1213     case Bytecodes::_l2d:
1214       __ scvtf_dx(dest->as_double_reg(), src->as_register_lo());
1215       break;
1216     case Bytecodes::_f2l:
1217       __ fcvtzs_xs(dest->as_register_lo(), src->as_float_reg());
1218       break;
1219     case Bytecodes::_d2l:
1220       __ fcvtzs_xd(dest->as_register_lo(), src->as_double_reg());
1221       break;
1222 #endif // AARCH64
1223     default:
1224       ShouldNotReachHere();
1225   }
1226 }
1227 
1228 
1229 void LIR_Assembler::emit_alloc_obj(LIR_OpAllocObj* op) {
1230   if (op->init_check()) {
1231     Register tmp = op->tmp1()->as_register();
1232     __ ldrb(tmp, Address(op->klass()->as_register(), InstanceKlass::init_state_offset()));
1233     add_debug_info_for_null_check_here(op->stub()->info());
1234     __ cmp(tmp, InstanceKlass::fully_initialized);
1235     __ b(*op->stub()->entry(), ne);
1236   }
1237   __ allocate_object(op->obj()->as_register(),
1238                      op->tmp1()->as_register(),
1239                      op->tmp2()->as_register(),
1240                      op->tmp3()->as_register(),
1241                      op->header_size(),
1242                      op->object_size(),
1243                      op->klass()->as_register(),
1244                      *op->stub()->entry());
1245   __ bind(*op->stub()->continuation());
1246 }
1247 
1248 void LIR_Assembler::emit_alloc_array(LIR_OpAllocArray* op) {
1249   if (UseSlowPath ||
1250       (!UseFastNewObjectArray && (op->type() == T_OBJECT || op->type() == T_ARRAY)) ||
1251       (!UseFastNewTypeArray   && (op->type() != T_OBJECT && op->type() != T_ARRAY))) {
1252     __ b(*op->stub()->entry());
1253   } else {
1254     __ allocate_array(op->obj()->as_register(),
1255                       op->len()->as_register(),
1256                       op->tmp1()->as_register(),
1257                       op->tmp2()->as_register(),
1258                       op->tmp3()->as_register(),
1259                       arrayOopDesc::header_size(op->type()),
1260                       type2aelembytes(op->type()),
1261                       op->klass()->as_register(),
1262                       *op->stub()->entry());
1263   }
1264   __ bind(*op->stub()->continuation());
1265 }
1266 
1267 void LIR_Assembler::type_profile_helper(Register mdo, int mdo_offset_bias,
1268                                         ciMethodData *md, ciProfileData *data,
1269                                         Register recv, Register tmp1, Label* update_done) {
1270   assert_different_registers(mdo, recv, tmp1);
1271   uint i;
1272   for (i = 0; i < VirtualCallData::row_limit(); i++) {
1273     Label next_test;
1274     // See if the receiver is receiver[n].
1275     Address receiver_addr(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_offset(i)) -
1276                           mdo_offset_bias);
1277     __ ldr(tmp1, receiver_addr);
1278     __ verify_klass_ptr(tmp1);
1279     __ cmp(recv, tmp1);
1280     __ b(next_test, ne);
1281     Address data_addr(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_count_offset(i)) -
1282                       mdo_offset_bias);
1283     __ ldr(tmp1, data_addr);
1284     __ add(tmp1, tmp1, DataLayout::counter_increment);
1285     __ str(tmp1, data_addr);
1286     __ b(*update_done);
1287     __ bind(next_test);
1288   }
1289 
1290   // Didn't find receiver; find next empty slot and fill it in
1291   for (i = 0; i < VirtualCallData::row_limit(); i++) {
1292     Label next_test;
1293     Address recv_addr(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_offset(i)) -
1294                       mdo_offset_bias);
1295     __ ldr(tmp1, recv_addr);
1296     __ cbnz(tmp1, next_test);
1297     __ str(recv, recv_addr);
1298     __ mov(tmp1, DataLayout::counter_increment);
1299     __ str(tmp1, Address(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_count_offset(i)) -
1300                          mdo_offset_bias));
1301     __ b(*update_done);
1302     __ bind(next_test);
1303   }
1304 }
1305 
1306 void LIR_Assembler::setup_md_access(ciMethod* method, int bci,
1307                                     ciMethodData*& md, ciProfileData*& data, int& mdo_offset_bias) {
1308   md = method->method_data_or_null();
1309   assert(md != NULL, "Sanity");
1310   data = md->bci_to_data(bci);
1311   assert(data != NULL,       "need data for checkcast");
1312   assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
1313   if (md->byte_offset_of_slot(data, DataLayout::header_offset()) + data->size_in_bytes() >= 4096) {
1314     // The offset is large so bias the mdo by the base of the slot so
1315     // that the ldr can use an immediate offset to reference the slots of the data
1316     mdo_offset_bias = md->byte_offset_of_slot(data, DataLayout::header_offset());
1317   }
1318 }
1319 
1320 // On 32-bit ARM, code before this helper should test obj for null (ZF should be set if obj is null).
1321 void LIR_Assembler::typecheck_profile_helper1(ciMethod* method, int bci,
1322                                               ciMethodData*& md, ciProfileData*& data, int& mdo_offset_bias,
1323                                               Register obj, Register mdo, Register data_val, Label* obj_is_null) {
1324   assert(method != NULL, "Should have method");
1325   assert_different_registers(obj, mdo, data_val);
1326   setup_md_access(method, bci, md, data, mdo_offset_bias);
1327   Label not_null;
1328 #ifdef AARCH64
1329   __ cbnz(obj, not_null);
1330 #else
1331   __ b(not_null, ne);
1332 #endif // AARCH64
1333   __ mov_metadata(mdo, md->constant_encoding());
1334   if (mdo_offset_bias > 0) {
1335     __ mov_slow(data_val, mdo_offset_bias);
1336     __ add(mdo, mdo, data_val);
1337   }
1338   Address flags_addr(mdo, md->byte_offset_of_slot(data, DataLayout::flags_offset()) - mdo_offset_bias);
1339   __ ldrb(data_val, flags_addr);
1340   __ orr(data_val, data_val, (uint)BitData::null_seen_byte_constant());
1341   __ strb(data_val, flags_addr);
1342   __ b(*obj_is_null);
1343   __ bind(not_null);
1344 }
1345 
1346 void LIR_Assembler::typecheck_profile_helper2(ciMethodData* md, ciProfileData* data, int mdo_offset_bias,
1347                                               Register mdo, Register recv, Register value, Register tmp1,
1348                                               Label* profile_cast_success, Label* profile_cast_failure,
1349                                               Label* success, Label* failure) {
1350   assert_different_registers(mdo, value, tmp1);
1351   __ bind(*profile_cast_success);
1352   __ mov_metadata(mdo, md->constant_encoding());
1353   if (mdo_offset_bias > 0) {
1354     __ mov_slow(tmp1, mdo_offset_bias);
1355     __ add(mdo, mdo, tmp1);
1356   }
1357   __ load_klass(recv, value);
1358   type_profile_helper(mdo, mdo_offset_bias, md, data, recv, tmp1, success);
1359   __ b(*success);
1360   // Cast failure case
1361   __ bind(*profile_cast_failure);
1362   __ mov_metadata(mdo, md->constant_encoding());
1363   if (mdo_offset_bias > 0) {
1364     __ mov_slow(tmp1, mdo_offset_bias);
1365     __ add(mdo, mdo, tmp1);
1366   }
1367   Address data_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()) - mdo_offset_bias);
1368   __ ldr(tmp1, data_addr);
1369   __ sub(tmp1, tmp1, DataLayout::counter_increment);
1370   __ str(tmp1, data_addr);
1371   __ b(*failure);
1372 }
1373 
1374 // Sets `res` to true, if `cond` holds. On AArch64 also sets `res` to false if `cond` does not hold.
1375 static void set_instanceof_result(MacroAssembler* _masm, Register res, AsmCondition cond) {
1376 #ifdef AARCH64
1377   __ cset(res, cond);
1378 #else
1379   __ mov(res, 1, cond);
1380 #endif // AARCH64
1381 }
1382 
1383 
1384 void LIR_Assembler::emit_opTypeCheck(LIR_OpTypeCheck* op) {
1385   // TODO: ARM - can be more effective with one more register
1386   switch (op->code()) {
1387     case lir_store_check: {
1388       CodeStub* stub = op->stub();
1389       Register value = op->object()->as_register();
1390       Register array = op->array()->as_register();
1391       Register klass_RInfo = op->tmp1()->as_register();
1392       Register k_RInfo = op->tmp2()->as_register();
1393       assert_different_registers(klass_RInfo, k_RInfo, Rtemp);
1394       if (op->should_profile()) {
1395         assert_different_registers(value, klass_RInfo, k_RInfo, Rtemp);
1396       }
1397 
1398       // check if it needs to be profiled
1399       ciMethodData* md;
1400       ciProfileData* data;
1401       int mdo_offset_bias = 0;
1402       Label profile_cast_success, profile_cast_failure, done;
1403       Label *success_target = op->should_profile() ? &profile_cast_success : &done;
1404       Label *failure_target = op->should_profile() ? &profile_cast_failure : stub->entry();
1405 
1406       if (op->should_profile()) {
1407 #ifndef AARCH64
1408         __ cmp(value, 0);
1409 #endif // !AARCH64
1410         typecheck_profile_helper1(op->profiled_method(), op->profiled_bci(), md, data, mdo_offset_bias, value, k_RInfo, Rtemp, &done);
1411       } else {
1412         __ cbz(value, done);
1413       }
1414       assert_different_registers(k_RInfo, value);
1415       add_debug_info_for_null_check_here(op->info_for_exception());
1416       __ load_klass(k_RInfo, array);
1417       __ load_klass(klass_RInfo, value);
1418       __ ldr(k_RInfo, Address(k_RInfo, ObjArrayKlass::element_klass_offset()));
1419       __ ldr_u32(Rtemp, Address(k_RInfo, Klass::super_check_offset_offset()));
1420       // check for immediate positive hit
1421       __ ldr(Rtemp, Address(klass_RInfo, Rtemp));
1422       __ cmp(klass_RInfo, k_RInfo);
1423       __ cond_cmp(Rtemp, k_RInfo, ne);
1424       __ b(*success_target, eq);
1425       // check for immediate negative hit
1426       __ ldr_u32(Rtemp, Address(k_RInfo, Klass::super_check_offset_offset()));
1427       __ cmp(Rtemp, in_bytes(Klass::secondary_super_cache_offset()));
1428       __ b(*failure_target, ne);
1429       // slow case
1430       assert(klass_RInfo == R0 && k_RInfo == R1, "runtime call setup");
1431       __ call(Runtime1::entry_for(Runtime1::slow_subtype_check_id), relocInfo::runtime_call_type);
1432       __ cbz(R0, *failure_target);
1433       if (op->should_profile()) {
1434         Register mdo  = klass_RInfo, recv = k_RInfo, tmp1 = Rtemp;
1435         if (mdo == value) {
1436           mdo = k_RInfo;
1437           recv = klass_RInfo;
1438         }
1439         typecheck_profile_helper2(md, data, mdo_offset_bias, mdo, recv, value, tmp1,
1440                                   &profile_cast_success, &profile_cast_failure,
1441                                   &done, stub->entry());
1442       }
1443       __ bind(done);
1444       break;
1445     }
1446 
1447     case lir_checkcast: {
1448       CodeStub* stub = op->stub();
1449       Register obj = op->object()->as_register();
1450       Register res = op->result_opr()->as_register();
1451       Register klass_RInfo = op->tmp1()->as_register();
1452       Register k_RInfo = op->tmp2()->as_register();
1453       ciKlass* k = op->klass();
1454       assert_different_registers(res, k_RInfo, klass_RInfo, Rtemp);
1455 
1456       // TODO: ARM - Late binding is used to prevent confusion of register allocator
1457       assert(stub->is_exception_throw_stub(), "must be");
1458       ((SimpleExceptionStub*)stub)->set_obj(op->result_opr());
1459 
1460       ciMethodData* md;
1461       ciProfileData* data;
1462       int mdo_offset_bias = 0;
1463 
1464       Label done;
1465 
1466       Label profile_cast_failure, profile_cast_success;
1467       Label *failure_target = op->should_profile() ? &profile_cast_failure : op->stub()->entry();
1468       Label *success_target = op->should_profile() ? &profile_cast_success : &done;
1469 
1470 #ifdef AARCH64
1471       move_regs(obj, res);
1472       if (op->should_profile()) {
1473         typecheck_profile_helper1(op->profiled_method(), op->profiled_bci(), md, data, mdo_offset_bias, res, klass_RInfo, Rtemp, &done);
1474       } else {
1475         __ cbz(obj, done);
1476       }
1477       if (k->is_loaded()) {
1478         __ mov_metadata(k_RInfo, k->constant_encoding());
1479       } else {
1480         if (res != obj) {
1481           op->info_for_patch()->add_register_oop(FrameMap::as_oop_opr(res));
1482         }
1483         klass2reg_with_patching(k_RInfo, op->info_for_patch());
1484       }
1485       __ load_klass(klass_RInfo, res);
1486 
1487       if (op->fast_check()) {
1488         __ cmp(klass_RInfo, k_RInfo);
1489         __ b(*failure_target, ne);
1490       } else if (k->is_loaded()) {
1491         __ ldr(Rtemp, Address(klass_RInfo, k->super_check_offset()));
1492         if (in_bytes(Klass::secondary_super_cache_offset()) != (int) k->super_check_offset()) {
1493           __ cmp(Rtemp, k_RInfo);
1494           __ b(*failure_target, ne);
1495         } else {
1496           __ cmp(klass_RInfo, k_RInfo);
1497           __ cond_cmp(Rtemp, k_RInfo, ne);
1498           __ b(*success_target, eq);
1499           assert(klass_RInfo == R0 && k_RInfo == R1, "runtime call setup");
1500           __ call(Runtime1::entry_for(Runtime1::slow_subtype_check_id), relocInfo::runtime_call_type);
1501           __ cbz(R0, *failure_target);
1502         }
1503       } else {
1504         __ ldr_u32(Rtemp, Address(k_RInfo, Klass::super_check_offset_offset()));
1505         // check for immediate positive hit
1506         __ ldr(Rtemp, Address(klass_RInfo, Rtemp));
1507         __ cmp(klass_RInfo, k_RInfo);
1508         __ cond_cmp(Rtemp, k_RInfo, ne);
1509         __ b(*success_target, eq);
1510         // check for immediate negative hit
1511         __ ldr_u32(Rtemp, Address(k_RInfo, Klass::super_check_offset_offset()));
1512         __ cmp(Rtemp, in_bytes(Klass::secondary_super_cache_offset()));
1513         __ b(*failure_target, ne);
1514         // slow case
1515         assert(klass_RInfo == R0 && k_RInfo == R1, "runtime call setup");
1516         __ call(Runtime1::entry_for(Runtime1::slow_subtype_check_id), relocInfo::runtime_call_type);
1517         __ cbz(R0, *failure_target);
1518       }
1519 
1520 #else // AARCH64
1521 
1522       __ movs(res, obj);
1523       if (op->should_profile()) {
1524         typecheck_profile_helper1(op->profiled_method(), op->profiled_bci(), md, data, mdo_offset_bias, res, klass_RInfo, Rtemp, &done);
1525       } else {
1526         __ b(done, eq);
1527       }
1528       if (k->is_loaded()) {
1529         __ mov_metadata(k_RInfo, k->constant_encoding());
1530       } else if (k_RInfo != obj) {
1531         klass2reg_with_patching(k_RInfo, op->info_for_patch());
1532         __ movs(res, obj);
1533       } else {
1534         // Patching doesn't update "res" register after GC, so do patching first
1535         klass2reg_with_patching(Rtemp, op->info_for_patch());
1536         __ movs(res, obj);
1537         __ mov(k_RInfo, Rtemp);
1538       }
1539       __ load_klass(klass_RInfo, res, ne);
1540 
1541       if (op->fast_check()) {
1542         __ cmp(klass_RInfo, k_RInfo, ne);
1543         __ b(*failure_target, ne);
1544       } else if (k->is_loaded()) {
1545         __ b(*success_target, eq);
1546         __ ldr(Rtemp, Address(klass_RInfo, k->super_check_offset()));
1547         if (in_bytes(Klass::secondary_super_cache_offset()) != (int) k->super_check_offset()) {
1548           __ cmp(Rtemp, k_RInfo);
1549           __ b(*failure_target, ne);
1550         } else {
1551           __ cmp(klass_RInfo, k_RInfo);
1552           __ cmp(Rtemp, k_RInfo, ne);
1553           __ b(*success_target, eq);
1554           assert(klass_RInfo == R0 && k_RInfo == R1, "runtime call setup");
1555           __ call(Runtime1::entry_for(Runtime1::slow_subtype_check_id), relocInfo::runtime_call_type);
1556           __ cbz(R0, *failure_target);
1557         }
1558       } else {
1559         __ ldr_u32(Rtemp, Address(k_RInfo, Klass::super_check_offset_offset()));
1560         __ b(*success_target, eq);
1561         // check for immediate positive hit
1562         __ ldr(Rtemp, Address(klass_RInfo, Rtemp));
1563         __ cmp(klass_RInfo, k_RInfo);
1564         __ cmp(Rtemp, k_RInfo, ne);
1565         __ b(*success_target, eq);
1566         // check for immediate negative hit
1567         __ ldr_u32(Rtemp, Address(k_RInfo, Klass::super_check_offset_offset()));
1568         __ cmp(Rtemp, in_bytes(Klass::secondary_super_cache_offset()));
1569         __ b(*failure_target, ne);
1570         // slow case
1571         assert(klass_RInfo == R0 && k_RInfo == R1, "runtime call setup");
1572         __ call(Runtime1::entry_for(Runtime1::slow_subtype_check_id), relocInfo::runtime_call_type);
1573         __ cbz(R0, *failure_target);
1574       }
1575 #endif // AARCH64
1576 
1577       if (op->should_profile()) {
1578         Register mdo  = klass_RInfo, recv = k_RInfo, tmp1 = Rtemp;
1579         typecheck_profile_helper2(md, data, mdo_offset_bias, mdo, recv, res, tmp1,
1580                                   &profile_cast_success, &profile_cast_failure,
1581                                   &done, stub->entry());
1582       }
1583       __ bind(done);
1584       break;
1585     }
1586 
1587     case lir_instanceof: {
1588       Register obj = op->object()->as_register();
1589       Register res = op->result_opr()->as_register();
1590       Register klass_RInfo = op->tmp1()->as_register();
1591       Register k_RInfo = op->tmp2()->as_register();
1592       ciKlass* k = op->klass();
1593       assert_different_registers(res, klass_RInfo, k_RInfo, Rtemp);
1594 
1595       ciMethodData* md;
1596       ciProfileData* data;
1597       int mdo_offset_bias = 0;
1598 
1599       Label done;
1600 
1601       Label profile_cast_failure, profile_cast_success;
1602       Label *failure_target = op->should_profile() ? &profile_cast_failure : &done;
1603       Label *success_target = op->should_profile() ? &profile_cast_success : &done;
1604 
1605 #ifdef AARCH64
1606       move_regs(obj, res);
1607 #else
1608       __ movs(res, obj);
1609 #endif // AARCH64
1610 
1611       if (op->should_profile()) {
1612         typecheck_profile_helper1(op->profiled_method(), op->profiled_bci(), md, data, mdo_offset_bias, res, klass_RInfo, Rtemp, &done);
1613       } else {
1614 #ifdef AARCH64
1615         __ cbz(obj, done); // If obj == NULL, res is false
1616 #else
1617         __ b(done, eq);
1618 #endif // AARCH64
1619       }
1620 
1621       if (k->is_loaded()) {
1622         __ mov_metadata(k_RInfo, k->constant_encoding());
1623       } else {
1624         op->info_for_patch()->add_register_oop(FrameMap::as_oop_opr(res));
1625         klass2reg_with_patching(k_RInfo, op->info_for_patch());
1626       }
1627       __ load_klass(klass_RInfo, res);
1628 
1629 #ifndef AARCH64
1630       if (!op->should_profile()) {
1631         __ mov(res, 0);
1632       }
1633 #endif // !AARCH64
1634 
1635       if (op->fast_check()) {
1636         __ cmp(klass_RInfo, k_RInfo);
1637         if (!op->should_profile()) {
1638           set_instanceof_result(_masm, res, eq);
1639         } else {
1640           __ b(profile_cast_failure, ne);
1641         }
1642       } else if (k->is_loaded()) {
1643         __ ldr(Rtemp, Address(klass_RInfo, k->super_check_offset()));
1644         if (in_bytes(Klass::secondary_super_cache_offset()) != (int) k->super_check_offset()) {
1645           __ cmp(Rtemp, k_RInfo);
1646           if (!op->should_profile()) {
1647             set_instanceof_result(_masm, res, eq);
1648           } else {
1649             __ b(profile_cast_failure, ne);
1650           }
1651         } else {
1652           __ cmp(klass_RInfo, k_RInfo);
1653           __ cond_cmp(Rtemp, k_RInfo, ne);
1654           if (!op->should_profile()) {
1655             set_instanceof_result(_masm, res, eq);
1656           }
1657           __ b(*success_target, eq);
1658           assert(klass_RInfo == R0 && k_RInfo == R1, "runtime call setup");
1659           __ call(Runtime1::entry_for(Runtime1::slow_subtype_check_id), relocInfo::runtime_call_type);
1660           if (!op->should_profile()) {
1661             move_regs(R0, res);
1662           } else {
1663             __ cbz(R0, *failure_target);
1664           }
1665         }
1666       } else {
1667         __ ldr_u32(Rtemp, Address(k_RInfo, Klass::super_check_offset_offset()));
1668         // check for immediate positive hit
1669         __ cmp(klass_RInfo, k_RInfo);
1670         if (!op->should_profile()) {
1671 #ifdef AARCH64
1672           // TODO-AARCH64 check if separate conditional branch is more efficient than ldr+cond_cmp
1673           __ ldr(res, Address(klass_RInfo, Rtemp));
1674 #else
1675           __ ldr(res, Address(klass_RInfo, Rtemp), ne);
1676 #endif // AARCH64
1677           __ cond_cmp(res, k_RInfo, ne);
1678           set_instanceof_result(_masm, res, eq);
1679         } else {
1680 #ifdef AARCH64
1681           // TODO-AARCH64 check if separate conditional branch is more efficient than ldr+cond_cmp
1682           __ ldr(Rtemp, Address(klass_RInfo, Rtemp));
1683 #else
1684           __ ldr(Rtemp, Address(klass_RInfo, Rtemp), ne);
1685 #endif // AARCH64
1686           __ cond_cmp(Rtemp, k_RInfo, ne);
1687         }
1688         __ b(*success_target, eq);
1689         // check for immediate negative hit
1690         if (op->should_profile()) {
1691           __ ldr_u32(Rtemp, Address(k_RInfo, Klass::super_check_offset_offset()));
1692         }
1693         __ cmp(Rtemp, in_bytes(Klass::secondary_super_cache_offset()));
1694         if (!op->should_profile()) {
1695 #ifdef AARCH64
1696           __ mov(res, 0);
1697 #else
1698           __ mov(res, 0, ne);
1699 #endif // AARCH64
1700         }
1701         __ b(*failure_target, ne);
1702         // slow case
1703         assert(klass_RInfo == R0 && k_RInfo == R1, "runtime call setup");
1704         __ call(Runtime1::entry_for(Runtime1::slow_subtype_check_id), relocInfo::runtime_call_type);
1705         if (!op->should_profile()) {
1706           move_regs(R0, res);
1707         }
1708         if (op->should_profile()) {
1709           __ cbz(R0, *failure_target);
1710         }
1711       }
1712 
1713       if (op->should_profile()) {
1714         Label done_ok, done_failure;
1715         Register mdo  = klass_RInfo, recv = k_RInfo, tmp1 = Rtemp;
1716         typecheck_profile_helper2(md, data, mdo_offset_bias, mdo, recv, res, tmp1,
1717                                   &profile_cast_success, &profile_cast_failure,
1718                                   &done_ok, &done_failure);
1719         __ bind(done_failure);
1720         __ mov(res, 0);
1721         __ b(done);
1722         __ bind(done_ok);
1723         __ mov(res, 1);
1724       }
1725       __ bind(done);
1726       break;
1727     }
1728     default:
1729       ShouldNotReachHere();
1730   }
1731 }
1732 
1733 
1734 void LIR_Assembler::emit_compare_and_swap(LIR_OpCompareAndSwap* op) {
1735   //   if (*addr == cmpval) {
1736   //     *addr = newval;
1737   //     dest = 1;
1738   //   } else {
1739   //     dest = 0;
1740   //   }
1741 #ifdef AARCH64
1742   Label retry, done;
1743   Register addr = op->addr()->as_pointer_register();
1744   Register cmpval = op->cmp_value()->as_pointer_register();
1745   Register newval = op->new_value()->as_pointer_register();
1746   Register dest = op->result_opr()->as_pointer_register();
1747   assert_different_registers(dest, addr, cmpval, newval, Rtemp);
1748 
1749   if (UseCompressedOops && op->code() == lir_cas_obj) {
1750     Register tmp1 = op->tmp1()->as_pointer_register();
1751     Register tmp2 = op->tmp2()->as_pointer_register();
1752     assert_different_registers(dest, addr, cmpval, newval, tmp1, tmp2, Rtemp);
1753     __ encode_heap_oop(tmp1, cmpval); cmpval = tmp1;
1754     __ encode_heap_oop(tmp2, newval); newval = tmp2;
1755   }
1756 
1757   __ mov(dest, ZR);
1758   __ bind(retry);
1759   if (((op->code() == lir_cas_obj) && !UseCompressedOops) || op->code() == lir_cas_long) {
1760     __ ldaxr(Rtemp, addr);
1761     __ cmp(Rtemp, cmpval);
1762     __ b(done, ne);
1763     __ stlxr(Rtemp, newval, addr);
1764   } else if (((op->code() == lir_cas_obj) && UseCompressedOops) || op->code() == lir_cas_int) {
1765     __ ldaxr_w(Rtemp, addr);
1766     __ cmp_w(Rtemp, cmpval);
1767     __ b(done, ne);
1768     __ stlxr_w(Rtemp, newval, addr);
1769   } else {
1770     ShouldNotReachHere();
1771   }
1772   __ cbnz_w(Rtemp, retry);
1773   __ mov(dest, 1);
1774   __ bind(done);
1775 #else
1776   // FIXME: membar_release
1777   __ membar(MacroAssembler::Membar_mask_bits(MacroAssembler::StoreStore | MacroAssembler::LoadStore), Rtemp);
1778   if (op->code() == lir_cas_int || op->code() == lir_cas_obj) {
1779     Register addr = op->addr()->as_register();
1780     Register cmpval = op->cmp_value()->as_register();
1781     Register newval = op->new_value()->as_register();
1782     Register dest = op->result_opr()->as_register();
1783     assert_different_registers(dest, addr, cmpval, newval, Rtemp);
1784 
1785     __ atomic_cas_bool(cmpval, newval, addr, 0, Rtemp); // Rtemp free by default at C1 LIR layer
1786     __ mov(dest, 1, eq);
1787     __ mov(dest, 0, ne);
1788   } else if (op->code() == lir_cas_long) {
1789     assert(VM_Version::supports_cx8(), "wrong machine");
1790     Register addr = op->addr()->as_pointer_register();
1791     Register cmp_value_lo = op->cmp_value()->as_register_lo();
1792     Register cmp_value_hi = op->cmp_value()->as_register_hi();
1793     Register new_value_lo = op->new_value()->as_register_lo();
1794     Register new_value_hi = op->new_value()->as_register_hi();
1795     Register dest = op->result_opr()->as_register();
1796     Register tmp_lo = op->tmp1()->as_register_lo();
1797     Register tmp_hi = op->tmp1()->as_register_hi();
1798 
1799     assert_different_registers(tmp_lo, tmp_hi, cmp_value_lo, cmp_value_hi, dest, new_value_lo, new_value_hi, addr);
1800     assert(tmp_hi->encoding() == tmp_lo->encoding() + 1, "non aligned register pair");
1801     assert(new_value_hi->encoding() == new_value_lo->encoding() + 1, "non aligned register pair");
1802     assert((tmp_lo->encoding() & 0x1) == 0, "misaligned register pair");
1803     assert((new_value_lo->encoding() & 0x1) == 0, "misaligned register pair");
1804     __ atomic_cas64(tmp_lo, tmp_hi, dest, cmp_value_lo, cmp_value_hi,
1805                     new_value_lo, new_value_hi, addr, 0);
1806   } else {
1807     Unimplemented();
1808   }
1809 #endif // AARCH64
1810   // FIXME: is full membar really needed instead of just membar_acquire?
1811   __ membar(MacroAssembler::Membar_mask_bits(MacroAssembler::StoreLoad | MacroAssembler::StoreStore), Rtemp);
1812 }
1813 
1814 
1815 void LIR_Assembler::cmove(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Opr result, BasicType type) {
1816   AsmCondition acond = al;
1817   AsmCondition ncond = nv;
1818   if (opr1 != opr2) {
1819     switch (condition) {
1820       case lir_cond_equal:        acond = eq; ncond = ne; break;
1821       case lir_cond_notEqual:     acond = ne; ncond = eq; break;
1822       case lir_cond_less:         acond = lt; ncond = ge; break;
1823       case lir_cond_lessEqual:    acond = le; ncond = gt; break;
1824       case lir_cond_greaterEqual: acond = ge; ncond = lt; break;
1825       case lir_cond_greater:      acond = gt; ncond = le; break;
1826       case lir_cond_aboveEqual:   acond = hs; ncond = lo; break;
1827       case lir_cond_belowEqual:   acond = ls; ncond = hi; break;
1828       default: ShouldNotReachHere();
1829     }
1830   }
1831 
1832 #ifdef AARCH64
1833 
1834   // TODO-AARCH64 implement it more efficiently
1835 
1836   if (opr1->is_register()) {
1837     reg2reg(opr1, result);
1838   } else if (opr1->is_stack()) {
1839     stack2reg(opr1, result, result->type());
1840   } else if (opr1->is_constant()) {
1841     const2reg(opr1, result, lir_patch_none, NULL);
1842   } else {
1843     ShouldNotReachHere();
1844   }
1845 
1846   Label skip;
1847   __ b(skip, acond);
1848 
1849   if (opr2->is_register()) {
1850     reg2reg(opr2, result);
1851   } else if (opr2->is_stack()) {
1852     stack2reg(opr2, result, result->type());
1853   } else if (opr2->is_constant()) {
1854     const2reg(opr2, result, lir_patch_none, NULL);
1855   } else {
1856     ShouldNotReachHere();
1857   }
1858 
1859   __ bind(skip);
1860 
1861 #else
1862   for (;;) {                         // two iterations only
1863     if (opr1 == result) {
1864       // do nothing
1865     } else if (opr1->is_single_cpu()) {
1866       __ mov(result->as_register(), opr1->as_register(), acond);
1867     } else if (opr1->is_double_cpu()) {
1868       __ long_move(result->as_register_lo(), result->as_register_hi(),
1869                    opr1->as_register_lo(), opr1->as_register_hi(), acond);
1870     } else if (opr1->is_single_stack()) {
1871       __ ldr(result->as_register(), frame_map()->address_for_slot(opr1->single_stack_ix()), acond);
1872     } else if (opr1->is_double_stack()) {
1873       __ ldr(result->as_register_lo(),
1874              frame_map()->address_for_slot(opr1->double_stack_ix(), lo_word_offset_in_bytes), acond);
1875       __ ldr(result->as_register_hi(),
1876              frame_map()->address_for_slot(opr1->double_stack_ix(), hi_word_offset_in_bytes), acond);
1877     } else if (opr1->is_illegal()) {
1878       // do nothing: this part of the cmove has been optimized away in the peephole optimizer
1879     } else {
1880       assert(opr1->is_constant(), "must be");
1881       LIR_Const* c = opr1->as_constant_ptr();
1882 
1883       switch (c->type()) {
1884         case T_INT:
1885           __ mov_slow(result->as_register(), c->as_jint(), acond);
1886           break;
1887         case T_LONG:
1888           __ mov_slow(result->as_register_lo(), c->as_jint_lo(), acond);
1889           __ mov_slow(result->as_register_hi(), c->as_jint_hi(), acond);
1890           break;
1891         case T_OBJECT:
1892           __ mov_oop(result->as_register(), c->as_jobject(), 0, acond);
1893           break;
1894         case T_FLOAT:
1895 #ifdef __SOFTFP__
1896           // not generated now.
1897           __ mov_slow(result->as_register(), c->as_jint(), acond);
1898 #else
1899           __ mov_float(result->as_float_reg(), c->as_jfloat(), acond);
1900 #endif // __SOFTFP__
1901           break;
1902         case T_DOUBLE:
1903 #ifdef __SOFTFP__
1904           // not generated now.
1905           __ mov_slow(result->as_register_lo(), c->as_jint_lo(), acond);
1906           __ mov_slow(result->as_register_hi(), c->as_jint_hi(), acond);
1907 #else
1908           __ mov_double(result->as_double_reg(), c->as_jdouble(), acond);
1909 #endif // __SOFTFP__
1910           break;
1911         default:
1912           ShouldNotReachHere();
1913       }
1914     }
1915 
1916     // Negate the condition and repeat the algorithm with the second operand
1917     if (opr1 == opr2) { break; }
1918     opr1 = opr2;
1919     acond = ncond;
1920   }
1921 #endif // AARCH64
1922 }
1923 
1924 #if defined(AARCH64) || defined(ASSERT)
1925 static int reg_size(LIR_Opr op) {
1926   switch (op->type()) {
1927   case T_FLOAT:
1928   case T_INT:      return BytesPerInt;
1929   case T_LONG:
1930   case T_DOUBLE:   return BytesPerLong;
1931   case T_OBJECT:
1932   case T_ARRAY:
1933   case T_METADATA: return BytesPerWord;
1934   case T_ADDRESS:
1935   case T_ILLEGAL:  // fall through
1936   default: ShouldNotReachHere(); return -1;
1937   }
1938 }
1939 #endif
1940 
1941 void LIR_Assembler::arith_op(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dest, CodeEmitInfo* info, bool pop_fpu_stack) {
1942   assert(info == NULL, "unused on this code path");
1943   assert(dest->is_register(), "wrong items state");
1944 
1945   if (right->is_address()) {
1946     // special case for adding shifted/extended register
1947     const Register res = dest->as_pointer_register();
1948     const Register lreg = left->as_pointer_register();
1949     const LIR_Address* addr = right->as_address_ptr();
1950 
1951     assert(addr->base()->as_pointer_register() == lreg && addr->index()->is_register() && addr->disp() == 0, "must be");
1952 
1953     int scale = addr->scale();
1954     AsmShift shift = lsl;
1955 
1956 #ifdef AARCH64
1957     bool is_index_extended = reg_size(addr->base()) > reg_size(addr->index());
1958     if (scale < 0) {
1959       scale = -scale;
1960       shift = lsr;
1961     }
1962     assert(shift == lsl || !is_index_extended, "could not have extend and right shift in one operand");
1963     assert(0 <= scale && scale <= 63, "scale is too large");
1964 
1965     if (is_index_extended) {
1966       assert(scale <= 4, "scale is too large for add with extended register");
1967       assert(addr->index()->is_single_cpu(), "should be");
1968       assert(addr->index()->type() == T_INT, "should be");
1969       assert(dest->is_double_cpu(), "should be");
1970       assert(code == lir_add, "special case of add with extended register");
1971 
1972       __ add(res, lreg, addr->index()->as_register(), ex_sxtw, scale);
1973       return;
1974     } else if (reg_size(dest) == BytesPerInt) {
1975       assert(reg_size(addr->base()) == reg_size(addr->index()), "should be");
1976       assert(reg_size(addr->base()) == reg_size(dest), "should be");
1977 
1978       AsmOperand operand(addr->index()->as_pointer_register(), shift, scale);
1979       switch (code) {
1980         case lir_add: __ add_32(res, lreg, operand); break;
1981         case lir_sub: __ sub_32(res, lreg, operand); break;
1982         default: ShouldNotReachHere();
1983       }
1984       return;
1985     }
1986 #endif // AARCH64
1987 
1988     assert(reg_size(addr->base()) == reg_size(addr->index()), "should be");
1989     assert(reg_size(addr->base()) == reg_size(dest), "should be");
1990     assert(reg_size(dest) == wordSize, "should be");
1991 
1992     AsmOperand operand(addr->index()->as_pointer_register(), shift, scale);
1993     switch (code) {
1994       case lir_add: __ add(res, lreg, operand); break;
1995       case lir_sub: __ sub(res, lreg, operand); break;
1996       default: ShouldNotReachHere();
1997     }
1998 
1999 #ifndef AARCH64
2000   } else if (left->is_address()) {
2001     assert(code == lir_sub && right->is_single_cpu(), "special case used by strength_reduce_multiply()");
2002     const LIR_Address* addr = left->as_address_ptr();
2003     const Register res = dest->as_register();
2004     const Register rreg = right->as_register();
2005     assert(addr->base()->as_register() == rreg && addr->index()->is_register() && addr->disp() == 0, "must be");
2006     __ rsb(res, rreg, AsmOperand(addr->index()->as_register(), lsl, addr->scale()));
2007 #endif // !AARCH64
2008 
2009   } else if (dest->is_single_cpu()) {
2010     assert(left->is_single_cpu(), "unexpected left operand");
2011 #ifdef AARCH64
2012     assert(dest->type() == T_INT, "unexpected dest type");
2013     assert(left->type() == T_INT, "unexpected left type");
2014     assert(right->type() == T_INT, "unexpected right type");
2015 #endif // AARCH64
2016 
2017     const Register res = dest->as_register();
2018     const Register lreg = left->as_register();
2019 
2020     if (right->is_single_cpu()) {
2021       const Register rreg = right->as_register();
2022       switch (code) {
2023         case lir_add: __ add_32(res, lreg, rreg); break;
2024         case lir_sub: __ sub_32(res, lreg, rreg); break;
2025         case lir_mul: __ mul_32(res, lreg, rreg); break;
2026         default: ShouldNotReachHere();
2027       }
2028     } else {
2029       assert(right->is_constant(), "must be");
2030       const jint c = right->as_constant_ptr()->as_jint();
2031       if (!Assembler::is_arith_imm_in_range(c)) {
2032         BAILOUT("illegal arithmetic operand");
2033       }
2034       switch (code) {
2035         case lir_add: __ add_32(res, lreg, c); break;
2036         case lir_sub: __ sub_32(res, lreg, c); break;
2037         default: ShouldNotReachHere();
2038       }
2039     }
2040 
2041   } else if (dest->is_double_cpu()) {
2042 #ifdef AARCH64
2043     assert(left->is_double_cpu() ||
2044            (left->is_single_cpu() && ((left->type() == T_OBJECT) || (left->type() == T_ARRAY) || (left->type() == T_ADDRESS))),
2045            "unexpected left operand");
2046 
2047     const Register res = dest->as_register_lo();
2048     const Register lreg = left->as_pointer_register();
2049 
2050     if (right->is_constant()) {
2051       assert(right->type() == T_LONG, "unexpected right type");
2052       assert((right->as_constant_ptr()->as_jlong() >> 24) == 0, "out of range");
2053       jint imm = (jint)right->as_constant_ptr()->as_jlong();
2054       switch (code) {
2055         case lir_add: __ add(res, lreg, imm); break;
2056         case lir_sub: __ sub(res, lreg, imm); break;
2057         default: ShouldNotReachHere();
2058       }
2059     } else {
2060       assert(right->is_double_cpu() ||
2061              (right->is_single_cpu() && ((right->type() == T_OBJECT) || (right->type() == T_ARRAY) || (right->type() == T_ADDRESS))),
2062              "unexpected right operand");
2063       const Register rreg = right->as_pointer_register();
2064       switch (code) {
2065         case lir_add: __ add(res, lreg, rreg); break;
2066         case lir_sub: __ sub(res, lreg, rreg); break;
2067         case lir_mul: __ mul(res, lreg, rreg); break;
2068         default: ShouldNotReachHere();
2069       }
2070     }
2071 #else // AARCH64
2072     Register res_lo = dest->as_register_lo();
2073     Register res_hi = dest->as_register_hi();
2074     Register lreg_lo = left->as_register_lo();
2075     Register lreg_hi = left->as_register_hi();
2076     if (right->is_double_cpu()) {
2077       Register rreg_lo = right->as_register_lo();
2078       Register rreg_hi = right->as_register_hi();
2079       if (res_lo == lreg_hi || res_lo == rreg_hi) {
2080         res_lo = Rtemp;
2081       }
2082       switch (code) {
2083         case lir_add:
2084           __ adds(res_lo, lreg_lo, rreg_lo);
2085           __ adc(res_hi, lreg_hi, rreg_hi);
2086           break;
2087         case lir_sub:
2088           __ subs(res_lo, lreg_lo, rreg_lo);
2089           __ sbc(res_hi, lreg_hi, rreg_hi);
2090           break;
2091         default:
2092           ShouldNotReachHere();
2093       }
2094     } else {
2095       assert(right->is_constant(), "must be");
2096       assert((right->as_constant_ptr()->as_jlong() >> 32) == 0, "out of range");
2097       const jint c = (jint) right->as_constant_ptr()->as_jlong();
2098       if (res_lo == lreg_hi) {
2099         res_lo = Rtemp;
2100       }
2101       switch (code) {
2102         case lir_add:
2103           __ adds(res_lo, lreg_lo, c);
2104           __ adc(res_hi, lreg_hi, 0);
2105           break;
2106         case lir_sub:
2107           __ subs(res_lo, lreg_lo, c);
2108           __ sbc(res_hi, lreg_hi, 0);
2109           break;
2110         default:
2111           ShouldNotReachHere();
2112       }
2113     }
2114     move_regs(res_lo, dest->as_register_lo());
2115 #endif // AARCH64
2116 
2117   } else if (dest->is_single_fpu()) {
2118     assert(left->is_single_fpu(), "must be");
2119     assert(right->is_single_fpu(), "must be");
2120     const FloatRegister res = dest->as_float_reg();
2121     const FloatRegister lreg = left->as_float_reg();
2122     const FloatRegister rreg = right->as_float_reg();
2123     switch (code) {
2124       case lir_add: __ add_float(res, lreg, rreg); break;
2125       case lir_sub: __ sub_float(res, lreg, rreg); break;
2126       case lir_mul_strictfp: // fall through
2127       case lir_mul: __ mul_float(res, lreg, rreg); break;
2128       case lir_div_strictfp: // fall through
2129       case lir_div: __ div_float(res, lreg, rreg); break;
2130       default: ShouldNotReachHere();
2131     }
2132   } else if (dest->is_double_fpu()) {
2133     assert(left->is_double_fpu(), "must be");
2134     assert(right->is_double_fpu(), "must be");
2135     const FloatRegister res = dest->as_double_reg();
2136     const FloatRegister lreg = left->as_double_reg();
2137     const FloatRegister rreg = right->as_double_reg();
2138     switch (code) {
2139       case lir_add: __ add_double(res, lreg, rreg); break;
2140       case lir_sub: __ sub_double(res, lreg, rreg); break;
2141       case lir_mul_strictfp: // fall through
2142       case lir_mul: __ mul_double(res, lreg, rreg); break;
2143       case lir_div_strictfp: // fall through
2144       case lir_div: __ div_double(res, lreg, rreg); break;
2145       default: ShouldNotReachHere();
2146     }
2147   } else {
2148     ShouldNotReachHere();
2149   }
2150 }
2151 
2152 
2153 void LIR_Assembler::intrinsic_op(LIR_Code code, LIR_Opr value, LIR_Opr unused, LIR_Opr dest, LIR_Op* op) {
2154   switch (code) {
2155     case lir_abs:
2156       __ abs_double(dest->as_double_reg(), value->as_double_reg());
2157       break;
2158     case lir_sqrt:
2159       __ sqrt_double(dest->as_double_reg(), value->as_double_reg());
2160       break;
2161     default:
2162       ShouldNotReachHere();
2163   }
2164 }
2165 
2166 
2167 void LIR_Assembler::logic_op(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dest) {
2168   assert(dest->is_register(), "wrong items state");
2169   assert(left->is_register(), "wrong items state");
2170 
2171   if (dest->is_single_cpu()) {
2172 #ifdef AARCH64
2173     assert (dest->type() == T_INT, "unexpected result type");
2174     assert (left->type() == T_INT, "unexpected left type");
2175     assert (right->type() == T_INT, "unexpected right type");
2176 #endif // AARCH64
2177 
2178     const Register res = dest->as_register();
2179     const Register lreg = left->as_register();
2180 
2181     if (right->is_single_cpu()) {
2182       const Register rreg = right->as_register();
2183       switch (code) {
2184         case lir_logic_and: __ and_32(res, lreg, rreg); break;
2185         case lir_logic_or:  __ orr_32(res, lreg, rreg); break;
2186         case lir_logic_xor: __ eor_32(res, lreg, rreg); break;
2187         default: ShouldNotReachHere();
2188       }
2189     } else {
2190       assert(right->is_constant(), "must be");
2191       const uint c = (uint)right->as_constant_ptr()->as_jint();
2192       switch (code) {
2193         case lir_logic_and: __ and_32(res, lreg, c); break;
2194         case lir_logic_or:  __ orr_32(res, lreg, c); break;
2195         case lir_logic_xor: __ eor_32(res, lreg, c); break;
2196         default: ShouldNotReachHere();
2197       }
2198     }
2199   } else {
2200     assert(dest->is_double_cpu(), "should be");
2201     Register res_lo = dest->as_register_lo();
2202 
2203 #ifdef AARCH64
2204     assert ((left->is_single_cpu() && left->is_oop_register()) || left->is_double_cpu(), "should be");
2205     const Register lreg_lo = left->as_pointer_register();
2206 #else
2207     assert (dest->type() == T_LONG, "unexpected result type");
2208     assert (left->type() == T_LONG, "unexpected left type");
2209     assert (right->type() == T_LONG, "unexpected right type");
2210 
2211     const Register res_hi = dest->as_register_hi();
2212     const Register lreg_lo = left->as_register_lo();
2213     const Register lreg_hi = left->as_register_hi();
2214 #endif // AARCH64
2215 
2216     if (right->is_register()) {
2217 #ifdef AARCH64
2218       assert ((right->is_single_cpu() && right->is_oop_register()) || right->is_double_cpu(), "should be");
2219       const Register rreg_lo = right->as_pointer_register();
2220       switch (code) {
2221         case lir_logic_and: __ andr(res_lo, lreg_lo, rreg_lo); break;
2222         case lir_logic_or:  __ orr (res_lo, lreg_lo, rreg_lo); break;
2223         case lir_logic_xor: __ eor (res_lo, lreg_lo, rreg_lo); break;
2224         default: ShouldNotReachHere();
2225       }
2226 #else
2227       const Register rreg_lo = right->as_register_lo();
2228       const Register rreg_hi = right->as_register_hi();
2229       if (res_lo == lreg_hi || res_lo == rreg_hi) {
2230         res_lo = Rtemp; // Temp register helps to avoid overlap between result and input
2231       }
2232       switch (code) {
2233         case lir_logic_and:
2234           __ andr(res_lo, lreg_lo, rreg_lo);
2235           __ andr(res_hi, lreg_hi, rreg_hi);
2236           break;
2237         case lir_logic_or:
2238           __ orr(res_lo, lreg_lo, rreg_lo);
2239           __ orr(res_hi, lreg_hi, rreg_hi);
2240           break;
2241         case lir_logic_xor:
2242           __ eor(res_lo, lreg_lo, rreg_lo);
2243           __ eor(res_hi, lreg_hi, rreg_hi);
2244           break;
2245         default:
2246           ShouldNotReachHere();
2247       }
2248       move_regs(res_lo, dest->as_register_lo());
2249 #endif // AARCH64
2250     } else {
2251       assert(right->is_constant(), "must be");
2252 #ifdef AARCH64
2253       const julong c = (julong)right->as_constant_ptr()->as_jlong();
2254       Assembler::LogicalImmediate imm(c, false);
2255       if (imm.is_encoded()) {
2256         switch (code) {
2257           case lir_logic_and: __ andr(res_lo, lreg_lo, imm); break;
2258           case lir_logic_or:  __ orr (res_lo, lreg_lo, imm); break;
2259           case lir_logic_xor: __ eor (res_lo, lreg_lo, imm); break;
2260           default: ShouldNotReachHere();
2261         }
2262       } else {
2263         BAILOUT("64 bit constant cannot be inlined");
2264       }
2265 #else
2266       const jint c_lo = (jint) right->as_constant_ptr()->as_jlong();
2267       const jint c_hi = (jint) (right->as_constant_ptr()->as_jlong() >> 32);
2268       // Case for logic_or from do_ClassIDIntrinsic()
2269       if (c_hi == 0 && AsmOperand::is_rotated_imm(c_lo)) {
2270         switch (code) {
2271           case lir_logic_and:
2272             __ andr(res_lo, lreg_lo, c_lo);
2273             __ mov(res_hi, 0);
2274             break;
2275           case lir_logic_or:
2276             __ orr(res_lo, lreg_lo, c_lo);
2277             break;
2278           case lir_logic_xor:
2279             __ eor(res_lo, lreg_lo, c_lo);
2280             break;
2281         default:
2282           ShouldNotReachHere();
2283         }
2284       } else if (code == lir_logic_and &&
2285                  c_hi == -1 &&
2286                  (AsmOperand::is_rotated_imm(c_lo) ||
2287                   AsmOperand::is_rotated_imm(~c_lo))) {
2288         // Another case which handles logic_and from do_ClassIDIntrinsic()
2289         if (AsmOperand::is_rotated_imm(c_lo)) {
2290           __ andr(res_lo, lreg_lo, c_lo);
2291         } else {
2292           __ bic(res_lo, lreg_lo, ~c_lo);
2293         }
2294         if (res_hi != lreg_hi) {
2295           __ mov(res_hi, lreg_hi);
2296         }
2297       } else {
2298         BAILOUT("64 bit constant cannot be inlined");
2299       }
2300 #endif // AARCH64
2301     }
2302   }
2303 }
2304 
2305 
2306 #ifdef AARCH64
2307 
2308 void LIR_Assembler::long_compare_helper(LIR_Opr opr1, LIR_Opr opr2) {
2309   assert(opr1->is_double_cpu(), "should be");
2310   Register x = opr1->as_register_lo();
2311 
2312   if (opr2->is_double_cpu()) {
2313     Register y = opr2->as_register_lo();
2314     __ cmp(x, y);
2315 
2316   } else {
2317     assert(opr2->is_constant(), "should be");
2318     assert(opr2->as_constant_ptr()->type() == T_LONG, "long constant expected");
2319     jlong c = opr2->as_jlong();
2320     assert(((c >> 31) == 0) || ((c >> 31) == -1), "immediate is out of range");
2321     if (c >= 0) {
2322       __ cmp(x, (jint)c);
2323     } else {
2324       __ cmn(x, (jint)(-c));
2325     }
2326   }
2327 }
2328 
2329 #endif // AARCH64
2330 
2331 void LIR_Assembler::comp_op(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Op2* op) {
2332   if (opr1->is_single_cpu()) {
2333     if (opr2->is_constant()) {
2334       switch (opr2->as_constant_ptr()->type()) {
2335         case T_INT: {
2336           const jint c = opr2->as_constant_ptr()->as_jint();
2337           if (Assembler::is_arith_imm_in_range(c)) {
2338             __ cmp_32(opr1->as_register(), c);
2339           } else if (Assembler::is_arith_imm_in_range(-c)) {
2340             __ cmn_32(opr1->as_register(), -c);
2341           } else {
2342             // This can happen when compiling lookupswitch
2343             __ mov_slow(Rtemp, c);
2344             __ cmp_32(opr1->as_register(), Rtemp);
2345           }
2346           break;
2347         }
2348         case T_OBJECT:
2349           assert(opr2->as_constant_ptr()->as_jobject() == NULL, "cannot handle otherwise");
2350           __ cmp(opr1->as_register(), 0);
2351           break;
2352         default:
2353           ShouldNotReachHere();
2354       }
2355     } else if (opr2->is_single_cpu()) {
2356       if (opr1->type() == T_OBJECT || opr1->type() == T_ARRAY || opr1->type() == T_METADATA || opr1->type() == T_ADDRESS) {
2357         assert(opr2->type() == T_OBJECT || opr2->type() == T_ARRAY || opr2->type() == T_METADATA || opr2->type() == T_ADDRESS, "incompatibe type");
2358         __ cmp(opr1->as_register(), opr2->as_register());
2359       } else {
2360         assert(opr2->type() != T_OBJECT && opr2->type() != T_ARRAY && opr2->type() != T_METADATA && opr2->type() != T_ADDRESS, "incompatibe type");
2361         __ cmp_32(opr1->as_register(), opr2->as_register());
2362       }
2363     } else {
2364       ShouldNotReachHere();
2365     }
2366   } else if (opr1->is_double_cpu()) {
2367 #ifdef AARCH64
2368     long_compare_helper(opr1, opr2);
2369 #else
2370     Register xlo = opr1->as_register_lo();
2371     Register xhi = opr1->as_register_hi();
2372     if (opr2->is_constant() && opr2->as_jlong() == 0) {
2373       assert(condition == lir_cond_equal || condition == lir_cond_notEqual, "cannot handle otherwise");
2374       __ orrs(Rtemp, xlo, xhi);
2375     } else if (opr2->is_register()) {
2376       Register ylo = opr2->as_register_lo();
2377       Register yhi = opr2->as_register_hi();
2378       if (condition == lir_cond_equal || condition == lir_cond_notEqual) {
2379         __ teq(xhi, yhi);
2380         __ teq(xlo, ylo, eq);
2381       } else {
2382         __ subs(xlo, xlo, ylo);
2383         __ sbcs(xhi, xhi, yhi);
2384       }
2385     } else {
2386       ShouldNotReachHere();
2387     }
2388 #endif // AARCH64
2389   } else if (opr1->is_single_fpu()) {
2390     if (opr2->is_constant()) {
2391       assert(opr2->as_jfloat() == 0.0f, "cannot handle otherwise");
2392       __ cmp_zero_float(opr1->as_float_reg());
2393     } else {
2394       __ cmp_float(opr1->as_float_reg(), opr2->as_float_reg());
2395     }
2396   } else if (opr1->is_double_fpu()) {
2397     if (opr2->is_constant()) {
2398       assert(opr2->as_jdouble() == 0.0, "cannot handle otherwise");
2399       __ cmp_zero_double(opr1->as_double_reg());
2400     } else {
2401       __ cmp_double(opr1->as_double_reg(), opr2->as_double_reg());
2402     }
2403   } else {
2404     ShouldNotReachHere();
2405   }
2406 }
2407 
2408 void LIR_Assembler::comp_fl2i(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dst, LIR_Op2* op) {
2409   const Register res = dst->as_register();
2410   if (code == lir_cmp_fd2i || code == lir_ucmp_fd2i) {
2411     comp_op(lir_cond_unknown, left, right, op);
2412 #ifdef AARCH64
2413     if (code == lir_ucmp_fd2i) {         // unordered is less
2414       __ cset(res, gt);                  // 1 if '>', else 0
2415       __ csinv(res, res, ZR, ge);        // previous value if '>=', else -1
2416     } else {
2417       __ cset(res, hi);                  // 1 if '>' or unordered, else 0
2418       __ csinv(res, res, ZR, pl);        // previous value if '>=' or unordered, else -1
2419     }
2420 #else
2421     __ fmstat();
2422     if (code == lir_ucmp_fd2i) {  // unordered is less
2423       __ mvn(res, 0, lt);
2424       __ mov(res, 1, ge);
2425     } else {                      // unordered is greater
2426       __ mov(res, 1, cs);
2427       __ mvn(res, 0, cc);
2428     }
2429     __ mov(res, 0, eq);
2430 #endif // AARCH64
2431 
2432   } else {
2433     assert(code == lir_cmp_l2i, "must be");
2434 
2435 #ifdef AARCH64
2436     long_compare_helper(left, right);
2437 
2438     __ cset(res, gt);            // 1 if '>', else 0
2439     __ csinv(res, res, ZR, ge);  // previous value if '>=', else -1
2440 #else
2441     Label done;
2442     const Register xlo = left->as_register_lo();
2443     const Register xhi = left->as_register_hi();
2444     const Register ylo = right->as_register_lo();
2445     const Register yhi = right->as_register_hi();
2446     __ cmp(xhi, yhi);
2447     __ mov(res, 1, gt);
2448     __ mvn(res, 0, lt);
2449     __ b(done, ne);
2450     __ subs(res, xlo, ylo);
2451     __ mov(res, 1, hi);
2452     __ mvn(res, 0, lo);
2453     __ bind(done);
2454 #endif // AARCH64
2455   }
2456 }
2457 
2458 
2459 void LIR_Assembler::align_call(LIR_Code code) {
2460   // Not needed
2461 }
2462 
2463 
2464 void LIR_Assembler::call(LIR_OpJavaCall *op, relocInfo::relocType rtype) {
2465   int ret_addr_offset = __ patchable_call(op->addr(), rtype);
2466   assert(ret_addr_offset == __ offset(), "embedded return address not allowed");
2467   add_call_info_here(op->info());
2468 }
2469 
2470 
2471 void LIR_Assembler::ic_call(LIR_OpJavaCall *op) {
2472   bool near_range = __ cache_fully_reachable();
2473   address oop_address = pc();
2474 
2475   bool use_movw = AARCH64_ONLY(false) NOT_AARCH64(VM_Version::supports_movw());
2476 
2477   // Ricklass may contain something that is not a metadata pointer so
2478   // mov_metadata can't be used
2479   InlinedAddress value((address)Universe::non_oop_word());
2480   InlinedAddress addr(op->addr());
2481   if (use_movw) {
2482 #ifdef AARCH64
2483     ShouldNotReachHere();
2484 #else
2485     __ movw(Ricklass, ((unsigned int)Universe::non_oop_word()) & 0xffff);
2486     __ movt(Ricklass, ((unsigned int)Universe::non_oop_word()) >> 16);
2487 #endif // AARCH64
2488   } else {
2489     // No movw/movt, must be load a pc relative value but no
2490     // relocation so no metadata table to load from.
2491     // Use a b instruction rather than a bl, inline constant after the
2492     // branch, use a PC relative ldr to load the constant, arrange for
2493     // the call to return after the constant(s).
2494     __ ldr_literal(Ricklass, value);
2495   }
2496   __ relocate(virtual_call_Relocation::spec(oop_address));
2497   if (near_range && use_movw) {
2498     __ bl(op->addr());
2499   } else {
2500     Label call_return;
2501     __ adr(LR, call_return);
2502     if (near_range) {
2503       __ b(op->addr());
2504     } else {
2505       __ indirect_jump(addr, Rtemp);
2506       __ bind_literal(addr);
2507     }
2508     if (!use_movw) {
2509       __ bind_literal(value);
2510     }
2511     __ bind(call_return);
2512   }
2513   add_call_info(code_offset(), op->info());
2514 }
2515 
2516 
2517 /* Currently, vtable-dispatch is only enabled for sparc platforms */
2518 void LIR_Assembler::vtable_call(LIR_OpJavaCall* op) {
2519   ShouldNotReachHere();
2520 }
2521 
2522 void LIR_Assembler::emit_static_call_stub() {
2523   address call_pc = __ pc();
2524   address stub = __ start_a_stub(call_stub_size());
2525   if (stub == NULL) {
2526     BAILOUT("static call stub overflow");
2527   }
2528 
2529   DEBUG_ONLY(int offset = code_offset();)
2530 
2531   InlinedMetadata metadata_literal(NULL);
2532   __ relocate(static_stub_Relocation::spec(call_pc));
2533   // If not a single instruction, NativeMovConstReg::next_instruction_address()
2534   // must jump over the whole following ldr_literal.
2535   // (See CompiledStaticCall::set_to_interpreted())
2536 #ifdef ASSERT
2537   address ldr_site = __ pc();
2538 #endif
2539   __ ldr_literal(Rmethod, metadata_literal);
2540   assert(nativeMovConstReg_at(ldr_site)->next_instruction_address() == __ pc(), "Fix ldr_literal or its parsing");
2541   bool near_range = __ cache_fully_reachable();
2542   InlinedAddress dest((address)-1);
2543   if (near_range) {
2544     address branch_site = __ pc();
2545     __ b(branch_site); // b to self maps to special NativeJump -1 destination
2546   } else {
2547     __ indirect_jump(dest, Rtemp);
2548   }
2549   __ bind_literal(metadata_literal); // includes spec_for_immediate reloc
2550   if (!near_range) {
2551     __ bind_literal(dest); // special NativeJump -1 destination
2552   }
2553 
2554   assert(code_offset() - offset <= call_stub_size(), "overflow");
2555   __ end_a_stub();
2556 }
2557 
2558 void LIR_Assembler::throw_op(LIR_Opr exceptionPC, LIR_Opr exceptionOop, CodeEmitInfo* info) {
2559   assert(exceptionOop->as_register() == Rexception_obj, "must match");
2560   assert(exceptionPC->as_register()  == Rexception_pc, "must match");
2561   info->add_register_oop(exceptionOop);
2562 
2563   Runtime1::StubID handle_id = compilation()->has_fpu_code() ?
2564                                Runtime1::handle_exception_id :
2565                                Runtime1::handle_exception_nofpu_id;
2566   Label return_address;
2567   __ adr(Rexception_pc, return_address);
2568   __ call(Runtime1::entry_for(handle_id), relocInfo::runtime_call_type);
2569   __ bind(return_address);
2570   add_call_info_here(info);  // for exception handler
2571 }
2572 
2573 void LIR_Assembler::unwind_op(LIR_Opr exceptionOop) {
2574   assert(exceptionOop->as_register() == Rexception_obj, "must match");
2575   __ b(_unwind_handler_entry);
2576 }
2577 
2578 void LIR_Assembler::shift_op(LIR_Code code, LIR_Opr left, LIR_Opr count, LIR_Opr dest, LIR_Opr tmp) {
2579 #ifdef AARCH64
2580   if (dest->is_single_cpu()) {
2581     Register res = dest->as_register();
2582     Register x = left->as_register();
2583     Register y = count->as_register();
2584     assert (dest->type() == T_INT, "unexpected result type");
2585     assert (left->type() == T_INT, "unexpected left type");
2586 
2587     switch (code) {
2588       case lir_shl:  __ lslv_w(res, x, y); break;
2589       case lir_shr:  __ asrv_w(res, x, y); break;
2590       case lir_ushr: __ lsrv_w(res, x, y); break;
2591       default: ShouldNotReachHere();
2592     }
2593   } else if (dest->is_double_cpu()) {
2594     Register res = dest->as_register_lo();
2595     Register x = left->as_register_lo();
2596     Register y = count->as_register();
2597 
2598     switch (code) {
2599       case lir_shl:  __ lslv(res, x, y); break;
2600       case lir_shr:  __ asrv(res, x, y); break;
2601       case lir_ushr: __ lsrv(res, x, y); break;
2602       default: ShouldNotReachHere();
2603     }
2604   } else {
2605     ShouldNotReachHere();
2606   }
2607 #else
2608   AsmShift shift = lsl;
2609   switch (code) {
2610     case lir_shl:  shift = lsl; break;
2611     case lir_shr:  shift = asr; break;
2612     case lir_ushr: shift = lsr; break;
2613     default: ShouldNotReachHere();
2614   }
2615 
2616   if (dest->is_single_cpu()) {
2617     __ andr(Rtemp, count->as_register(), 31);
2618     __ mov(dest->as_register(), AsmOperand(left->as_register(), shift, Rtemp));
2619   } else if (dest->is_double_cpu()) {
2620     Register dest_lo = dest->as_register_lo();
2621     Register dest_hi = dest->as_register_hi();
2622     Register src_lo  = left->as_register_lo();
2623     Register src_hi  = left->as_register_hi();
2624     Register Rcount  = count->as_register();
2625     // Resolve possible register conflicts
2626     if (shift == lsl && dest_hi == src_lo) {
2627       dest_hi = Rtemp;
2628     } else if (shift != lsl && dest_lo == src_hi) {
2629       dest_lo = Rtemp;
2630     } else if (dest_lo == src_lo && dest_hi == src_hi) {
2631       dest_lo = Rtemp;
2632     } else if (dest_lo == Rcount || dest_hi == Rcount) {
2633       Rcount = Rtemp;
2634     }
2635     __ andr(Rcount, count->as_register(), 63);
2636     __ long_shift(dest_lo, dest_hi, src_lo, src_hi, shift, Rcount);
2637     move_regs(dest_lo, dest->as_register_lo());
2638     move_regs(dest_hi, dest->as_register_hi());
2639   } else {
2640     ShouldNotReachHere();
2641   }
2642 #endif // AARCH64
2643 }
2644 
2645 
2646 void LIR_Assembler::shift_op(LIR_Code code, LIR_Opr left, jint count, LIR_Opr dest) {
2647 #ifdef AARCH64
2648   if (dest->is_single_cpu()) {
2649     assert (dest->type() == T_INT, "unexpected result type");
2650     assert (left->type() == T_INT, "unexpected left type");
2651     count &= 31;
2652     if (count != 0) {
2653       switch (code) {
2654         case lir_shl:  __ _lsl_w(dest->as_register(), left->as_register(), count); break;
2655         case lir_shr:  __ _asr_w(dest->as_register(), left->as_register(), count); break;
2656         case lir_ushr: __ _lsr_w(dest->as_register(), left->as_register(), count); break;
2657         default: ShouldNotReachHere();
2658       }
2659     } else {
2660       move_regs(left->as_register(), dest->as_register());
2661     }
2662   } else if (dest->is_double_cpu()) {
2663     count &= 63;
2664     if (count != 0) {
2665       switch (code) {
2666         case lir_shl:  __ _lsl(dest->as_register_lo(), left->as_register_lo(), count); break;
2667         case lir_shr:  __ _asr(dest->as_register_lo(), left->as_register_lo(), count); break;
2668         case lir_ushr: __ _lsr(dest->as_register_lo(), left->as_register_lo(), count); break;
2669         default: ShouldNotReachHere();
2670       }
2671     } else {
2672       move_regs(left->as_register_lo(), dest->as_register_lo());
2673     }
2674   } else {
2675     ShouldNotReachHere();
2676   }
2677 
2678 #else
2679   AsmShift shift = lsl;
2680   switch (code) {
2681     case lir_shl:  shift = lsl; break;
2682     case lir_shr:  shift = asr; break;
2683     case lir_ushr: shift = lsr; break;
2684     default: ShouldNotReachHere();
2685   }
2686 
2687   if (dest->is_single_cpu()) {
2688     count &= 31;
2689     if (count != 0) {
2690       __ mov(dest->as_register(), AsmOperand(left->as_register(), shift, count));
2691     } else {
2692       move_regs(left->as_register(), dest->as_register());
2693     }
2694   } else if (dest->is_double_cpu()) {
2695     count &= 63;
2696     if (count != 0) {
2697       Register dest_lo = dest->as_register_lo();
2698       Register dest_hi = dest->as_register_hi();
2699       Register src_lo  = left->as_register_lo();
2700       Register src_hi  = left->as_register_hi();
2701       // Resolve possible register conflicts
2702       if (shift == lsl && dest_hi == src_lo) {
2703         dest_hi = Rtemp;
2704       } else if (shift != lsl && dest_lo == src_hi) {
2705         dest_lo = Rtemp;
2706       }
2707       __ long_shift(dest_lo, dest_hi, src_lo, src_hi, shift, count);
2708       move_regs(dest_lo, dest->as_register_lo());
2709       move_regs(dest_hi, dest->as_register_hi());
2710     } else {
2711       __ long_move(dest->as_register_lo(), dest->as_register_hi(),
2712                    left->as_register_lo(), left->as_register_hi());
2713     }
2714   } else {
2715     ShouldNotReachHere();
2716   }
2717 #endif // AARCH64
2718 }
2719 
2720 
2721 // Saves 4 given registers in reserved argument area.
2722 void LIR_Assembler::save_in_reserved_area(Register r1, Register r2, Register r3, Register r4) {
2723   verify_reserved_argument_area_size(4);
2724 #ifdef AARCH64
2725   __ stp(r1, r2, Address(SP, 0));
2726   __ stp(r3, r4, Address(SP, 2*wordSize));
2727 #else
2728   __ stmia(SP, RegisterSet(r1) | RegisterSet(r2) | RegisterSet(r3) | RegisterSet(r4));
2729 #endif // AARCH64
2730 }
2731 
2732 // Restores 4 given registers from reserved argument area.
2733 void LIR_Assembler::restore_from_reserved_area(Register r1, Register r2, Register r3, Register r4) {
2734 #ifdef AARCH64
2735   __ ldp(r1, r2, Address(SP, 0));
2736   __ ldp(r3, r4, Address(SP, 2*wordSize));
2737 #else
2738   __ ldmia(SP, RegisterSet(r1) | RegisterSet(r2) | RegisterSet(r3) | RegisterSet(r4), no_writeback);
2739 #endif // AARCH64
2740 }
2741 
2742 
2743 void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) {
2744   ciArrayKlass* default_type = op->expected_type();
2745   Register src = op->src()->as_register();
2746   Register src_pos = op->src_pos()->as_register();
2747   Register dst = op->dst()->as_register();
2748   Register dst_pos = op->dst_pos()->as_register();
2749   Register length  = op->length()->as_register();
2750   Register tmp = op->tmp()->as_register();
2751   Register tmp2 = Rtemp;
2752 
2753   assert(src == R0 && src_pos == R1 && dst == R2 && dst_pos == R3, "code assumption");
2754 #ifdef AARCH64
2755   assert(length == R4, "code assumption");
2756 #endif // AARCH64
2757 
2758   CodeStub* stub = op->stub();
2759 
2760   int flags = op->flags();
2761   BasicType basic_type = default_type != NULL ? default_type->element_type()->basic_type() : T_ILLEGAL;
2762   if (basic_type == T_ARRAY) basic_type = T_OBJECT;
2763 
2764   // If we don't know anything or it's an object array, just go through the generic arraycopy
2765   if (default_type == NULL) {
2766 
2767     // save arguments, because they will be killed by a runtime call
2768     save_in_reserved_area(R0, R1, R2, R3);
2769 
2770 #ifdef AARCH64
2771     // save length argument, will be killed by a runtime call
2772     __ raw_push(length, ZR);
2773 #else
2774     // pass length argument on SP[0]
2775     __ str(length, Address(SP, -2*wordSize, pre_indexed));  // 2 words for a proper stack alignment
2776 #endif // AARCH64
2777 
2778     address copyfunc_addr = StubRoutines::generic_arraycopy();
2779     if (copyfunc_addr == NULL) { // Use C version if stub was not generated
2780       __ call(CAST_FROM_FN_PTR(address, Runtime1::arraycopy));
2781     } else {
2782 #ifndef PRODUCT
2783       if (PrintC1Statistics) {
2784         __ inc_counter((address)&Runtime1::_generic_arraycopystub_cnt, tmp, tmp2);
2785       }
2786 #endif // !PRODUCT
2787       // the stub is in the code cache so close enough
2788       __ call(copyfunc_addr, relocInfo::runtime_call_type);
2789     }
2790 
2791 #ifdef AARCH64
2792     __ raw_pop(length, ZR);
2793 #else
2794     __ add(SP, SP, 2*wordSize);
2795 #endif // AARCH64
2796 
2797     __ cbz_32(R0, *stub->continuation());
2798 
2799     if (copyfunc_addr != NULL) {
2800       __ mvn_32(tmp, R0);
2801       restore_from_reserved_area(R0, R1, R2, R3);  // load saved arguments in slow case only
2802       __ sub_32(length, length, tmp);
2803       __ add_32(src_pos, src_pos, tmp);
2804       __ add_32(dst_pos, dst_pos, tmp);
2805     } else {
2806       restore_from_reserved_area(R0, R1, R2, R3);  // load saved arguments in slow case only
2807     }
2808 
2809     __ b(*stub->entry());
2810 
2811     __ bind(*stub->continuation());
2812     return;
2813   }
2814 
2815   assert(default_type != NULL && default_type->is_array_klass() && default_type->is_loaded(),
2816          "must be true at this point");
2817   int elem_size = type2aelembytes(basic_type);
2818   int shift = exact_log2(elem_size);
2819 
2820   // Check for NULL
2821   if (flags & LIR_OpArrayCopy::src_null_check) {
2822     if (flags & LIR_OpArrayCopy::dst_null_check) {
2823       __ cmp(src, 0);
2824       __ cond_cmp(dst, 0, ne);  // make one instruction shorter if both checks are needed
2825       __ b(*stub->entry(), eq);
2826     } else {
2827       __ cbz(src, *stub->entry());
2828     }
2829   } else if (flags & LIR_OpArrayCopy::dst_null_check) {
2830     __ cbz(dst, *stub->entry());
2831   }
2832 
2833   // If the compiler was not able to prove that exact type of the source or the destination
2834   // of the arraycopy is an array type, check at runtime if the source or the destination is
2835   // an instance type.
2836   if (flags & LIR_OpArrayCopy::type_check) {
2837     if (!(flags & LIR_OpArrayCopy::LIR_OpArrayCopy::dst_objarray)) {
2838       __ load_klass(tmp, dst);
2839       __ ldr_u32(tmp2, Address(tmp, in_bytes(Klass::layout_helper_offset())));
2840       __ mov_slow(tmp, Klass::_lh_neutral_value);
2841       __ cmp_32(tmp2, tmp);
2842       __ b(*stub->entry(), ge);
2843     }
2844 
2845     if (!(flags & LIR_OpArrayCopy::LIR_OpArrayCopy::src_objarray)) {
2846       __ load_klass(tmp, src);
2847       __ ldr_u32(tmp2, Address(tmp, in_bytes(Klass::layout_helper_offset())));
2848       __ mov_slow(tmp, Klass::_lh_neutral_value);
2849       __ cmp_32(tmp2, tmp);
2850       __ b(*stub->entry(), ge);
2851     }
2852   }
2853 
2854   // Check if negative
2855   const int all_positive_checks = LIR_OpArrayCopy::src_pos_positive_check |
2856                                   LIR_OpArrayCopy::dst_pos_positive_check |
2857                                   LIR_OpArrayCopy::length_positive_check;
2858   switch (flags & all_positive_checks) {
2859     case LIR_OpArrayCopy::src_pos_positive_check:
2860       __ branch_if_negative_32(src_pos, *stub->entry());
2861       break;
2862     case LIR_OpArrayCopy::dst_pos_positive_check:
2863       __ branch_if_negative_32(dst_pos, *stub->entry());
2864       break;
2865     case LIR_OpArrayCopy::length_positive_check:
2866       __ branch_if_negative_32(length, *stub->entry());
2867       break;
2868     case LIR_OpArrayCopy::src_pos_positive_check | LIR_OpArrayCopy::dst_pos_positive_check:
2869       __ branch_if_any_negative_32(src_pos, dst_pos, tmp, *stub->entry());
2870       break;
2871     case LIR_OpArrayCopy::src_pos_positive_check | LIR_OpArrayCopy::length_positive_check:
2872       __ branch_if_any_negative_32(src_pos, length, tmp, *stub->entry());
2873       break;
2874     case LIR_OpArrayCopy::dst_pos_positive_check | LIR_OpArrayCopy::length_positive_check:
2875       __ branch_if_any_negative_32(dst_pos, length, tmp, *stub->entry());
2876       break;
2877     case all_positive_checks:
2878       __ branch_if_any_negative_32(src_pos, dst_pos, length, tmp, *stub->entry());
2879       break;
2880     default:
2881       assert((flags & all_positive_checks) == 0, "the last option");
2882   }
2883 
2884   // Range checks
2885   if (flags & LIR_OpArrayCopy::src_range_check) {
2886     __ ldr_s32(tmp2, Address(src, arrayOopDesc::length_offset_in_bytes()));
2887     __ add_32(tmp, src_pos, length);
2888     __ cmp_32(tmp, tmp2);
2889     __ b(*stub->entry(), hi);
2890   }
2891   if (flags & LIR_OpArrayCopy::dst_range_check) {
2892     __ ldr_s32(tmp2, Address(dst, arrayOopDesc::length_offset_in_bytes()));
2893     __ add_32(tmp, dst_pos, length);
2894     __ cmp_32(tmp, tmp2);
2895     __ b(*stub->entry(), hi);
2896   }
2897 
2898   // Check if src and dst are of the same type
2899   if (flags & LIR_OpArrayCopy::type_check) {
2900     // We don't know the array types are compatible
2901     if (basic_type != T_OBJECT) {
2902       // Simple test for basic type arrays
2903       if (UseCompressedClassPointers) {
2904         // We don't need decode because we just need to compare
2905         __ ldr_u32(tmp, Address(src, oopDesc::klass_offset_in_bytes()));
2906         __ ldr_u32(tmp2, Address(dst, oopDesc::klass_offset_in_bytes()));
2907         __ cmp_32(tmp, tmp2);
2908       } else {
2909         __ load_klass(tmp, src);
2910         __ load_klass(tmp2, dst);
2911         __ cmp(tmp, tmp2);
2912       }
2913       __ b(*stub->entry(), ne);
2914     } else {
2915       // For object arrays, if src is a sub class of dst then we can
2916       // safely do the copy.
2917       Label cont, slow;
2918 
2919       address copyfunc_addr = StubRoutines::checkcast_arraycopy();
2920 
2921       __ load_klass(tmp, src);
2922       __ load_klass(tmp2, dst);
2923 
2924       // We are at a call so all live registers are saved before we
2925       // get here
2926       assert_different_registers(tmp, tmp2, R6, altFP_7_11);
2927 
2928       __ check_klass_subtype_fast_path(tmp, tmp2, R6, altFP_7_11, &cont, copyfunc_addr == NULL ? stub->entry() : &slow, NULL);
2929 
2930       __ mov(R6, R0);
2931       __ mov(altFP_7_11, R1);
2932       __ mov(R0, tmp);
2933       __ mov(R1, tmp2);
2934       __ call(Runtime1::entry_for(Runtime1::slow_subtype_check_id), relocInfo::runtime_call_type); // does not blow any registers except R0, LR and Rtemp
2935       __ cmp_32(R0, 0);
2936       __ mov(R0, R6);
2937       __ mov(R1, altFP_7_11);
2938 
2939       if (copyfunc_addr != NULL) { // use stub if available
2940         // src is not a sub class of dst so we have to do a
2941         // per-element check.
2942 
2943         __ b(cont, ne);
2944 
2945         __ bind(slow);
2946 
2947         int mask = LIR_OpArrayCopy::src_objarray|LIR_OpArrayCopy::dst_objarray;
2948         if ((flags & mask) != mask) {
2949           // Check that at least both of them object arrays.
2950           assert(flags & mask, "one of the two should be known to be an object array");
2951 
2952           if (!(flags & LIR_OpArrayCopy::src_objarray)) {
2953             __ load_klass(tmp, src);
2954           } else if (!(flags & LIR_OpArrayCopy::dst_objarray)) {
2955             __ load_klass(tmp, dst);
2956           }
2957           int lh_offset = in_bytes(Klass::layout_helper_offset());
2958 
2959           __ ldr_u32(tmp2, Address(tmp, lh_offset));
2960 
2961           jint objArray_lh = Klass::array_layout_helper(T_OBJECT);
2962           __ mov_slow(tmp, objArray_lh);
2963           __ cmp_32(tmp, tmp2);
2964           __ b(*stub->entry(), ne);
2965         }
2966 
2967         save_in_reserved_area(R0, R1, R2, R3);
2968 
2969         Register src_ptr = R0;
2970         Register dst_ptr = R1;
2971         Register len     = R2;
2972         Register chk_off = R3;
2973         Register super_k = AARCH64_ONLY(R4) NOT_AARCH64(tmp);
2974 
2975         __ add(src_ptr, src, arrayOopDesc::base_offset_in_bytes(basic_type));
2976         __ add_ptr_scaled_int32(src_ptr, src_ptr, src_pos, shift);
2977 
2978         __ add(dst_ptr, dst, arrayOopDesc::base_offset_in_bytes(basic_type));
2979         __ add_ptr_scaled_int32(dst_ptr, dst_ptr, dst_pos, shift);
2980         __ load_klass(tmp, dst);
2981 
2982         int ek_offset = in_bytes(ObjArrayKlass::element_klass_offset());
2983         int sco_offset = in_bytes(Klass::super_check_offset_offset());
2984 
2985 #ifdef AARCH64
2986         __ raw_push(length, ZR); // Preserve length around *copyfunc_addr call
2987 
2988         __ mov(len, length);
2989         __ ldr(super_k, Address(tmp, ek_offset)); // super_k == R4 == length, so this load cannot be performed earlier
2990         // TODO-AARCH64: check whether it is faster to load super klass early by using tmp and additional mov.
2991         __ ldr_u32(chk_off, Address(super_k, sco_offset));
2992 #else // AARCH64
2993         __ ldr(super_k, Address(tmp, ek_offset));
2994 
2995         __ mov(len, length);
2996         __ ldr_u32(chk_off, Address(super_k, sco_offset));
2997         __ push(super_k);
2998 #endif // AARCH64
2999 
3000         __ call(copyfunc_addr, relocInfo::runtime_call_type);
3001 
3002 #ifndef PRODUCT
3003         if (PrintC1Statistics) {
3004           Label failed;
3005           __ cbnz_32(R0, failed);
3006           __ inc_counter((address)&Runtime1::_arraycopy_checkcast_cnt, tmp, tmp2);
3007           __ bind(failed);
3008         }
3009 #endif // PRODUCT
3010 
3011 #ifdef AARCH64
3012         __ raw_pop(length, ZR);
3013 #else
3014         __ add(SP, SP, wordSize);  // Drop super_k argument
3015 #endif // AARCH64
3016 
3017         __ cbz_32(R0, *stub->continuation());
3018         __ mvn_32(tmp, R0);
3019 
3020         // load saved arguments in slow case only
3021         restore_from_reserved_area(R0, R1, R2, R3);
3022 
3023         __ sub_32(length, length, tmp);
3024         __ add_32(src_pos, src_pos, tmp);
3025         __ add_32(dst_pos, dst_pos, tmp);
3026 
3027 #ifndef PRODUCT
3028         if (PrintC1Statistics) {
3029           __ inc_counter((address)&Runtime1::_arraycopy_checkcast_attempt_cnt, tmp, tmp2);
3030         }
3031 #endif
3032 
3033         __ b(*stub->entry());
3034 
3035         __ bind(cont);
3036       } else {
3037         __ b(*stub->entry(), eq);
3038         __ bind(cont);
3039       }
3040     }
3041   }
3042 
3043 #ifndef PRODUCT
3044   if (PrintC1Statistics) {
3045     address counter = Runtime1::arraycopy_count_address(basic_type);
3046     __ inc_counter(counter, tmp, tmp2);
3047   }
3048 #endif // !PRODUCT
3049 
3050   bool disjoint = (flags & LIR_OpArrayCopy::overlapping) == 0;
3051   bool aligned = (flags & LIR_OpArrayCopy::unaligned) == 0;
3052   const char *name;
3053   address entry = StubRoutines::select_arraycopy_function(basic_type, aligned, disjoint, name, false);
3054 
3055   Register src_ptr = R0;
3056   Register dst_ptr = R1;
3057   Register len     = R2;
3058 
3059   __ add(src_ptr, src, arrayOopDesc::base_offset_in_bytes(basic_type));
3060   __ add_ptr_scaled_int32(src_ptr, src_ptr, src_pos, shift);
3061 
3062   __ add(dst_ptr, dst, arrayOopDesc::base_offset_in_bytes(basic_type));
3063   __ add_ptr_scaled_int32(dst_ptr, dst_ptr, dst_pos, shift);
3064 
3065   __ mov(len, length);
3066 
3067   __ call(entry, relocInfo::runtime_call_type);
3068 
3069   __ bind(*stub->continuation());
3070 }
3071 
3072 #ifdef ASSERT
3073  // emit run-time assertion
3074 void LIR_Assembler::emit_assert(LIR_OpAssert* op) {
3075   assert(op->code() == lir_assert, "must be");
3076 
3077 #ifdef AARCH64
3078   __ NOT_IMPLEMENTED();
3079 #else
3080   if (op->in_opr1()->is_valid()) {
3081     assert(op->in_opr2()->is_valid(), "both operands must be valid");
3082     comp_op(op->condition(), op->in_opr1(), op->in_opr2(), op);
3083   } else {
3084     assert(op->in_opr2()->is_illegal(), "both operands must be illegal");
3085     assert(op->condition() == lir_cond_always, "no other conditions allowed");
3086   }
3087 
3088   Label ok;
3089   if (op->condition() != lir_cond_always) {
3090     AsmCondition acond;
3091     switch (op->condition()) {
3092       case lir_cond_equal:        acond = eq; break;
3093       case lir_cond_notEqual:     acond = ne; break;
3094       case lir_cond_less:         acond = lt; break;
3095       case lir_cond_lessEqual:    acond = le; break;
3096       case lir_cond_greaterEqual: acond = ge; break;
3097       case lir_cond_greater:      acond = gt; break;
3098       case lir_cond_aboveEqual:   acond = hs; break;
3099       case lir_cond_belowEqual:   acond = ls; break;
3100       default:                    ShouldNotReachHere();
3101     }
3102     __ b(ok, acond);
3103   }
3104   if (op->halt()) {
3105     const char* str = __ code_string(op->msg());
3106     __ stop(str);
3107   } else {
3108     breakpoint();
3109   }
3110   __ bind(ok);
3111 #endif // AARCH64
3112 }
3113 #endif // ASSERT
3114 
3115 void LIR_Assembler::emit_updatecrc32(LIR_OpUpdateCRC32* op) {
3116   fatal("CRC32 intrinsic is not implemented on this platform");
3117 }
3118 
3119 void LIR_Assembler::emit_lock(LIR_OpLock* op) {
3120   Register obj = op->obj_opr()->as_pointer_register();
3121   Register hdr = op->hdr_opr()->as_pointer_register();
3122   Register lock = op->lock_opr()->as_pointer_register();
3123   Register tmp = op->scratch_opr()->is_illegal() ? noreg :
3124                  op->scratch_opr()->as_pointer_register();
3125 
3126   if (!UseFastLocking) {
3127     __ b(*op->stub()->entry());
3128   } else if (op->code() == lir_lock) {
3129     assert(BasicLock::displaced_header_offset_in_bytes() == 0, "lock_reg must point to the displaced header");
3130     int null_check_offset = __ lock_object(hdr, obj, lock, tmp, *op->stub()->entry());
3131     if (op->info() != NULL) {
3132       add_debug_info_for_null_check(null_check_offset, op->info());
3133     }
3134   } else if (op->code() == lir_unlock) {
3135     __ unlock_object(hdr, obj, lock, tmp, *op->stub()->entry());
3136   } else {
3137     ShouldNotReachHere();
3138   }
3139   __ bind(*op->stub()->continuation());
3140 }
3141 
3142 
3143 void LIR_Assembler::emit_profile_call(LIR_OpProfileCall* op) {
3144   ciMethod* method = op->profiled_method();
3145   int bci          = op->profiled_bci();
3146   ciMethod* callee = op->profiled_callee();
3147 
3148   // Update counter for all call types
3149   ciMethodData* md = method->method_data_or_null();
3150   assert(md != NULL, "Sanity");
3151   ciProfileData* data = md->bci_to_data(bci);
3152   assert(data->is_CounterData(), "need CounterData for calls");
3153   assert(op->mdo()->is_single_cpu(),  "mdo must be allocated");
3154   Register mdo  = op->mdo()->as_register();
3155   assert(op->tmp1()->is_register(), "tmp1 must be allocated");
3156   Register tmp1 = op->tmp1()->as_pointer_register();
3157   assert_different_registers(mdo, tmp1);
3158   __ mov_metadata(mdo, md->constant_encoding());
3159   int mdo_offset_bias = 0;
3160   int max_offset = AARCH64_ONLY(4096 << LogBytesPerWord) NOT_AARCH64(4096);
3161   if (md->byte_offset_of_slot(data, CounterData::count_offset()) + data->size_in_bytes() >= max_offset) {
3162     // The offset is large so bias the mdo by the base of the slot so
3163     // that the ldr can use an immediate offset to reference the slots of the data
3164     mdo_offset_bias = md->byte_offset_of_slot(data, CounterData::count_offset());
3165     __ mov_slow(tmp1, mdo_offset_bias);
3166     __ add(mdo, mdo, tmp1);
3167   }
3168 
3169   Address counter_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()) - mdo_offset_bias);
3170   Bytecodes::Code bc = method->java_code_at_bci(bci);
3171   const bool callee_is_static = callee->is_loaded() && callee->is_static();
3172   // Perform additional virtual call profiling for invokevirtual and
3173   // invokeinterface bytecodes
3174   if ((bc == Bytecodes::_invokevirtual || bc == Bytecodes::_invokeinterface) &&
3175       !callee_is_static &&  // required for optimized MH invokes
3176       C1ProfileVirtualCalls) {
3177 
3178     assert(op->recv()->is_single_cpu(), "recv must be allocated");
3179     Register recv = op->recv()->as_register();
3180     assert_different_registers(mdo, tmp1, recv);
3181     assert(data->is_VirtualCallData(), "need VirtualCallData for virtual calls");
3182     ciKlass* known_klass = op->known_holder();
3183     if (C1OptimizeVirtualCallProfiling && known_klass != NULL) {
3184       // We know the type that will be seen at this call site; we can
3185       // statically update the MethodData* rather than needing to do
3186       // dynamic tests on the receiver type
3187 
3188       // NOTE: we should probably put a lock around this search to
3189       // avoid collisions by concurrent compilations
3190       ciVirtualCallData* vc_data = (ciVirtualCallData*) data;
3191       uint i;
3192       for (i = 0; i < VirtualCallData::row_limit(); i++) {
3193         ciKlass* receiver = vc_data->receiver(i);
3194         if (known_klass->equals(receiver)) {
3195           Address data_addr(mdo, md->byte_offset_of_slot(data,
3196                                                          VirtualCallData::receiver_count_offset(i)) -
3197                             mdo_offset_bias);
3198           __ ldr(tmp1, data_addr);
3199           __ add(tmp1, tmp1, DataLayout::counter_increment);
3200           __ str(tmp1, data_addr);
3201           return;
3202         }
3203       }
3204 
3205       // Receiver type not found in profile data; select an empty slot
3206 
3207       // Note that this is less efficient than it should be because it
3208       // always does a write to the receiver part of the
3209       // VirtualCallData rather than just the first time
3210       for (i = 0; i < VirtualCallData::row_limit(); i++) {
3211         ciKlass* receiver = vc_data->receiver(i);
3212         if (receiver == NULL) {
3213           Address recv_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_offset(i)) -
3214                             mdo_offset_bias);
3215           __ mov_metadata(tmp1, known_klass->constant_encoding());
3216           __ str(tmp1, recv_addr);
3217           Address data_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_count_offset(i)) -
3218                             mdo_offset_bias);
3219           __ ldr(tmp1, data_addr);
3220           __ add(tmp1, tmp1, DataLayout::counter_increment);
3221           __ str(tmp1, data_addr);
3222           return;
3223         }
3224       }
3225     } else {
3226       __ load_klass(recv, recv);
3227       Label update_done;
3228       type_profile_helper(mdo, mdo_offset_bias, md, data, recv, tmp1, &update_done);
3229       // Receiver did not match any saved receiver and there is no empty row for it.
3230       // Increment total counter to indicate polymorphic case.
3231       __ ldr(tmp1, counter_addr);
3232       __ add(tmp1, tmp1, DataLayout::counter_increment);
3233       __ str(tmp1, counter_addr);
3234 
3235       __ bind(update_done);
3236     }
3237   } else {
3238     // Static call
3239     __ ldr(tmp1, counter_addr);
3240     __ add(tmp1, tmp1, DataLayout::counter_increment);
3241     __ str(tmp1, counter_addr);
3242   }
3243 }
3244 
3245 void LIR_Assembler::emit_profile_type(LIR_OpProfileType* op) {
3246   fatal("Type profiling not implemented on this platform");
3247 }
3248 
3249 void LIR_Assembler::emit_delay(LIR_OpDelay*) {
3250   Unimplemented();
3251 }
3252 
3253 
3254 void LIR_Assembler::monitor_address(int monitor_no, LIR_Opr dst) {
3255   Address mon_addr = frame_map()->address_for_monitor_lock(monitor_no);
3256   __ add_slow(dst->as_pointer_register(), mon_addr.base(), mon_addr.disp());
3257 }
3258 
3259 
3260 void LIR_Assembler::align_backward_branch_target() {
3261   // TODO-AARCH64 review it
3262   // Some ARM processors do better with 8-byte branch target alignment
3263   __ align(8);
3264 }
3265 
3266 
3267 void LIR_Assembler::negate(LIR_Opr left, LIR_Opr dest) {
3268 
3269   if (left->is_single_cpu()) {
3270     assert (dest->type() == T_INT, "unexpected result type");
3271     assert (left->type() == T_INT, "unexpected left type");
3272     __ neg_32(dest->as_register(), left->as_register());
3273   } else if (left->is_double_cpu()) {
3274 #ifdef AARCH64
3275     __ neg(dest->as_register_lo(), left->as_register_lo());
3276 #else
3277     Register dest_lo = dest->as_register_lo();
3278     Register dest_hi = dest->as_register_hi();
3279     Register src_lo = left->as_register_lo();
3280     Register src_hi = left->as_register_hi();
3281     if (dest_lo == src_hi) {
3282       dest_lo = Rtemp;
3283     }
3284     __ rsbs(dest_lo, src_lo, 0);
3285     __ rsc(dest_hi, src_hi, 0);
3286     move_regs(dest_lo, dest->as_register_lo());
3287 #endif // AARCH64
3288   } else if (left->is_single_fpu()) {
3289     __ neg_float(dest->as_float_reg(), left->as_float_reg());
3290   } else if (left->is_double_fpu()) {
3291     __ neg_double(dest->as_double_reg(), left->as_double_reg());
3292   } else {
3293     ShouldNotReachHere();
3294   }
3295 }
3296 
3297 
3298 void LIR_Assembler::leal(LIR_Opr addr_opr, LIR_Opr dest) {
3299   LIR_Address* addr = addr_opr->as_address_ptr();
3300   if (addr->index()->is_illegal()) {
3301     jint c = addr->disp();
3302     if (!Assembler::is_arith_imm_in_range(c)) {
3303       BAILOUT("illegal arithmetic operand");
3304     }
3305     __ add(dest->as_pointer_register(), addr->base()->as_pointer_register(), c);
3306   } else {
3307     assert(addr->disp() == 0, "cannot handle otherwise");
3308 #ifdef AARCH64
3309     assert(addr->index()->is_double_cpu(), "should be");
3310 #endif // AARCH64
3311     __ add(dest->as_pointer_register(), addr->base()->as_pointer_register(),
3312            AsmOperand(addr->index()->as_pointer_register(), lsl, addr->scale()));
3313   }
3314 }
3315 
3316 
3317 void LIR_Assembler::rt_call(LIR_Opr result, address dest, const LIR_OprList* args, LIR_Opr tmp, CodeEmitInfo* info) {
3318   assert(!tmp->is_valid(), "don't need temporary");
3319   __ call(dest);
3320   if (info != NULL) {
3321     add_call_info_here(info);
3322   }
3323 }
3324 
3325 
3326 void LIR_Assembler::volatile_move_op(LIR_Opr src, LIR_Opr dest, BasicType type, CodeEmitInfo* info) {
3327 #ifdef AARCH64
3328   Unimplemented(); // TODO-AARCH64: Use stlr/ldar instructions for volatile load/store
3329 #else
3330   assert(src->is_double_cpu() && dest->is_address() ||
3331          src->is_address() && dest->is_double_cpu(),
3332          "Simple move_op is called for all other cases");
3333 
3334   int null_check_offset;
3335   if (dest->is_address()) {
3336     // Store
3337     const LIR_Address* addr = dest->as_address_ptr();
3338     const Register src_lo = src->as_register_lo();
3339     const Register src_hi = src->as_register_hi();
3340     assert(addr->index()->is_illegal() && addr->disp() == 0, "The address is simple already");
3341 
3342     if (src_lo < src_hi) {
3343       null_check_offset = __ offset();
3344       __ stmia(addr->base()->as_register(), RegisterSet(src_lo) | RegisterSet(src_hi));
3345     } else {
3346       assert(src_lo < Rtemp, "Rtemp is higher than any allocatable register");
3347       __ mov(Rtemp, src_hi);
3348       null_check_offset = __ offset();
3349       __ stmia(addr->base()->as_register(), RegisterSet(src_lo) | RegisterSet(Rtemp));
3350     }
3351   } else {
3352     // Load
3353     const LIR_Address* addr = src->as_address_ptr();
3354     const Register dest_lo = dest->as_register_lo();
3355     const Register dest_hi = dest->as_register_hi();
3356     assert(addr->index()->is_illegal() && addr->disp() == 0, "The address is simple already");
3357 
3358     null_check_offset = __ offset();
3359     if (dest_lo < dest_hi) {
3360       __ ldmia(addr->base()->as_register(), RegisterSet(dest_lo) | RegisterSet(dest_hi));
3361     } else {
3362       assert(dest_lo < Rtemp, "Rtemp is higher than any allocatable register");
3363       __ ldmia(addr->base()->as_register(), RegisterSet(dest_lo) | RegisterSet(Rtemp));
3364       __ mov(dest_hi, Rtemp);
3365     }
3366   }
3367 
3368   if (info != NULL) {
3369     add_debug_info_for_null_check(null_check_offset, info);
3370   }
3371 #endif // AARCH64
3372 }
3373 
3374 
3375 void LIR_Assembler::membar() {
3376   __ membar(MacroAssembler::StoreLoad, Rtemp);
3377 }
3378 
3379 void LIR_Assembler::membar_acquire() {
3380   __ membar(MacroAssembler::Membar_mask_bits(MacroAssembler::LoadLoad | MacroAssembler::LoadStore), Rtemp);
3381 }
3382 
3383 void LIR_Assembler::membar_release() {
3384   __ membar(MacroAssembler::Membar_mask_bits(MacroAssembler::StoreStore | MacroAssembler::LoadStore), Rtemp);
3385 }
3386 
3387 void LIR_Assembler::membar_loadload() {
3388   __ membar(MacroAssembler::LoadLoad, Rtemp);
3389 }
3390 
3391 void LIR_Assembler::membar_storestore() {
3392   __ membar(MacroAssembler::StoreStore, Rtemp);
3393 }
3394 
3395 void LIR_Assembler::membar_loadstore() {
3396   __ membar(MacroAssembler::LoadStore, Rtemp);
3397 }
3398 
3399 void LIR_Assembler::membar_storeload() {
3400   __ membar(MacroAssembler::StoreLoad, Rtemp);
3401 }
3402 
3403 void LIR_Assembler::on_spin_wait() {
3404   Unimplemented();
3405 }
3406 
3407 void LIR_Assembler::get_thread(LIR_Opr result_reg) {
3408   // Not used on ARM
3409   Unimplemented();
3410 }
3411 
3412 void LIR_Assembler::peephole(LIR_List* lir) {
3413 #ifdef AARCH64
3414   return; // TODO-AARCH64 implement peephole optimizations
3415 #endif
3416   LIR_OpList* inst = lir->instructions_list();
3417   const int inst_length = inst->length();
3418   for (int i = 0; i < inst_length; i++) {
3419     LIR_Op* op = inst->at(i);
3420     switch (op->code()) {
3421       case lir_cmp: {
3422         // Replace:
3423         //   cmp rX, y
3424         //   cmove [EQ] y, z, rX
3425         // with
3426         //   cmp rX, y
3427         //   cmove [EQ] illegalOpr, z, rX
3428         //
3429         // or
3430         //   cmp rX, y
3431         //   cmove [NE] z, y, rX
3432         // with
3433         //   cmp rX, y
3434         //   cmove [NE] z, illegalOpr, rX
3435         //
3436         // moves from illegalOpr should be removed when converting LIR to native assembly
3437 
3438         LIR_Op2* cmp = op->as_Op2();
3439         assert(cmp != NULL, "cmp LIR instruction is not an op2");
3440 
3441         if (i + 1 < inst_length) {
3442           LIR_Op2* cmove = inst->at(i + 1)->as_Op2();
3443           if (cmove != NULL && cmove->code() == lir_cmove) {
3444             LIR_Opr cmove_res = cmove->result_opr();
3445             bool res_is_op1 = cmove_res == cmp->in_opr1();
3446             bool res_is_op2 = cmove_res == cmp->in_opr2();
3447             LIR_Opr cmp_res, cmp_arg;
3448             if (res_is_op1) {
3449               cmp_res = cmp->in_opr1();
3450               cmp_arg = cmp->in_opr2();
3451             } else if (res_is_op2) {
3452               cmp_res = cmp->in_opr2();
3453               cmp_arg = cmp->in_opr1();
3454             } else {
3455               cmp_res = LIR_OprFact::illegalOpr;
3456               cmp_arg = LIR_OprFact::illegalOpr;
3457             }
3458 
3459             if (cmp_res != LIR_OprFact::illegalOpr) {
3460               LIR_Condition cond = cmove->condition();
3461               if (cond == lir_cond_equal && cmove->in_opr1() == cmp_arg) {
3462                 cmove->set_in_opr1(LIR_OprFact::illegalOpr);
3463               } else if (cond == lir_cond_notEqual && cmove->in_opr2() == cmp_arg) {
3464                 cmove->set_in_opr2(LIR_OprFact::illegalOpr);
3465               }
3466             }
3467           }
3468         }
3469         break;
3470       }
3471 
3472       default:
3473         break;
3474     }
3475   }
3476 }
3477 
3478 void LIR_Assembler::atomic_op(LIR_Code code, LIR_Opr src, LIR_Opr data, LIR_Opr dest, LIR_Opr tmp) {
3479   Register ptr = src->as_pointer_register();
3480 
3481   if (code == lir_xchg) {
3482 #ifdef AARCH64
3483     if (UseCompressedOops && data->is_oop()) {
3484       __ encode_heap_oop(tmp->as_pointer_register(), data->as_register());
3485     }
3486 #endif // AARCH64
3487   } else {
3488     assert (!data->is_oop(), "xadd for oops");
3489   }
3490 
3491 #ifndef AARCH64
3492   __ membar(MacroAssembler::Membar_mask_bits(MacroAssembler::StoreStore | MacroAssembler::LoadStore), Rtemp);
3493 #endif // !AARCH64
3494 
3495   Label retry;
3496   __ bind(retry);
3497 
3498   if ((data->type() == T_INT) || (data->is_oop() AARCH64_ONLY(&& UseCompressedOops))) {
3499     Register dst = dest->as_register();
3500     Register new_val = noreg;
3501 #ifdef AARCH64
3502     __ ldaxr_w(dst, ptr);
3503 #else
3504     __ ldrex(dst, Address(ptr));
3505 #endif
3506     if (code == lir_xadd) {
3507       Register tmp_reg = tmp->as_register();
3508       if (data->is_constant()) {
3509         assert_different_registers(dst, ptr, tmp_reg);
3510         __ add_32(tmp_reg, dst, data->as_constant_ptr()->as_jint());
3511       } else {
3512         assert_different_registers(dst, ptr, tmp_reg, data->as_register());
3513         __ add_32(tmp_reg, dst, data->as_register());
3514       }
3515       new_val = tmp_reg;
3516     } else {
3517       if (UseCompressedOops && data->is_oop()) {
3518         new_val = tmp->as_pointer_register();
3519       } else {
3520         new_val = data->as_register();
3521       }
3522       assert_different_registers(dst, ptr, new_val);
3523     }
3524 #ifdef AARCH64
3525     __ stlxr_w(Rtemp, new_val, ptr);
3526 #else
3527     __ strex(Rtemp, new_val, Address(ptr));
3528 #endif // AARCH64
3529 
3530 #ifdef AARCH64
3531   } else if ((data->type() == T_LONG) || (data->is_oop() && !UseCompressedOops)) {
3532     Register dst = dest->as_pointer_register();
3533     Register new_val = noreg;
3534     __ ldaxr(dst, ptr);
3535     if (code == lir_xadd) {
3536       Register tmp_reg = tmp->as_pointer_register();
3537       if (data->is_constant()) {
3538         assert_different_registers(dst, ptr, tmp_reg);
3539         jlong c = data->as_constant_ptr()->as_jlong();
3540         assert((jlong)((jint)c) == c, "overflow");
3541         __ add(tmp_reg, dst, (jint)c);
3542       } else {
3543         assert_different_registers(dst, ptr, tmp_reg, data->as_pointer_register());
3544         __ add(tmp_reg, dst, data->as_pointer_register());
3545       }
3546       new_val = tmp_reg;
3547     } else {
3548       new_val = data->as_pointer_register();
3549       assert_different_registers(dst, ptr, new_val);
3550     }
3551     __ stlxr(Rtemp, new_val, ptr);
3552 #else
3553   } else if (data->type() == T_LONG) {
3554     Register dst_lo = dest->as_register_lo();
3555     Register new_val_lo = noreg;
3556     Register dst_hi = dest->as_register_hi();
3557 
3558     assert(dst_hi->encoding() == dst_lo->encoding() + 1, "non aligned register pair");
3559     assert((dst_lo->encoding() & 0x1) == 0, "misaligned register pair");
3560 
3561     __ bind(retry);
3562     __ ldrexd(dst_lo, Address(ptr));
3563     if (code == lir_xadd) {
3564       Register tmp_lo = tmp->as_register_lo();
3565       Register tmp_hi = tmp->as_register_hi();
3566 
3567       assert(tmp_hi->encoding() == tmp_lo->encoding() + 1, "non aligned register pair");
3568       assert((tmp_lo->encoding() & 0x1) == 0, "misaligned register pair");
3569 
3570       if (data->is_constant()) {
3571         jlong c = data->as_constant_ptr()->as_jlong();
3572         assert((jlong)((jint)c) == c, "overflow");
3573         assert_different_registers(dst_lo, dst_hi, ptr, tmp_lo, tmp_hi);
3574         __ adds(tmp_lo, dst_lo, (jint)c);
3575         __ adc(tmp_hi, dst_hi, 0);
3576       } else {
3577         Register new_val_lo = data->as_register_lo();
3578         Register new_val_hi = data->as_register_hi();
3579         __ adds(tmp_lo, dst_lo, new_val_lo);
3580         __ adc(tmp_hi, dst_hi, new_val_hi);
3581         assert_different_registers(dst_lo, dst_hi, ptr, tmp_lo, tmp_hi, new_val_lo, new_val_hi);
3582       }
3583       new_val_lo = tmp_lo;
3584     } else {
3585       new_val_lo = data->as_register_lo();
3586       Register new_val_hi = data->as_register_hi();
3587 
3588       assert_different_registers(dst_lo, dst_hi, ptr, new_val_lo, new_val_hi);
3589       assert(new_val_hi->encoding() == new_val_lo->encoding() + 1, "non aligned register pair");
3590       assert((new_val_lo->encoding() & 0x1) == 0, "misaligned register pair");
3591     }
3592     __ strexd(Rtemp, new_val_lo, Address(ptr));
3593 #endif // AARCH64
3594   } else {
3595     ShouldNotReachHere();
3596   }
3597 
3598   __ cbnz_32(Rtemp, retry);
3599   __ membar(MacroAssembler::Membar_mask_bits(MacroAssembler::StoreLoad | MacroAssembler::StoreStore), Rtemp);
3600 
3601 #ifdef AARCH64
3602   if (UseCompressedOops && data->is_oop()) {
3603     __ decode_heap_oop(dest->as_register());
3604   }
3605 #endif // AARCH64
3606 }
3607 
3608 #undef __