1 /*
   2  * Copyright (c) 2000, 2018, Oracle and/or its affiliates. All rights reserved.
   3  * Copyright (c) 2014, Red Hat Inc. All rights reserved.
   4  * Copyright (c) 2015-2018, Azul Systems, Inc. All rights reserved.
   5  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   6  *
   7  * This code is free software; you can redistribute it and/or modify it
   8  * under the terms of the GNU General Public License version 2 only, as
   9  * published by the Free Software Foundation.
  10  *
  11  * This code is distributed in the hope that it will be useful, but WITHOUT
  12  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  13  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  14  * version 2 for more details (a copy is included in the LICENSE file that
  15  * accompanied this code).
  16  *
  17  * You should have received a copy of the GNU General Public License version
  18  * 2 along with this work; if not, write to the Free Software Foundation,
  19  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  20  *
  21  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  22  * or visit www.oracle.com if you need additional information or have any
  23  * questions.
  24  *
  25  */
  26 
  27 #include "precompiled.hpp"
  28 #include "asm/macroAssembler.inline.hpp"
  29 #include "asm/assembler.hpp"
  30 #include "c1/c1_CodeStubs.hpp"
  31 #include "c1/c1_Compilation.hpp"
  32 #include "c1/c1_LIRAssembler.hpp"
  33 #include "c1/c1_MacroAssembler.hpp"
  34 #include "c1/c1_Runtime1.hpp"
  35 #include "c1/c1_ValueStack.hpp"
  36 #include "ci/ciArrayKlass.hpp"
  37 #include "ci/ciInstance.hpp"
  38 #include "gc/shared/barrierSet.hpp"
  39 #include "gc/shared/cardTableBarrierSet.hpp"
  40 #include "gc/shared/collectedHeap.hpp"
  41 #include "nativeInst_aarch32.hpp"
  42 #include "oops/objArrayKlass.hpp"
  43 #include "runtime/frame.inline.hpp"
  44 #include "runtime/sharedRuntime.hpp"
  45 #include "vmreg_aarch32.inline.hpp"
  46 
  47 #include "register_aarch32.hpp"
  48 
  49 #ifndef PRODUCT
  50 #define COMMENT(x)   do { __ block_comment(x); } while (0)
  51 #else
  52 #define COMMENT(x)
  53 #endif
  54 
  55 NEEDS_CLEANUP // remove this definitions ?
  56 const Register IC_Klass    = rscratch2;   // where the IC klass is cached
  57 const Register SYNC_header = r0;   // synchronization header
  58 const Register SHIFT_count = r0;   // where count for shift operations must be
  59 
  60 #define __ _masm->
  61 
  62 
  63 static void select_different_registers(Register preserve,
  64                                        Register extra,
  65                                        Register &tmp1,
  66                                        Register &tmp2) {
  67   if (tmp1 == preserve) {
  68     assert_different_registers(tmp1, tmp2, extra);
  69     tmp1 = extra;
  70   } else if (tmp2 == preserve) {
  71     assert_different_registers(tmp1, tmp2, extra);
  72     tmp2 = extra;
  73   }
  74   assert_different_registers(preserve, tmp1, tmp2);
  75 }
  76 
  77 
  78 
  79 static void select_different_registers(Register preserve,
  80                                        Register extra,
  81                                        Register &tmp1,
  82                                        Register &tmp2,
  83                                        Register &tmp3) {
  84   if (tmp1 == preserve) {
  85     assert_different_registers(tmp1, tmp2, tmp3, extra);
  86     tmp1 = extra;
  87   } else if (tmp2 == preserve) {
  88     assert_different_registers(tmp1, tmp2, tmp3, extra);
  89     tmp2 = extra;
  90   } else if (tmp3 == preserve) {
  91     assert_different_registers(tmp1, tmp2, tmp3, extra);
  92     tmp3 = extra;
  93   }
  94   assert_different_registers(preserve, tmp1, tmp2, tmp3);
  95 }
  96 
  97 bool LIR_Assembler::is_small_constant(LIR_Opr opr) { Unimplemented(); return false; }
  98 
  99 
 100 LIR_Opr LIR_Assembler::receiverOpr() {
 101   return FrameMap::receiver_opr;
 102 }
 103 
 104 LIR_Opr LIR_Assembler::osrBufferPointer() {
 105   return FrameMap::as_pointer_opr(receiverOpr()->as_register());
 106 }
 107 
 108 //--------------fpu register translations-----------------------
 109 
 110 
 111 address LIR_Assembler::float_constant(float f) {
 112   address const_addr = __ float_constant(f);
 113   if (const_addr == NULL) {
 114     bailout("const section overflow");
 115     return __ code()->consts()->start();
 116   } else {
 117     return const_addr;
 118   }
 119 }
 120 
 121 
 122 address LIR_Assembler::double_constant(double d) {
 123   address const_addr = __ double_constant(d);
 124   if (const_addr == NULL) {
 125     bailout("const section overflow");
 126     return __ code()->consts()->start();
 127   } else {
 128     return const_addr;
 129   }
 130 }
 131 
 132 void LIR_Assembler::set_24bit_FPU() { Unimplemented(); }
 133 
 134 void LIR_Assembler::reset_FPU() { Unimplemented(); }
 135 
 136 void LIR_Assembler::fpop() { Unimplemented(); }
 137 
 138 void LIR_Assembler::fxch(int i) { Unimplemented(); }
 139 
 140 void LIR_Assembler::fld(int i) { Unimplemented(); }
 141 
 142 void LIR_Assembler::ffree(int i) { Unimplemented(); }
 143 
 144 void LIR_Assembler::breakpoint() { __ bkpt(0); }
 145 
 146 void LIR_Assembler::push(LIR_Opr opr) { Unimplemented(); }
 147 
 148 void LIR_Assembler::pop(LIR_Opr opr) { Unimplemented(); }
 149 
 150 //-------------------------------------------
 151 
 152 static Register as_reg(LIR_Opr op) {
 153   return op->is_double_cpu() ? op->as_register_lo() : op->as_register();
 154 }
 155 
 156 Address LIR_Assembler::as_Address(LIR_Address* addr) {
 157   // as_Address(LIR_Address*, Address::InsnDataType) should be used instead
 158   ShouldNotCallThis();
 159   return Address();
 160 }
 161 
 162 Address LIR_Assembler::as_Address_hi(LIR_Address* addr) {
 163   // as_Address_hi(LIR_Address*, Address::InsnDataType) should be used instead
 164   ShouldNotCallThis();
 165   return Address();
 166 }
 167 
 168 Address LIR_Assembler::as_Address_lo(LIR_Address* addr) {
 169   // as_Address_lo(LIR_Address*, Address::InsnDataType) should be used instead
 170   ShouldNotCallThis();
 171   return Address();
 172 }
 173 
 174 Address LIR_Assembler::as_Address(LIR_Address* addr, Register tmp, Address::InsnDataType type) {
 175   if (addr->base()->is_illegal()) {
 176     assert(addr->index()->is_illegal(), "must be illegal too");
 177     __ mov(tmp, addr->disp());
 178     return Address(tmp); // encoding is ok for any data type
 179   }
 180 
 181   Register base = addr->base()->as_pointer_register();
 182 
 183   if (addr->index()->is_illegal()) {
 184     return Address(base, addr->disp()).safe_for(type, _masm, tmp);
 185   } else if (addr->index()->is_cpu_register()) {
 186     assert(addr->disp() == 0, "must be");
 187     Register index = addr->index()->as_pointer_register();
 188     return Address(base, index, lsl(addr->scale())).safe_for(type, _masm, tmp);
 189   } else if (addr->index()->is_constant()) {
 190     intptr_t addr_offset = (addr->index()->as_constant_ptr()->as_jint() << addr->scale()) + addr->disp();
 191     return Address(base, addr_offset).safe_for(type, _masm, tmp);
 192   }
 193 
 194   Unimplemented();
 195   return Address();
 196 }
 197 
 198 Address LIR_Assembler::as_Address_hi(LIR_Address* addr, Address::InsnDataType type) {
 199   assert(type == Address::IDT_INT, "only to be used for accessing high word of jlong");
 200 
 201   if (addr->base()->is_illegal()) {
 202     assert(addr->index()->is_illegal(), "must be illegal too");
 203     __ mov(rscratch1, addr->disp() + wordSize);
 204     return Address(rscratch1); // encoding is ok for IDR_INT
 205   }
 206 
 207   Register base = addr->base()->as_pointer_register();
 208 
 209   if (addr->index()->is_illegal()) {
 210     return Address(base, addr->disp() + wordSize).safe_for(Address::IDT_INT, _masm, rscratch1);
 211   } else if (addr->index()->is_cpu_register()) {
 212     assert(addr->disp() == 0, "must be");
 213     Register index = addr->index()->as_pointer_register();
 214     __ add(rscratch1, base, wordSize);
 215     return Address(rscratch1, index, lsl(addr->scale())); // encoding is ok for IDT_INT
 216   } else if (addr->index()->is_constant()) {
 217     intptr_t addr_offset = (addr->index()->as_constant_ptr()->as_jint() << addr->scale()) + addr->disp() + wordSize;
 218     return Address(base, addr_offset).safe_for(Address::IDT_INT, _masm, rscratch1);
 219   }
 220 
 221   Unimplemented();
 222   return Address();
 223 }
 224 
 225 Address LIR_Assembler::as_Address_lo(LIR_Address* addr, Address::InsnDataType type) {
 226   return as_Address(addr, rscratch1, type);
 227 }
 228 
 229 
 230 void LIR_Assembler::osr_entry() {
 231   offsets()->set_value(CodeOffsets::OSR_Entry, code_offset());
 232   BlockBegin* osr_entry = compilation()->hir()->osr_entry();
 233   ValueStack* entry_state = osr_entry->state();
 234   int number_of_locks = entry_state->locks_size();
 235 
 236   // we jump here if osr happens with the interpreter
 237   // state set up to continue at the beginning of the
 238   // loop that triggered osr - in particular, we have
 239   // the following registers setup:
 240   //
 241   // r1: osr buffer
 242   //
 243 
 244   // build frame
 245   ciMethod* m = compilation()->method();
 246   __ build_frame(initial_frame_size_in_bytes(), bang_size_in_bytes());
 247 
 248   // OSR buffer is
 249   //
 250   // locals[nlocals-1..0]
 251   // monitors[0..number_of_locks]
 252   //
 253   // locals is a direct copy of the interpreter frame so in the osr buffer
 254   // so first slot in the local array is the last local from the interpreter
 255   // and last slot is local[0] (receiver) from the interpreter
 256   //
 257   // Similarly with locks. The first lock slot in the osr buffer is the nth lock
 258   // from the interpreter frame, the nth lock slot in the osr buffer is 0th lock
 259   // in the interpreter frame (the method lock if a sync method)
 260 
 261   // Initialize monitors in the compiled activation.
 262   //   r1: pointer to osr buffer
 263   //
 264   // All other registers are dead at this point and the locals will be
 265   // copied into place by code emitted in the IR.
 266 
 267   Register OSR_buf = osrBufferPointer()->as_pointer_register();
 268   { assert(frame::interpreter_frame_monitor_size() == BasicObjectLock::size(), "adjust code below");
 269     int monitor_offset = BytesPerWord * method()->max_locals() +
 270       (2 * BytesPerWord) * (number_of_locks - 1);
 271     // SharedRuntime::OSR_migration_begin() packs BasicObjectLocks in
 272     // the OSR buffer using 2 word entries: first the lock and then
 273     // the oop.
 274     for (int i = 0; i < number_of_locks; i++) {
 275       int slot_offset = monitor_offset - ((i * 2) * BytesPerWord);
 276 #ifdef ASSERT
 277       // verify the interpreter's monitor has a non-null object
 278       {
 279         Label L;
 280         __ ldr(rscratch1, Address(OSR_buf, slot_offset + 1*BytesPerWord));
 281         __ cbnz(rscratch1, L);
 282         __ stop("locked object is NULL");
 283         __ bind(L);
 284       }
 285 #endif
 286       __ ldr(rscratch1, Address(OSR_buf, slot_offset + 0));
 287       __ str(rscratch1, frame_map()->address_for_monitor_lock(i));
 288       __ ldr(rscratch1, Address(OSR_buf, slot_offset + 1*BytesPerWord));
 289       __ str(rscratch1, frame_map()->address_for_monitor_object(i));
 290     }
 291   }
 292 }
 293 
 294 
 295 // inline cache check; done before the frame is built.
 296 int LIR_Assembler::check_icache() {
 297   Register receiver = FrameMap::receiver_opr->as_register();
 298   Register ic_klass = IC_Klass;
 299   int start_offset = __ offset();
 300   __ inline_cache_check(receiver, ic_klass);
 301 
 302   // if icache check fails, then jump to runtime routine
 303   // Note: RECEIVER must still contain the receiver!
 304   Label dont;
 305   __ b(dont, Assembler::EQ);
 306   __ far_jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
 307 
 308   // We align the verified entry point unless the method body
 309   // (including its inline cache check) will fit in a single 64-byte
 310   // icache line.
 311   if (! method()->is_accessor() || __ offset() - start_offset > 4 * 4) {
 312     // force alignment after the cache check.
 313     __ align(CodeEntryAlignment);
 314   }
 315 
 316   __ bind(dont);
 317   return start_offset;
 318 }
 319 
 320 
 321 void LIR_Assembler::jobject2reg(jobject o, Register reg) {
 322   if (o == NULL) {
 323     __ mov(reg, 0);
 324   } else {
 325     __ movoop(reg, o, /*immediate*/true);
 326   }
 327 }
 328 
 329 void LIR_Assembler::deoptimize_trap(CodeEmitInfo *info) {
 330   __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::deoptimize_id)));
 331   add_call_info_here(info);
 332 }
 333 
 334 void LIR_Assembler::jobject2reg_with_patching(Register reg, CodeEmitInfo *info) {
 335   PatchingStub* patch = new PatchingStub(_masm, patching_id(info));
 336   __ relocate(oop_Relocation::spec(__ oop_recorder()->allocate_oop_index(NULL)));
 337   __ patchable_load(reg, pc());
 338   patching_epilog(patch, lir_patch_normal, reg, info);
 339 }
 340 
 341 // Return sp decrement needed to build a frame
 342 int LIR_Assembler::initial_frame_size_in_bytes() const {
 343   // We need to subtract size of saved backtrace structure in counting frame size
 344   return in_bytes(frame_map()->framesize_in_bytes()) - frame::get_frame_size() * wordSize;
 345 }
 346 
 347 int LIR_Assembler::emit_exception_handler() {
 348   // if the last instruction is a call (typically to do a throw which
 349   // is coming at the end after block reordering) the return address
 350   // must still point into the code area in order to avoid assertion
 351   // failures when searching for the corresponding bci => add a nop
 352   // (was bug 5/14/1999 - gri)
 353   __ nop();
 354 
 355   // generate code for exception handler
 356   address handler_base = __ start_a_stub(exception_handler_size());
 357   if (handler_base == NULL) {
 358     // not enough space left for the handler
 359     bailout("exception handler overflow");
 360     return -1;
 361   }
 362 
 363   int offset = code_offset();
 364 
 365   // the exception oop and pc are in r0, and r3
 366   // no other registers need to be preserved, so invalidate them
 367   __ invalidate_registers(false, true, false);
 368 
 369   // check that there is really an exception
 370   __ verify_not_null_oop(r0);
 371 
 372   // search an exception handler (r0: exception oop, r3: throwing pc)
 373   __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::handle_exception_from_callee_id)));  __ should_not_reach_here();
 374   guarantee(code_offset() - offset <= exception_handler_size(), "overflow");
 375   __ end_a_stub();
 376 
 377   return offset;
 378 }
 379 
 380 
 381 // Emit the code to remove the frame from the stack in the exception
 382 // unwind path.
 383 int LIR_Assembler::emit_unwind_handler() {
 384 #ifndef PRODUCT
 385   if (CommentedAssembly) {
 386     _masm->block_comment("Unwind handler");
 387   }
 388 #endif
 389 
 390   int offset = code_offset();
 391 
 392   // Fetch the exception from TLS and clear out exception related thread state
 393   __ ldr(r0, Address(rthread, JavaThread::exception_oop_offset()));
 394   __ mov(rscratch1, 0);
 395   __ str(rscratch1, Address(rthread, JavaThread::exception_oop_offset()));
 396   __ str(rscratch1, Address(rthread, JavaThread::exception_pc_offset()));
 397 
 398   __ bind(_unwind_handler_entry);
 399   __ verify_not_null_oop(r0);
 400 
 401   // Preform needed unlocking
 402   MonitorExitStub* stub = NULL;
 403   if (method()->is_synchronized()) {
 404     monitor_address(0, FrameMap::r1_opr);
 405     stub = new MonitorExitStub(FrameMap::r1_opr, true, 0);
 406     __ unlock_object(r5, r4, r1, *stub->entry());
 407     __ bind(*stub->continuation());
 408   }
 409 
 410   if (compilation()->env()->dtrace_method_probes()) {
 411     __ call_Unimplemented();
 412 #if 0
 413     // FIXME check exception_store is not clobbered below!
 414     __ movptr(Address(rsp, 0), rax);
 415     __ mov_metadata(Address(rsp, sizeof(void*)), method()->constant_encoding());
 416     __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_exit)));
 417 #endif
 418   }
 419 
 420   // remove the activation and dispatch to the unwind handler
 421   __ block_comment("remove_frame and dispatch to the unwind handler");
 422   __ remove_frame(initial_frame_size_in_bytes());
 423   __ far_jump(RuntimeAddress(Runtime1::entry_for(Runtime1::unwind_exception_id)));
 424 
 425   // Emit the slow path assembly
 426   if (stub != NULL) {
 427     stub->emit_code(this);
 428   }
 429 
 430   return offset;
 431 }
 432 
 433 
 434 int LIR_Assembler::emit_deopt_handler() {
 435   // if the last instruction is a call (typically to do a throw which
 436   // is coming at the end after block reordering) the return address
 437   // must still point into the code area in order to avoid assertion
 438   // failures when searching for the corresponding bci => add a nop
 439   // (was bug 5/14/1999 - gri)
 440   __ nop();
 441 
 442   // generate code for exception handler
 443   address handler_base = __ start_a_stub(deopt_handler_size());
 444   if (handler_base == NULL) {
 445     // not enough space left for the handler
 446     bailout("deopt handler overflow");
 447     return -1;
 448   }
 449 
 450   int offset = code_offset();
 451 
 452   __ adr(lr, pc());
 453   // deopt handler expects deopt pc already pushed to stack, since for C2
 454   // it's not possible to allocate any register to hold the value
 455   __ push(RegSet::of(lr), sp);
 456   __ far_jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
 457   guarantee(code_offset() - offset <= deopt_handler_size(), "overflow");
 458   __ end_a_stub();
 459 
 460   return offset;
 461 }
 462 
 463 void LIR_Assembler::add_debug_info_for_branch(address adr, CodeEmitInfo* info) {
 464   _masm->code_section()->relocate(adr, relocInfo::poll_type);
 465   int pc_offset = code_offset();
 466   flush_debug_info(pc_offset);
 467   info->record_debug_info(compilation()->debug_info_recorder(), pc_offset);
 468   if (info->exception_handlers() != NULL) {
 469     compilation()->add_exception_handlers_for_pco(pc_offset, info->exception_handlers());
 470   }
 471 }
 472 
 473 void LIR_Assembler::return_op(LIR_Opr result) {
 474   assert(result->is_illegal() || !result->is_single_cpu() || result->as_register() == r0, "word returns are in r0,");
 475   // Pop the stack before the safepoint code
 476   __ remove_frame(initial_frame_size_in_bytes());
 477 
 478   if (StackReservedPages > 0 && compilation()->has_reserved_stack_access()) {
 479     __ reserved_stack_check();
 480   }
 481 
 482   address polling_page(os::get_polling_page());
 483   __ read_polling_page(rscratch2, polling_page, relocInfo::poll_return_type);
 484   __ ret(lr);
 485 }
 486 
 487 int LIR_Assembler::safepoint_poll(LIR_Opr tmp, CodeEmitInfo* info) {
 488   address polling_page(os::get_polling_page());
 489   guarantee(info != NULL, "Shouldn't be NULL");
 490   assert(os::is_poll_address(polling_page), "should be");
 491   __ get_polling_page(rscratch2, polling_page, relocInfo::poll_type);
 492   add_debug_info_for_branch(info);  // This isn't just debug info:
 493                                     // it's the oop map
 494   __ read_polling_page(rscratch2, relocInfo::poll_type);
 495   return __ offset();
 496 }
 497 
 498 void LIR_Assembler::move_regs(Register from_reg, Register to_reg) {
 499   if (from_reg != to_reg) {
 500     __ mov(to_reg, from_reg);
 501   }
 502 }
 503 
 504 void LIR_Assembler::swap_reg(Register a, Register b) {
 505   Unimplemented();
 506 }
 507 
 508 void LIR_Assembler::const2reg(LIR_Opr src, LIR_Opr dest, LIR_PatchCode patch_code, CodeEmitInfo* info) {
 509   assert(src->is_constant(), "should not call otherwise");
 510   assert(dest->is_register(), "should not call otherwise");
 511   LIR_Const* c = src->as_constant_ptr();
 512 
 513   switch (c->type()) {
 514     case T_INT: {
 515       assert(patch_code == lir_patch_none, "no patching handled here");
 516       __ mov(dest->as_register(), c->as_jint_bits());
 517       break;
 518     }
 519 
 520     case T_ADDRESS: {
 521       assert(patch_code == lir_patch_none, "no patching handled here");
 522       __ mov(dest->as_register(), c->as_jint());
 523       break;
 524     }
 525 
 526     case T_LONG: {
 527       assert(patch_code == lir_patch_none, "no patching handled here");
 528       __ mov(dest->as_register_lo(), c->as_jint_lo_bits());
 529       __ mov(dest->as_register_hi(), c->as_jint_hi_bits());
 530       break;
 531     }
 532 
 533     case T_OBJECT: {
 534         if (patch_code == lir_patch_none) {
 535           jobject2reg(c->as_jobject(), dest->as_register());
 536         } else {
 537           jobject2reg_with_patching(dest->as_register(), info);
 538         }
 539       break;
 540     }
 541 
 542     case T_METADATA: {
 543       if (patch_code != lir_patch_none) {
 544         klass2reg_with_patching(dest->as_register(), info);
 545       } else {
 546         __ mov_metadata(dest->as_register(), c->as_metadata());
 547       }
 548       break;
 549     }
 550 
 551     case T_FLOAT: {
 552         if(dest->is_single_fpu()) {
 553             if (__ operand_valid_for_float_immediate(c->as_jfloat())) {
 554                 __ vmov_f32(dest->as_float_reg(), c->as_jfloat());
 555             } else {
 556                 __ lea(rscratch1, InternalAddress(float_constant(c->as_jfloat())));
 557                 __ vldr_f32(dest->as_float_reg(), Address(rscratch1));
 558             }
 559         } else {
 560             assert(patch_code == lir_patch_none, "no patching handled here");
 561             __ mov(dest->as_register(), c->as_jint_bits());
 562         }
 563       break;
 564     }
 565 
 566     case T_DOUBLE: {
 567         if(dest->is_double_fpu()) {
 568             if (__ operand_valid_for_double_immediate(c->as_jdouble())) {
 569                 __ vmov_f64(dest->as_double_reg(), c->as_jdouble());
 570             } else {
 571                 __ lea(rscratch1, InternalAddress(double_constant(c->as_jdouble())));
 572                 __ vldr_f64(dest->as_double_reg(), Address(rscratch1));
 573             }
 574         } else {
 575             assert(patch_code == lir_patch_none, "no patching handled here");
 576             __ mov(dest->as_register_lo(), c->as_jint_lo_bits());
 577             __ mov(dest->as_register_hi(), c->as_jint_hi_bits());
 578         }
 579       break;
 580     }
 581 
 582     default:
 583       ShouldNotReachHere();
 584   }
 585 }
 586 
 587 void LIR_Assembler::const2stack(LIR_Opr src, LIR_Opr dest) {
 588   LIR_Const* c = src->as_constant_ptr();
 589   switch (c->type()) {
 590   case T_OBJECT:
 591     {
 592       if (! c->as_jobject()) {
 593         __ mov(rscratch1, 0);
 594         __ str(rscratch1, frame_map()->address_for_slot(dest->single_stack_ix()));
 595       } else {
 596         const2reg(src, FrameMap::rscratch1_opr, lir_patch_none, NULL);
 597         reg2stack(FrameMap::rscratch1_opr, dest, c->type(), false);
 598       }
 599     }
 600     break;
 601   case T_ADDRESS:
 602     {
 603       const2reg(src, FrameMap::rscratch1_opr, lir_patch_none, NULL);
 604       reg2stack(FrameMap::rscratch1_opr, dest, c->type(), false);
 605     }
 606   case T_INT:
 607   case T_FLOAT:
 608     {
 609       __ mov(rscratch1, c->as_jint_bits());
 610       __ str(rscratch1, frame_map()->address_for_slot(dest->single_stack_ix()));
 611     }
 612     break;
 613   case T_LONG:
 614   case T_DOUBLE:
 615     {
 616         __ mov(rscratch1, c->as_jint_lo());
 617         __ str(rscratch1, frame_map()->address_for_slot(dest->double_stack_ix(),
 618                                                         lo_word_offset_in_bytes));
 619         if (c->as_jint_lo() != c->as_jint_hi())
 620             __ mov(rscratch1, c->as_jint_hi());
 621         __ str(rscratch1, frame_map()->address_for_slot(dest->double_stack_ix(),
 622                                                         hi_word_offset_in_bytes));
 623     }
 624     break;
 625   default:
 626     ShouldNotReachHere();
 627   }
 628 }
 629 
 630 /*
 631  * For now this code can load only zero constants as in aarch32.
 632  * It seems like this implementation can break some tests in future.
 633  * TODO: ensure, write test, and rewrite if need.
 634  */
 635 void LIR_Assembler::const2mem(LIR_Opr src, LIR_Opr dest, BasicType type, CodeEmitInfo* info, bool wide) {
 636   assert(src->is_constant(), "should not call otherwise");
 637   LIR_Const* c = src->as_constant_ptr();
 638   LIR_Address* to_addr = dest->as_address_ptr();
 639 
 640   void (Assembler::* insn)(Register Rt, const Address &adr, Assembler::Condition cnd);
 641 
 642   __ mov(rscratch2, 0);
 643 
 644   int null_check_here = code_offset();
 645 
 646   Address::InsnDataType idt = Address::toInsnDataType(type);
 647   switch (type) {
 648   case T_ADDRESS:
 649     assert(c->as_jint() == 0, "should be");
 650     insn = &Assembler::str;
 651     break;
 652   case T_LONG: {
 653     assert(c->as_jlong() == 0, "should be");
 654     insn = &Assembler::str;
 655     Address addr = as_Address_hi(to_addr, Address::IDT_INT);
 656     null_check_here = code_offset();
 657     __ str(rscratch2, addr);
 658     idt = Address::IDT_INT;
 659     break;
 660   }
 661   case T_INT:
 662     assert(c->as_jint() == 0, "should be");
 663     insn = &Assembler::str;
 664     break;
 665   case T_OBJECT:
 666   case T_ARRAY:
 667     assert(c->as_jobject() == 0, "should be");
 668     insn = &Assembler::str;
 669     break;
 670   case T_CHAR:
 671   case T_SHORT:
 672     assert(c->as_jint() == 0, "should be");
 673     insn = &Assembler::strh;
 674     break;
 675   case T_BOOLEAN:
 676   case T_BYTE:
 677     assert(c->as_jint() == 0, "should be");
 678     insn = &Assembler::strb;
 679     break;
 680   default:
 681     ShouldNotReachHere();
 682     insn = &Assembler::str;  // unreachable
 683   }
 684 
 685   (_masm->*insn)(rscratch2, as_Address(to_addr, idt), Assembler::C_DFLT);
 686   if (info) add_debug_info_for_null_check(null_check_here, info);
 687 }
 688 
 689 void LIR_Assembler::reg2reg(LIR_Opr src, LIR_Opr dest) {
 690   assert(src->is_register(), "should not call otherwise");
 691   assert(dest->is_register(), "should not call otherwise");
 692 
 693   // move between cpu-registers
 694   if (dest->is_single_cpu()) {
 695     if (src->type() == T_LONG) {
 696       // Can do LONG -> OBJECT
 697       __ stop("investigate how \"LONG -> OBJECT\" works especially when high part is != 0");
 698       move_regs(src->as_register_lo(), dest->as_register());
 699       return;
 700     }
 701     if(src->is_single_fpu()) {
 702         __ vmov_f32(dest->as_register(), src->as_float_reg());
 703     } else {
 704         assert(src->is_single_cpu(), "must match");
 705         if (src->type() == T_OBJECT) {
 706           __ verify_oop(src->as_register());
 707         }
 708         move_regs(src->as_register(), dest->as_register());
 709     }
 710   } else if (dest->is_double_cpu()) {
 711       if(src->is_double_fpu()) {
 712         __ vmov_f64(dest->as_register_lo(), dest->as_register_hi(), src->as_double_reg());
 713       } else {
 714         assert(src->is_double_cpu(), "must match");
 715         Register f_lo = src->as_register_lo();
 716         Register f_hi = src->as_register_hi();
 717         Register t_lo = dest->as_register_lo();
 718         Register t_hi = dest->as_register_hi();
 719         assert(f_hi != f_lo, "must be different");
 720         assert(t_hi != t_lo, "must be different");
 721         check_register_collision(t_lo, &f_hi);
 722         move_regs(f_lo, t_lo);
 723         move_regs(f_hi, t_hi);
 724       }
 725   } else if (dest->is_single_fpu()) {
 726       if(src->is_single_cpu()) {
 727         __ vmov_f32(dest->as_float_reg(), src->as_register());
 728       } else {
 729         __ vmov_f32(dest->as_float_reg(), src->as_float_reg());
 730       }
 731   } else if (dest->is_double_fpu()) {
 732       if(src->is_double_cpu()) {
 733         __ vmov_f64(dest->as_double_reg(), src->as_register_lo(), src->as_register_hi());
 734       } else {
 735         __ vmov_f64(dest->as_double_reg(), src->as_double_reg());
 736       }
 737   } else {
 738     ShouldNotReachHere();
 739   }
 740 }
 741 
 742 void LIR_Assembler::reg2stack(LIR_Opr src, LIR_Opr dest, BasicType type, bool pop_fpu_stack) {
 743   if (src->is_single_cpu()) {
 744     if (type == T_ARRAY || type == T_OBJECT) {
 745       __ str(src->as_register(), frame_map()->address_for_slot(dest->single_stack_ix()));
 746       __ verify_oop(src->as_register());
 747     } else {
 748       __ str(src->as_register(), frame_map()->address_for_slot(dest->single_stack_ix()));
 749     }
 750 
 751   } else if (src->is_double_cpu()) {
 752     Address dest_addr_LO = frame_map()->address_for_slot(dest->double_stack_ix(), lo_word_offset_in_bytes);
 753     __ strd(src->as_register_lo(), src->as_register_hi(), dest_addr_LO);
 754   } else if (src->is_single_fpu()) {
 755     Address dest_addr = frame_map()->address_for_slot(dest->single_stack_ix());
 756     __ vstr_f32(src->as_float_reg(), dest_addr.safe_for(Address::IDT_FLOAT, _masm, rscratch1));
 757   } else if (src->is_double_fpu()) {
 758     Address dest_addr = frame_map()->address_for_slot(dest->double_stack_ix());
 759     __ vstr_f64(src->as_double_reg(), dest_addr.safe_for(Address::IDT_DOUBLE, _masm, rscratch1));
 760   } else {
 761     ShouldNotReachHere();
 762   }
 763 
 764 }
 765 
 766 
 767 void LIR_Assembler::reg2mem(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_PatchCode patch_code, CodeEmitInfo* info, bool pop_fpu_stack, bool wide, bool /* unaligned */) {
 768   LIR_Address* to_addr = dest->as_address_ptr();
 769 
 770   if (type == T_ARRAY || type == T_OBJECT) {
 771     __ verify_oop(src->as_register());
 772   }
 773 
 774   PatchingStub* patch = NULL;
 775   if (patch_code != lir_patch_none) {
 776     assert(to_addr->disp() != 0, "must have");
 777 
 778     patch = new PatchingStub(_masm, PatchingStub::access_field_id);
 779     address const_addr = __ address_constant(0);
 780     if (!const_addr) BAILOUT("patchable offset");
 781     __ relocate(section_word_Relocation::spec(const_addr, CodeBuffer::SECT_CONSTS));
 782     __ patchable_load(rscratch1, const_addr);
 783     patching_epilog(patch, patch_code, to_addr->base()->as_register(), info);
 784 
 785     to_addr = new LIR_Address(to_addr->base(), FrameMap::rscratch1_opr, to_addr->type());
 786   }
 787 
 788 
 789   int null_check_here = code_offset();
 790   switch (type) {
 791     case T_FLOAT:
 792         if(src->is_single_fpu()) {
 793             Address addr = as_Address(to_addr, Address::IDT_FLOAT);
 794             null_check_here = code_offset();
 795             __ vstr_f32(src->as_float_reg(), addr);
 796             break;
 797         } // fall through at FPUless system
 798     case T_ARRAY:   // fall through
 799     case T_OBJECT:  // fall through
 800     case T_ADDRESS: // fall though
 801     case T_INT: {
 802       Address addr = as_Address(to_addr, Address::toInsnDataType(type));
 803       null_check_here = code_offset();
 804       __ str(src->as_register(), addr);
 805       break;
 806     }
 807     case T_METADATA:
 808       // We get here to store a method pointer to the stack to pass to
 809       // a dtrace runtime call. This can't work on 64 bit with
 810       // compressed klass ptrs: T_METADATA can be a compressed klass
 811       // ptr or a 64 bit method pointer.
 812       ShouldNotReachHere();
 813 //      __ str(src->as_register(), as_Address(to_addr));
 814       break;
 815 
 816     case T_DOUBLE:
 817         if(src->is_double_fpu()) {
 818             Address addr = as_Address(to_addr, Address::IDT_DOUBLE);
 819             null_check_here = code_offset();
 820             __ vstr_f64(src->as_double_reg(), addr);
 821             break;
 822         } // fall through at FPUless system
 823     case T_LONG: {
 824       Address addr = as_Address_lo(to_addr, Address::IDT_LONG);
 825       null_check_here = code_offset();
 826       null_check_here += __ strd(src->as_register_lo(), src->as_register_hi(), addr);
 827       break;
 828     }
 829 
 830     case T_BYTE:    // fall through
 831     case T_BOOLEAN: {
 832       Address addr = as_Address(to_addr, Address::toInsnDataType(type));
 833       null_check_here = code_offset();
 834       __ strb(src->as_register(), addr);
 835       break;
 836     }
 837     case T_CHAR:    // fall through
 838     case T_SHORT: {
 839       Address addr = as_Address(to_addr, Address::toInsnDataType(type));
 840       null_check_here = code_offset();
 841       __ strh(src->as_register(), addr);
 842       break;
 843     }
 844     default:
 845       ShouldNotReachHere();
 846   }
 847 
 848   if (info != NULL) {
 849     add_debug_info_for_null_check(null_check_here, info);
 850   }
 851 }
 852 
 853 
 854 void LIR_Assembler::stack2reg(LIR_Opr src, LIR_Opr dest, BasicType type) {
 855   assert(src->is_stack(), "should not call otherwise");
 856   assert(dest->is_register(), "should not call otherwise");
 857 
 858   if (dest->is_single_cpu()) {
 859     if (type == T_ARRAY || type == T_OBJECT) {
 860       __ ldr(dest->as_register(), frame_map()->address_for_slot(src->single_stack_ix()));
 861       __ verify_oop(dest->as_register());
 862     } else {
 863       __ ldr(dest->as_register(), frame_map()->address_for_slot(src->single_stack_ix()));
 864     }
 865 
 866   } else if (dest->is_double_cpu()) {
 867     Address src_addr_LO = frame_map()->address_for_slot(src->double_stack_ix(), lo_word_offset_in_bytes);
 868     __ ldrd(dest->as_register_lo(), dest->as_register_hi(), src_addr_LO);
 869   } else if (dest->is_single_fpu()) {
 870     Address src_addr = frame_map()->address_for_slot(src->single_stack_ix());
 871     __ vldr_f32(dest->as_float_reg(), src_addr.safe_for(Address::IDT_FLOAT, _masm, rscratch1));
 872   } else if (dest->is_double_fpu()) {
 873     Address src_addr = frame_map()->address_for_slot(src->double_stack_ix());
 874     __ vldr_f64(dest->as_double_reg(), src_addr.safe_for(Address::IDT_DOUBLE, _masm, rscratch1));
 875   } else {
 876     ShouldNotReachHere();
 877   }
 878 }
 879 
 880 void LIR_Assembler::klass2reg_with_patching(Register reg, CodeEmitInfo* info) {
 881   PatchingStub* patch = new PatchingStub(_masm, PatchingStub::load_klass_id);
 882   __ relocate(metadata_Relocation::spec(__ oop_recorder()->allocate_metadata_index(NULL)));
 883   __ patchable_load(reg, pc());
 884   patching_epilog(patch, lir_patch_normal, reg, info);
 885 }
 886 
 887 void LIR_Assembler::stack2stack(LIR_Opr src, LIR_Opr dest, BasicType type) {
 888 
 889   LIR_Opr temp;
 890   if (type == T_LONG || type == T_DOUBLE)
 891     temp = FrameMap::rscratch_long_opr;
 892   else
 893     temp = FrameMap::rscratch1_opr;
 894 
 895   stack2reg(src, temp, src->type());
 896   reg2stack(temp, dest, dest->type(), false);
 897 }
 898 
 899 
 900 void LIR_Assembler::mem2reg(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_PatchCode patch_code, CodeEmitInfo* info, bool wide, bool /* unaligned */) {
 901   LIR_Address* from_addr = src->as_address_ptr();
 902 
 903   if (from_addr->base()->type() == T_OBJECT) {
 904     __ verify_oop(from_addr->base()->as_pointer_register());
 905   }
 906 
 907   PatchingStub* patch = NULL;
 908   if (patch_code != lir_patch_none) {
 909     assert(from_addr->disp() != 0, "must have");
 910 
 911     patch = new PatchingStub(_masm, PatchingStub::access_field_id);
 912     address const_addr = __ address_constant(0);
 913     if (!const_addr) BAILOUT("patchable offset");
 914     __ relocate(section_word_Relocation::spec(const_addr, CodeBuffer::SECT_CONSTS));
 915     __ patchable_load(rscratch1, const_addr);
 916     patching_epilog(patch, patch_code, from_addr->base()->as_register(), info);
 917 
 918     from_addr = new LIR_Address(from_addr->base(), FrameMap::rscratch1_opr, from_addr->type());
 919   }
 920 
 921   int null_check_here = code_offset();
 922 
 923   switch (type) {
 924     case T_FLOAT:
 925         if(dest->is_single_fpu()){
 926             Address addr = as_Address(from_addr, Address::IDT_FLOAT);
 927             null_check_here = code_offset();
 928             __ vldr_f32(dest->as_float_reg(), addr);
 929               break;
 930         }  // fall through at FPUless systems
 931     case T_ARRAY:   // fall through
 932     case T_OBJECT:  // fall through
 933     case T_ADDRESS: // fall through
 934     case T_INT: {
 935       Address addr = as_Address(from_addr, Address::toInsnDataType(type));
 936       null_check_here = code_offset();
 937       __ ldr(dest->as_register(), addr);
 938       break;
 939     }
 940     case T_METADATA:
 941       // We get here to store a method pointer to the stack to pass to
 942       // a dtrace runtime call. This can't work on 64 bit with
 943       // compressed klass ptrs: T_METADATA can be a compressed klass
 944       // ptr or a 64 bit method pointer.
 945       ShouldNotReachHere();
 946 //      __ ldr(dest->as_register(), as_Address(from_addr));
 947       break;
 948     case T_DOUBLE:
 949         if(dest->is_double_fpu()){
 950             Address addr = as_Address(from_addr, Address::IDT_DOUBLE);
 951             null_check_here = code_offset();
 952             __ vldr_f64(dest->as_double_reg(), addr);
 953               break;
 954         } // fall through at FPUless systems
 955     case T_LONG: {
 956       Address addr = as_Address_lo(from_addr, Address::IDT_LONG);
 957       null_check_here = code_offset();
 958       null_check_here += __ ldrd(dest->as_register_lo(), dest->as_register_hi(), addr);
 959       break;
 960     }
 961 
 962     case T_BYTE: {
 963       Address addr =  as_Address(from_addr, Address::IDT_BYTE);
 964       null_check_here = code_offset();
 965       __ ldrsb(dest->as_register(), addr);
 966       break;
 967     }
 968     case T_BOOLEAN: {
 969       Address addr = as_Address(from_addr, Address::IDT_BOOLEAN);
 970       null_check_here = code_offset();
 971       __ ldrb(dest->as_register(), addr);
 972       break;
 973     }
 974 
 975     case T_CHAR: {
 976       Address addr = as_Address(from_addr, Address::IDT_CHAR);
 977       null_check_here = code_offset();
 978       __ ldrh(dest->as_register(), addr);
 979       break;
 980     }
 981     case T_SHORT: {
 982       Address addr = as_Address(from_addr, Address::IDT_SHORT);
 983       null_check_here = code_offset();
 984       __ ldrsh(dest->as_register(), addr);
 985       break;
 986     }
 987 
 988     default:
 989       ShouldNotReachHere();
 990   }
 991 
 992   if (type == T_ARRAY || type == T_OBJECT) {
 993     __ verify_oop(dest->as_register());
 994   }
 995 
 996   if (info != NULL) {
 997     add_debug_info_for_null_check(null_check_here, info);
 998   }
 999 }
1000 
1001 
1002 int LIR_Assembler::array_element_size(BasicType type) const {
1003   int elem_size = type2aelembytes(type);
1004   return exact_log2(elem_size);
1005 }
1006 
1007 void LIR_Assembler::emit_op3(LIR_Op3* op) {
1008   Register Rdividend = op->in_opr1()->as_register();
1009   Register Rdivisor  = op->in_opr2()->as_register();
1010   Register Rscratch  = op->in_opr3()->as_register();
1011   Register Rresult   = op->result_opr()->as_register();
1012   int divisor = -1;
1013 
1014   /*
1015   TODO: For some reason, using the Rscratch that gets passed in is
1016   not possible because the register allocator does not see the tmp reg
1017   as used, and assignes it the same register as Rdividend. We use rscratch1
1018    instead.
1019 
1020   assert(Rdividend != Rscratch, "");
1021   assert(Rdivisor  != Rscratch, "");
1022   */
1023 
1024   if (Rdivisor == noreg && is_power_of_2(divisor)) {
1025     // convert division by a power of two into some shifts and logical operations
1026   }
1027 
1028   assert(op->code() == lir_irem || op->code() == lir_idiv, "should be irem or idiv");
1029   bool want_remainder = op->code() == lir_irem;
1030 
1031   __ divide(Rresult, Rdividend, Rdivisor, 32, want_remainder);
1032 }
1033 
1034 void LIR_Assembler::emit_opBranch(LIR_OpBranch* op) {
1035 #ifdef ASSERT
1036   assert(op->block() == NULL || op->block()->label() == op->label(), "wrong label");
1037   if (op->block() != NULL)  _branch_target_blocks.append(op->block());
1038   if (op->ublock() != NULL) _branch_target_blocks.append(op->ublock());
1039 #endif
1040 
1041   if (op->cond() == lir_cond_always) {
1042     if (op->info() != NULL) add_debug_info_for_branch(op->info());
1043     __ b(*(op->label()));
1044   } else {
1045     Assembler::Condition acond;
1046     if (op->code() == lir_cond_float_branch) {
1047       bool is_unordered = (op->ublock() == op->block());
1048       // Assembler::EQ does not permit unordered branches, so we add
1049       // another branch here.  Likewise, Assembler::NE does not permit
1050       // ordered branches.
1051       if (is_unordered && op->cond() == lir_cond_equal
1052           || !is_unordered && op->cond() == lir_cond_notEqual)
1053         __ b(*(op->ublock()->label()), Assembler::VS);
1054       switch(op->cond()) {
1055       case lir_cond_equal:        acond = Assembler::EQ; break;
1056       case lir_cond_notEqual:     acond = Assembler::NE; break;
1057       case lir_cond_less:         acond = (is_unordered ? Assembler::LT : Assembler::LO); break;
1058       case lir_cond_lessEqual:    acond = (is_unordered ? Assembler::LE : Assembler::LS); break;
1059       case lir_cond_greaterEqual: acond = (is_unordered ? Assembler::HS : Assembler::GE); break;
1060       case lir_cond_greater:      acond = (is_unordered ? Assembler::HI : Assembler::GT); break;
1061       default:                    ShouldNotReachHere();
1062         acond = Assembler::EQ;  // unreachable
1063       }
1064     } else {
1065       switch (op->cond()) {
1066         case lir_cond_equal:        acond = Assembler::EQ; break;
1067         case lir_cond_notEqual:     acond = Assembler::NE; break;
1068         case lir_cond_less:         acond = Assembler::LT; break;
1069         case lir_cond_greaterEqual: acond = Assembler::GE; break;
1070         case lir_cond_lessEqual:    acond = Assembler::LE; break;
1071         case lir_cond_greater:      acond = Assembler::GT; break;
1072         case lir_cond_belowEqual:   acond = Assembler::LS; break;
1073         case lir_cond_aboveEqual:   acond = Assembler::HS; break;
1074         default:                         ShouldNotReachHere();
1075           acond = Assembler::EQ;  // unreachable
1076       }
1077       if (op->type() == T_LONG) {
1078         // a special trick here to be able to effectively compare jlongs
1079         // for the lessEqual and greater conditions the jlong operands are swapped
1080         // during comparison and hence should use mirror condition in conditional
1081         // instruction
1082         // see LIR_Assembler::comp_op and LIR_Assembler::cmove
1083         switch (op->cond()) {
1084           case lir_cond_lessEqual: acond = Assembler::GE;  break;
1085           case lir_cond_greater:   acond = Assembler::LT;    break;
1086         }
1087       }
1088     }
1089     __ b(*(op->label()), acond);
1090   }
1091 }
1092 
1093 FloatRegister LIR_Assembler::as_float_reg(LIR_Opr doubleReg) {
1094     assert(doubleReg->is_double_fpu(), "must be f64");
1095     return as_FloatRegister(doubleReg->fpu_regnrLo());
1096 }
1097 
1098 void LIR_Assembler::emit_opConvert(LIR_OpConvert* op) {
1099   LIR_Opr src  = op->in_opr();
1100   LIR_Opr dest = op->result_opr();
1101 
1102   switch (op->bytecode()) {
1103     case Bytecodes::_i2f:
1104       {
1105         __ vmov_f32(dest->as_float_reg(), src->as_register());
1106         __ vcvt_f32_s32(dest->as_float_reg(), dest->as_float_reg());
1107         break;
1108       }
1109     case Bytecodes::_i2d:
1110       {
1111         __ vmov_f32(as_float_reg(dest), src->as_register());
1112         __ vcvt_f64_s32(dest->as_double_reg(), as_float_reg(dest));
1113         break;
1114       }
1115     case Bytecodes::_f2d:
1116       {
1117         __ vcvt_f64_f32(dest->as_double_reg(), src->as_float_reg());
1118         break;
1119       }
1120     case Bytecodes::_d2f:
1121       {
1122         __ vcvt_f32_f64(dest->as_float_reg(), src->as_double_reg());
1123         break;
1124       }
1125     case Bytecodes::_i2c:
1126       {
1127         __ uxth(dest->as_register(), src->as_register());
1128         break;
1129       }
1130     case Bytecodes::_i2l:
1131       {
1132         const Register dst_hi = dest->as_register_hi();
1133         const Register dst_lo = dest->as_register_lo();
1134         const Register src_lo = as_reg(src);
1135         __ mov(dst_lo, src_lo);
1136         __ asr(dst_hi, src_lo, 31);
1137         break;
1138       }
1139     case Bytecodes::_i2s:
1140       {
1141         __ sxth(dest->as_register(), src->as_register());
1142         break;
1143       }
1144     case Bytecodes::_i2b:
1145       {
1146         __ sxtb(dest->as_register(), src->as_register());
1147         break;
1148       }
1149     case Bytecodes::_l2i:
1150       {
1151         assert(dest->is_single_cpu(), "must be single register");
1152         __ mov(dest->as_register(), src->as_register_lo());
1153         break;
1154       }
1155     case Bytecodes::_f2i:
1156       {
1157         __ vcvt_s32_f32(src->as_float_reg(), src->as_float_reg());
1158         __ vmov_f32(dest->as_register(), src->as_float_reg());
1159         break;
1160       }
1161     case Bytecodes::_d2i:
1162       {
1163         __ vcvt_s32_f64(as_float_reg(src), src->as_double_reg());
1164         __ vmov_f32(dest->as_register(), as_float_reg(src));
1165         break;
1166       }
1167     default: ShouldNotReachHere();
1168   }
1169 }
1170 
1171 void LIR_Assembler::emit_alloc_obj(LIR_OpAllocObj* op) {
1172   if (op->init_check()) {
1173     __ ldrb(rscratch1, Address(op->klass()->as_register(),
1174                                InstanceKlass::init_state_offset()));
1175     __ cmp(rscratch1, InstanceKlass::fully_initialized);
1176     add_debug_info_for_null_check_here(op->stub()->info());
1177     __ b(*op->stub()->entry(), Assembler::NE);
1178   }
1179   __ allocate_object(op->obj()->as_register(),
1180                      op->tmp1()->as_register(),
1181                      op->tmp2()->as_register(),
1182                      op->header_size(),
1183                      op->object_size(),
1184                      op->klass()->as_register(),
1185                      *op->stub()->entry());
1186   __ bind(*op->stub()->continuation());
1187 }
1188 
1189 void LIR_Assembler::emit_alloc_array(LIR_OpAllocArray* op) {
1190   Register len =  as_reg(op->len());
1191 
1192   if (UseSlowPath ||
1193       (!UseFastNewObjectArray && (op->type() == T_OBJECT || op->type() == T_ARRAY)) ||
1194       (!UseFastNewTypeArray   && (op->type() != T_OBJECT && op->type() != T_ARRAY))) {
1195     __ b(*op->stub()->entry());
1196   } else {
1197     Register tmp1 = op->tmp1()->as_register();
1198     Register tmp2 = op->tmp2()->as_register();
1199     Register tmp3 = op->tmp3()->as_register();
1200     if (len == tmp1) {
1201       tmp1 = tmp3;
1202     } else if (len == tmp2) {
1203       tmp2 = tmp3;
1204     } else if (len == tmp3) {
1205       // everything is ok
1206     } else {
1207       __ mov(tmp3, len);
1208     }
1209     __ allocate_array(op->obj()->as_register(),
1210                       len,
1211                       tmp1,
1212                       tmp2,
1213                       arrayOopDesc::header_size(op->type()),
1214                       array_element_size(op->type()),
1215                       op->klass()->as_register(),
1216                       *op->stub()->entry());
1217   }
1218   __ bind(*op->stub()->continuation());
1219 }
1220 
1221 void LIR_Assembler::type_profile_helper(Register mdo,
1222                                         ciMethodData *md, ciProfileData *data,
1223                                         Register recv, Label* update_done) {
1224   for (uint i = 0; i < ReceiverTypeData::row_limit(); i++) {
1225     Label next_test;
1226     // See if the receiver is receiver[n].
1227     __ lea(rscratch2, Address(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_offset(i))));
1228     __ ldr(rscratch1, Address(rscratch2));
1229     __ cmp(recv, rscratch1);
1230     __ b(next_test, Assembler::NE);
1231     Address data_addr(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_count_offset(i)));
1232     __ addptr(data_addr, DataLayout::counter_increment);
1233     __ b(*update_done);
1234     __ bind(next_test);
1235   }
1236 
1237   // Didn't find receiver; find next empty slot and fill it in
1238   for (uint i = 0; i < ReceiverTypeData::row_limit(); i++) {
1239     Label next_test;
1240     __ lea(rscratch2,
1241            Address(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_offset(i))));
1242     Address recv_addr(rscratch2);
1243     __ ldr(rscratch1, recv_addr);
1244     __ cbnz(rscratch1, next_test);
1245     __ str(recv, recv_addr);
1246     __ mov(rscratch1, DataLayout::counter_increment);
1247     __ lea(rscratch2, Address(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_count_offset(i))));
1248     __ str(rscratch1, Address(rscratch2));
1249     __ b(*update_done);
1250     __ bind(next_test);
1251   }
1252 }
1253 
1254 void LIR_Assembler::emit_typecheck_helper(LIR_OpTypeCheck *op, Label* success, Label* failure, Label* obj_is_null) {
1255   // we always need a stub for the failure case.
1256   CodeStub* stub = op->stub();
1257   Register obj = op->object()->as_register();
1258   Register k_RInfo = op->tmp1()->as_register();
1259   Register klass_RInfo = op->tmp2()->as_register();
1260   Register dst = op->result_opr()->as_register();
1261   ciKlass* k = op->klass();
1262   Register Rtmp1 = noreg;
1263 
1264   // check if it needs to be profiled
1265   ciMethodData* md;
1266   ciProfileData* data;
1267 
1268   const bool should_profile = op->should_profile();
1269   if (should_profile) {
1270     ciMethod* method = op->profiled_method();
1271     assert(method != NULL, "Should have method");
1272     int bci = op->profiled_bci();
1273     md = method->method_data_or_null();
1274     assert(md != NULL, "Sanity");
1275     data = md->bci_to_data(bci);
1276     assert(data != NULL,                "need data for type check");
1277     assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
1278   }
1279   Label profile_cast_success, profile_cast_failure;
1280   Label *success_target = op->should_profile() ? &profile_cast_success : success;
1281   Label *failure_target = op->should_profile() ? &profile_cast_failure : failure;
1282 
1283   if (obj == k_RInfo) {
1284     k_RInfo = dst;
1285   } else if (obj == klass_RInfo) {
1286     klass_RInfo = dst;
1287   }
1288   if (k->is_loaded()) {
1289     select_different_registers(obj, dst, k_RInfo, klass_RInfo);
1290   } else {
1291     Rtmp1 = op->tmp3()->as_register();
1292     select_different_registers(obj, dst, k_RInfo, klass_RInfo, Rtmp1);
1293   }
1294 
1295   assert_different_registers(obj, k_RInfo, klass_RInfo);
1296 
1297   if (should_profile) {
1298     Label not_null;
1299     __ cbnz(obj, not_null);
1300     // Object is null; update MDO and exit
1301     Register mdo  = klass_RInfo;
1302       __ mov_metadata(mdo, md->constant_encoding());
1303       Address data_addr
1304         = __ form_address(rscratch2, mdo,
1305                           md->byte_offset_of_slot(data, DataLayout::flags_offset()),
1306                           0);
1307       __ ldrb(rscratch1, data_addr);
1308       __ orr(rscratch1, rscratch1, BitData::null_seen_byte_constant());
1309       __ strb(rscratch1, data_addr);
1310       __ b(*obj_is_null);
1311       __ bind(not_null);
1312     } else {
1313     __ cbz(obj, *obj_is_null);
1314   }
1315 
1316   if (!k->is_loaded()) {
1317     klass2reg_with_patching(k_RInfo, op->info_for_patch());
1318   } else {
1319     __ mov_metadata(k_RInfo, k->constant_encoding());
1320   }
1321   __ verify_oop(obj);
1322 
1323   if (op->fast_check()) {
1324     // get object class
1325     // not a safepoint as obj null check happens earlier
1326     __ load_klass(rscratch1, obj);
1327     __ cmp( rscratch1, k_RInfo);
1328 
1329     __ b(*failure_target, Assembler::NE);
1330     // successful cast, fall through to profile or jump
1331   } else {
1332     // get object class
1333     // not a safepoint as obj null check happens earlier
1334     __ load_klass(klass_RInfo, obj);
1335     if (k->is_loaded()) {
1336       // See if we get an immediate positive hit
1337       __ ldr(rscratch1, Address(klass_RInfo, long(k->super_check_offset())));
1338       __ cmp(k_RInfo, rscratch1);
1339       if ((juint)in_bytes(Klass::secondary_super_cache_offset()) != k->super_check_offset()) {
1340         __ b(*failure_target, Assembler::NE);
1341         // successful cast, fall through to profile or jump
1342       } else {
1343         // See if we get an immediate positive hit
1344         __ b(*success_target, Assembler::EQ);
1345         // check for self
1346         __ cmp(klass_RInfo, k_RInfo);
1347         __ b(*success_target, Assembler::EQ);
1348 
1349         __ push(klass_RInfo);
1350         __ push(k_RInfo);
1351         __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
1352         __ ldr(klass_RInfo, Address(__ post(sp, 2 * wordSize)));
1353 
1354         // result is a boolean
1355         __ cbz(klass_RInfo, *failure_target);
1356         // successful cast, fall through to profile or jump
1357       }
1358     } else {
1359       // perform the fast part of the checking logic
1360       __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, NULL);
1361       // call out-of-line instance of __ check_klass_subtype_slow_path(...):
1362       __ push(klass_RInfo);
1363       __ push(k_RInfo);
1364       __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
1365       __ ldr(k_RInfo, Address(__ post(sp, 2 * wordSize)));
1366 
1367       // result is a boolean
1368       __ cbz(k_RInfo, *failure_target);
1369       // successful cast, fall through to profile or jump
1370     }
1371   }
1372   if (should_profile) {
1373     Register mdo  = klass_RInfo, recv = k_RInfo;
1374     __ bind(profile_cast_success);
1375     __ mov_metadata(mdo, md->constant_encoding());
1376     __ load_klass(recv, obj);
1377     Label update_done;
1378     type_profile_helper(mdo, md, data, recv, success);
1379     __ b(*success);
1380 
1381     __ bind(profile_cast_failure);
1382     __ mov_metadata(mdo, md->constant_encoding());
1383     Address counter_addr
1384       = __ form_address(rscratch2, mdo,
1385                         md->byte_offset_of_slot(data, CounterData::count_offset()),
1386                         0);
1387     __ ldr(rscratch1, counter_addr);
1388     __ sub(rscratch1, rscratch1, DataLayout::counter_increment);
1389     __ str(rscratch1, counter_addr);
1390     __ b(*failure);
1391   }
1392   __ b(*success);
1393 }
1394 
1395 
1396 void LIR_Assembler::emit_opTypeCheck(LIR_OpTypeCheck* op) {
1397   LIR_Code code = op->code();
1398   if (code == lir_store_check) {
1399     Register value = op->object()->as_register();
1400     Register array = op->array()->as_register();
1401     Register k_RInfo = op->tmp1()->as_register();
1402     Register klass_RInfo = op->tmp2()->as_register();
1403     Register Rtmp1 = op->tmp3()->as_register();
1404 
1405     CodeStub* stub = op->stub();
1406 
1407     // check if it needs to be profiled
1408     ciMethodData* md;
1409     ciProfileData* data;
1410 
1411     const bool should_profile = op->should_profile();
1412     if (should_profile) {
1413       ciMethod* method = op->profiled_method();
1414       assert(method != NULL, "Should have method");
1415       int bci = op->profiled_bci();
1416       md = method->method_data_or_null();
1417       assert(md != NULL, "Sanity");
1418       data = md->bci_to_data(bci);
1419       assert(data != NULL,                "need data for type check");
1420       assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
1421     }
1422     Label profile_cast_success, profile_cast_failure, done;
1423     Label *success_target = should_profile ? &profile_cast_success : &done;
1424     Label *failure_target = should_profile ? &profile_cast_failure : stub->entry();
1425 
1426     if (should_profile) {
1427       Label not_null;
1428       __ cbnz(value, not_null);
1429       // Object is null; update MDO and exit
1430       Register mdo  = klass_RInfo;
1431       __ mov_metadata(mdo, md->constant_encoding());
1432       Address data_addr
1433         = __ form_address(rscratch2, mdo,
1434                           md->byte_offset_of_slot(data, DataLayout::flags_offset()),
1435                           0);
1436       __ ldrb(rscratch1, data_addr);
1437       __ orr(rscratch1, rscratch1, BitData::null_seen_byte_constant());
1438       __ strb(rscratch1, data_addr);
1439       __ b(done);
1440       __ bind(not_null);
1441     } else {
1442       __ cbz(value, done);
1443     }
1444 
1445     add_debug_info_for_null_check_here(op->info_for_exception());
1446     __ load_klass(k_RInfo, array);
1447     __ load_klass(klass_RInfo, value);
1448 
1449     // get instance klass (it's already uncompressed)
1450     __ ldr(k_RInfo, Address(k_RInfo, ObjArrayKlass::element_klass_offset()));
1451     // perform the fast part of the checking logic
1452     __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, NULL);
1453     // call out-of-line instance of __ check_klass_subtype_slow_path(...):
1454     __ push(klass_RInfo);
1455     __ push(k_RInfo);
1456     __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
1457     __ ldr(k_RInfo, Address(__ post(sp, 2 * wordSize)));
1458     // result is a boolean
1459     __ cbz(k_RInfo, *failure_target);
1460     // fall through to the success case
1461 
1462     if (should_profile) {
1463       Register mdo  = klass_RInfo, recv = k_RInfo;
1464       __ bind(profile_cast_success);
1465       __ mov_metadata(mdo, md->constant_encoding());
1466       __ load_klass(recv, value);
1467       type_profile_helper(mdo, md, data, recv, &done);
1468       __ b(done);
1469 
1470       __ bind(profile_cast_failure);
1471       __ mov_metadata(mdo, md->constant_encoding());
1472       Address counter_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
1473       __ lea(rscratch2, counter_addr);
1474       __ ldr(rscratch1, Address(rscratch2));
1475       __ sub(rscratch1, rscratch1, DataLayout::counter_increment);
1476       __ str(rscratch1, Address(rscratch2));
1477       __ b(*stub->entry());
1478     }
1479 
1480     __ bind(done);
1481   } else if (code == lir_checkcast) {
1482     Register obj = op->object()->as_register();
1483     Register dst = op->result_opr()->as_register();
1484     Label success;
1485     emit_typecheck_helper(op, &success, op->stub()->entry(), &success);
1486     __ bind(success);
1487     if (dst != obj) {
1488       __ mov(dst, obj);
1489     }
1490   } else if (code == lir_instanceof) {
1491     Register obj = op->object()->as_register();
1492     Register dst = op->result_opr()->as_register();
1493     Label success, failure, done;
1494     emit_typecheck_helper(op, &success, &failure, &failure);
1495     __ bind(failure);
1496     __ mov(dst, 0);
1497     __ b(done);
1498     __ bind(success);
1499     __ mov(dst, 1);
1500     __ bind(done);
1501   } else {
1502     ShouldNotReachHere();
1503   }
1504 }
1505 
1506 // TODO: reuse masm cmpxchgw
1507 void LIR_Assembler::casw(Register addr, Register newval, Register cmpval, Register result) {
1508   assert(newval != cmpval, "must be different");
1509   Label retry_load, nope;
1510   // flush and load exclusive from the memory location
1511   // and fail if it is not what we expect
1512   __ bind(retry_load);
1513   __ ldrex(result, addr);
1514   __ cmp(result, cmpval);
1515   __ mov(result, 1, Assembler::NE);
1516   __ b(nope, Assembler::NE);
1517   // if we store+flush with no intervening write rscratch1 wil be zero
1518   __ strex(result, newval, addr);
1519   // retry so we only ever return after a load fails to compare
1520   // ensures we don't return a stale value after a failed write.
1521   __ cbnz(result, retry_load);
1522   __ membar(__ AnyAny);
1523   __ bind(nope);
1524 }
1525 
1526 void LIR_Assembler::casl(Register addr, Register newval_lo, Register newval_hi, Register cmpval_lo,  Register cmpval_hi,  Register tmp_lo, Register tmp_hi, Register result) {
1527   assert(newval_lo->successor() == newval_hi, "must be contiguous");
1528   assert(tmp_lo->successor() == tmp_hi, "must be contiguous");
1529   assert(tmp_lo->encoding_nocheck() % 2 == 0,  "Must be an even register");
1530   assert_different_registers(newval_lo, newval_hi, cmpval_lo,  cmpval_hi,  tmp_lo, tmp_hi);
1531 
1532   Label retry_load, nope;
1533   // flush and load exclusive from the memory location
1534   // and fail if it is not what we expect
1535   __ bind(retry_load);
1536   __ mov(result, 1);
1537   __ ldrexd(tmp_lo, addr);
1538   __ cmp(tmp_lo, cmpval_lo);
1539   __ b(nope, Assembler::NE);
1540   __ cmp(tmp_hi, cmpval_hi);
1541   __ b(nope, Assembler::NE);
1542   // if we store+flush with no intervening write rscratch1 wil be zero
1543   __ strexd(result, newval_lo, addr);
1544   // retry so we only ever return after a load fails to compare
1545   // ensures we don't return a stale value after a failed write.
1546   __ cbnz(result, retry_load);
1547   __ membar(__ AnyAny);
1548   __ bind(nope);
1549 }
1550 
1551 
1552 void LIR_Assembler::emit_compare_and_swap(LIR_OpCompareAndSwap* op) {
1553   Register addr;
1554   if (op->addr()->is_register()) {
1555     addr = as_reg(op->addr());
1556   } else {
1557     assert(op->addr()->is_address(), "what else?");
1558     LIR_Address* addr_ptr = op->addr()->as_address_ptr();
1559     assert(addr_ptr->disp() == 0, "need 0 disp");
1560     assert(addr_ptr->index() == LIR_OprDesc::illegalOpr(), "need 0 index");
1561     addr = as_reg(addr_ptr->base());
1562   }
1563   Register result = as_reg(op->result_opr());
1564   if (op->code() == lir_cas_obj || op->code() == lir_cas_int) {
1565     Register newval = as_reg(op->new_value());
1566     Register cmpval = as_reg(op->cmp_value());
1567     casw(addr, newval, cmpval, result);
1568   } else if (op->code() == lir_cas_long){
1569     Register newval_lo = op->new_value()->as_register_lo();
1570     Register newval_hi = op->new_value()->as_register_hi();
1571     Register cmpval_lo = op->cmp_value()->as_register_lo();
1572     Register cmpval_hi = op->cmp_value()->as_register_hi();
1573     Register tmp_lo = op->tmp1()->as_register_lo();
1574     Register tmp_hi = op->tmp1()->as_register_hi();
1575     casl(addr, newval_lo, newval_hi, cmpval_lo, cmpval_hi, tmp_lo, tmp_hi, result);
1576   } else {
1577       ShouldNotReachHere();
1578   }
1579 }
1580 
1581 static void patch_condition(address start_insn, address end_insn, Assembler::Condition cond) {
1582   for (uint32_t* insn_p = (uint32_t*) start_insn; (address) insn_p < end_insn; ++insn_p) {
1583     uint32_t insn = *insn_p;
1584     assert((insn >> 28) == Assembler::AL, "instructions in patch"
1585      " should allow conditional form and be in ALWAYS condition");
1586     *insn_p = (insn & 0x0fffffff) | (cond << 28);
1587   }
1588 }
1589 
1590 void LIR_Assembler::cmove(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Opr result, BasicType type) {
1591 
1592   Assembler::Condition acond, ncond;
1593   switch (condition) {
1594   case lir_cond_equal:        acond = Assembler::EQ; ncond = Assembler::NE; break;
1595   case lir_cond_notEqual:     acond = Assembler::NE; ncond = Assembler::EQ; break;
1596   case lir_cond_less:         acond = Assembler::LT; ncond = Assembler::GE; break;
1597   case lir_cond_greaterEqual: acond = Assembler::GE; ncond = Assembler::LT; break;
1598   case lir_cond_lessEqual:    acond = Assembler::LE; ncond = Assembler::GT; break;
1599   case lir_cond_greater:      acond = Assembler::GT; ncond = Assembler::LE; break;
1600   case lir_cond_belowEqual:   Unimplemented(); return;
1601   case lir_cond_aboveEqual:   Unimplemented(); return;
1602   default:                    ShouldNotReachHere(); return;
1603   }
1604   if (type == T_LONG) {
1605       // for the lessEqual and greater conditions the jlong operands are swapped
1606       // during comparison and hence should use mirror condition in conditional
1607       // instruction. see comp_op())
1608     switch (condition) {
1609     case lir_cond_lessEqual:    acond = Assembler::GE; ncond = Assembler::LT; break;
1610     case lir_cond_greater:      acond = Assembler::LT; ncond = Assembler::GE; break;
1611     }
1612   }
1613 
1614   address true_instrs = __ pc();
1615   if (opr1->is_cpu_register()) {
1616     reg2reg(opr1, result);
1617   } else if (opr1->is_stack()) {
1618     stack2reg(opr1, result, result->type());
1619   } else if (opr1->is_constant()) {
1620     const2reg(opr1, result, lir_patch_none, NULL);
1621   } else {
1622     ShouldNotReachHere();
1623   }
1624   patch_condition(true_instrs, __ pc(), acond);
1625 
1626   address false_instrs = __ pc();
1627   if (opr2->is_cpu_register()) {
1628     reg2reg(opr2, result);
1629   } else if (opr2->is_stack()) {
1630     stack2reg(opr2, result, result->type());
1631   } else if (opr2->is_constant()) {
1632     const2reg(opr2, result, lir_patch_none, NULL);
1633   } else {
1634     ShouldNotReachHere();
1635   }
1636   patch_condition(false_instrs, __ pc(), ncond);
1637 }
1638 
1639 void LIR_Assembler::arith_op(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dest, CodeEmitInfo* info, bool pop_fpu_stack) {
1640   assert(info == NULL, "should never be used, idiv/irem and ldiv/lrem not handled by this method");
1641 
1642   if (left->is_single_cpu()) {
1643     assert(left->type() != T_FLOAT, "expect integer type");
1644     assert(right->type() != T_FLOAT, "expect integer type");
1645     assert(dest->type() != T_FLOAT, "expect integer type");
1646 
1647     Register lreg = left->as_register();
1648     Register dreg = as_reg(dest);
1649 
1650     if (right->is_single_cpu()) {
1651       // cpu register - cpu register
1652 
1653       assert((left->type() == T_INT || left->type() == T_OBJECT)
1654              && right->type() == T_INT
1655              && dest->type() == T_INT,
1656              "should be");
1657       Register rreg = right->as_register();
1658       switch (code) {
1659       case lir_add: __ add (dest->as_register(), lreg, rreg); break;
1660       case lir_sub: __ sub (dest->as_register(), lreg, rreg); break;
1661       case lir_mul: __ mul (dest->as_register(), lreg, rreg); break;
1662       default:      ShouldNotReachHere();
1663       }
1664 
1665     } else if (right->is_double_cpu()) {
1666       ShouldNotReachHere(); // for obj+long op the generator casts long to int before invoking add
1667     } else if (right->is_constant()) {
1668       // cpu register - constant
1669       jint c = right->as_constant_ptr()->as_jint();
1670 
1671       assert(code == lir_add || code == lir_sub, "mismatched arithmetic op");
1672       if (c == 0 && dreg == lreg) {
1673         COMMENT("effective nop elided");
1674         return;
1675       }
1676 
1677       if (Assembler::operand_valid_for_add_sub_immediate(c)) {
1678         switch (code) {
1679         case lir_add: __ add(dreg, lreg, c); break;
1680         case lir_sub: __ sub(dreg, lreg, c); break;
1681         default: ShouldNotReachHere();
1682         }
1683       } else {
1684         __ mov(rscratch1, c);
1685         switch (code) {
1686         case lir_add: __ add(dreg, lreg, rscratch1); break;
1687         case lir_sub: __ sub(dreg, lreg, rscratch1); break;
1688         default: ShouldNotReachHere();
1689         }
1690       }
1691     } else {
1692       ShouldNotReachHere();
1693     }
1694 
1695   } else if (left->is_double_cpu()) {
1696     assert(left->type() != T_DOUBLE, "expect integer type");
1697     assert(right->type() != T_DOUBLE, "expect integer type");
1698     assert(dest->type() != T_DOUBLE, "expect integer type");
1699 
1700     Register lreg_lo = left->as_register_lo();
1701     Register lreg_hi = left->as_register_hi();
1702 
1703     if (right->is_double_cpu()) {
1704       // cpu register - cpu register
1705       Register rreg_lo = right->as_register_lo();
1706       Register rreg_hi = right->as_register_hi();
1707       Register dreg_lo = dest->as_register_lo();
1708       Register dreg_hi = dest->as_register_hi();
1709       if (code == lir_add || code == lir_sub) {
1710         check_register_collision(dreg_lo, &lreg_hi, &rreg_hi);
1711       }
1712       switch (code) {
1713       case lir_add: __ adds (dreg_lo, lreg_lo, rreg_lo);
1714                     __ adc (dreg_hi, lreg_hi, rreg_hi); break;
1715       case lir_sub: __ subs (dreg_lo, lreg_lo, rreg_lo);
1716                     __ sbc (dreg_hi, lreg_hi, rreg_hi); break;
1717       case lir_mul: __ mult_long (dreg_lo, dreg_hi,
1718                         lreg_lo, lreg_hi, rreg_lo, rreg_hi); break;
1719       default:
1720         ShouldNotReachHere();
1721       }
1722 
1723     } else if (right->is_constant()) {
1724       const jint c_lo = right->as_constant_ptr()->as_jint_lo_bits();
1725       const jint c_hi = right->as_constant_ptr()->as_jint_hi_bits();
1726       const Register dreg_lo = dest->as_register_lo();
1727       const Register dreg_hi = dest->as_register_hi();
1728       assert(code == lir_add || code == lir_sub, "mismatched arithmetic op");
1729       if (c_lo == 0 && c_hi == 0 && dreg_lo == lreg_lo && dreg_hi == lreg_hi) {
1730         COMMENT("effective nop elided");
1731         return;
1732       }
1733       check_register_collision(dreg_lo, &lreg_hi, NULL, rscratch2);
1734       switch (code) {
1735         case lir_add:
1736           if (Assembler::operand_valid_for_add_sub_immediate(c_lo))
1737             __ adds(dreg_lo, lreg_lo, c_lo);
1738           else {
1739             __ mov(rscratch1, c_lo);
1740             __ adds(dreg_lo, lreg_lo, rscratch1);
1741           }
1742           if (Assembler::operand_valid_for_add_sub_immediate(c_hi))
1743             __ adc(dreg_hi, lreg_hi, c_hi);
1744           else {
1745             __ mov(rscratch1, c_hi);
1746             __ adc(dreg_lo, lreg_hi, rscratch1);
1747           }
1748           break;
1749         case lir_sub:
1750           if (Assembler::operand_valid_for_add_sub_immediate(c_lo))
1751             __ subs(dreg_lo, lreg_lo, c_lo);
1752           else {
1753             __ mov(rscratch1, c_lo);
1754             __ subs(dreg_lo, lreg_lo, rscratch1);
1755           }
1756           if (Assembler::operand_valid_for_add_sub_immediate(c_hi))
1757             __ sbc(dreg_hi, lreg_hi, c_hi);
1758           else {
1759             __ mov(rscratch1, c_hi);
1760             __ sbc(dreg_hi, lreg_hi, rscratch1);
1761           }
1762           break;
1763         default:
1764           ShouldNotReachHere();
1765       }
1766     } else {
1767       ShouldNotReachHere();
1768     }
1769   } else if (left->is_single_fpu()) {
1770     assert(right->is_single_fpu(), "right hand side of float arithmetics needs to be float register");
1771     switch (code) {
1772     case lir_add: __ vadd_f32 (dest->as_float_reg(), left->as_float_reg(), right->as_float_reg()); break;
1773     case lir_sub: __ vsub_f32 (dest->as_float_reg(), left->as_float_reg(), right->as_float_reg()); break;
1774     case lir_mul: __ vmul_f32 (dest->as_float_reg(), left->as_float_reg(), right->as_float_reg()); break;
1775     case lir_div: __ vdiv_f32 (dest->as_float_reg(), left->as_float_reg(), right->as_float_reg()); break;
1776     default:
1777       ShouldNotReachHere();
1778     }
1779   } else if (left->is_double_fpu()) {
1780     if (right->is_double_fpu()) {
1781       // cpu register - cpu register
1782       switch (code) {
1783       case lir_add: __ vadd_f64 (dest->as_double_reg(), left->as_double_reg(), right->as_double_reg()); break;
1784       case lir_sub: __ vsub_f64 (dest->as_double_reg(), left->as_double_reg(), right->as_double_reg()); break;
1785       case lir_mul: __ vmul_f64 (dest->as_double_reg(), left->as_double_reg(), right->as_double_reg()); break;
1786       case lir_div: __ vdiv_f64 (dest->as_double_reg(), left->as_double_reg(), right->as_double_reg()); break;
1787       default:
1788         ShouldNotReachHere();
1789       }
1790     } else {
1791       if (right->is_constant()) {
1792         ShouldNotReachHere();
1793       }
1794       ShouldNotReachHere();
1795     }
1796   } else if (left->is_single_stack() || left->is_address()) {
1797     assert(left == dest, "left and dest must be equal");
1798     ShouldNotReachHere();
1799   } else {
1800     ShouldNotReachHere();
1801   }
1802 }
1803 
1804 void LIR_Assembler::intrinsic_op(LIR_Code code, LIR_Opr value, LIR_Opr unused, LIR_Opr dest, LIR_Op* op) {
1805   switch(code) {
1806   case lir_abs : __ vabs_f64(dest->as_double_reg(), value->as_double_reg()); break;
1807   case lir_sqrt: __ vsqrt_f64(dest->as_double_reg(), value->as_double_reg()); break;
1808   default      : ShouldNotReachHere();
1809   }
1810 }
1811 
1812 void LIR_Assembler::logic_op(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dst) {
1813 
1814   assert(left->is_single_cpu() || left->is_double_cpu(), "expect single or double register");
1815   Register Rleft = left->is_single_cpu() ? left->as_register() :
1816                                            left->as_register_lo();
1817    if (dst->is_single_cpu()) {
1818      Register Rdst = dst->as_register();
1819      if (right->is_constant()) {
1820        switch (code) {
1821          case lir_logic_and: __ andr (Rdst, Rleft, right->as_jint()); break;
1822          case lir_logic_or:  __ orr (Rdst, Rleft, right->as_jint()); break;
1823          case lir_logic_xor: __ eor (Rdst, Rleft, right->as_jint()); break;
1824          default: ShouldNotReachHere(); break;
1825        }
1826      } else {
1827        Register Rright = right->is_single_cpu() ? right->as_register() :
1828                                                   right->as_register_lo();
1829        switch (code) {
1830          case lir_logic_and: __ andr (Rdst, Rleft, Rright); break;
1831          case lir_logic_or:  __ orr (Rdst, Rleft, Rright); break;
1832          case lir_logic_xor: __ eor (Rdst, Rleft, Rright); break;
1833          default: ShouldNotReachHere(); break;
1834        }
1835      }
1836    } else {
1837      assert(dst->is_double_cpu(), "mismatched logic op operand size");
1838      const Register Rdst_lo = dst->as_register_lo();
1839      const Register Rdst_hi = dst->as_register_hi();
1840      Register Rleft_hi = left->as_register_hi();
1841      if (right->is_constant()) {
1842        // LIR generator enforces jlong constants to be valid_immediate12
1843        // so we know they fit into 32-bit int
1844        switch (code) {
1845          case lir_logic_and: __ andr (Rdst_lo, Rleft, (int)right->as_jlong()); break;
1846          case lir_logic_or:  __ orr (Rdst_lo, Rleft, (int)right->as_jlong()); break;
1847          case lir_logic_xor: __ eor (Rdst_lo, Rleft, (int)right->as_jlong()); break;
1848          default: ShouldNotReachHere(); break;
1849        }
1850      } else {
1851        assert(right->is_double_cpu(), "mismatched logic op operand size");
1852        Register Rright_lo = right->as_register_lo();
1853        Register Rright_hi = right->as_register_hi();
1854        check_register_collision(Rdst_lo, &Rleft_hi, &Rright_hi);
1855        switch (code) {
1856          case lir_logic_and: __ andr (Rdst_lo, Rleft, Rright_lo);
1857                              __ andr (Rdst_hi, Rleft_hi, Rright_hi); break;
1858          case lir_logic_or:  __ orr (Rdst_lo, Rleft, Rright_lo);
1859                              __ orr (Rdst_hi, Rleft_hi, Rright_hi); break;
1860          case lir_logic_xor: __ eor (Rdst_lo, Rleft, Rright_lo);
1861                              __ eor (Rdst_hi, Rleft_hi, Rright_hi); break;
1862          default: ShouldNotReachHere(); break;
1863        }
1864      }
1865    }
1866 }
1867 
1868 
1869 
1870 void LIR_Assembler::arithmetic_idiv(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr temp, LIR_Opr result, CodeEmitInfo* info) { Unimplemented(); }
1871 
1872 void LIR_Assembler::comp_op(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Op2* op) {
1873   if (opr1->is_single_cpu()) {
1874 
1875     assert(opr1->type() != T_FLOAT, "expect integer type");// softfp guard
1876     assert(opr2->type() != T_FLOAT, "expect integer type");
1877 
1878     Register reg1 = as_reg(opr1);
1879     if (opr2->is_single_cpu()) {
1880       // cpu register - cpu register
1881       Register reg2 = opr2->as_register();
1882       if (opr1->type() == T_OBJECT || opr1->type() == T_ARRAY) {
1883         __ cmpoop(reg1, reg2);
1884       } else {
1885         assert(opr2->type() != T_OBJECT && opr2->type() != T_ARRAY, "cmp int, oop?");
1886         __ cmp(reg1, reg2);
1887       }
1888     } else if (opr2->is_constant()) {
1889       LIR_Const* c = opr2->as_constant_ptr();
1890       if (c->type() == T_INT) {
1891         __ cmp(reg1, c->as_jint(), rscratch1, Assembler::C_DFLT);
1892       } else if (c->type() == T_OBJECT || c->type() == T_ARRAY) {
1893         jobject o = c->as_jobject();
1894         if (o == NULL) {
1895           __ cmp(reg1, (int32_t)NULL_WORD);
1896         } else {
1897           jobject2reg(o, rscratch1);
1898           __ cmpoop(reg1, rscratch1);
1899         }
1900       } else {
1901         fatal("unexpected type: %s", basictype_to_str(c->type()));
1902       }
1903     } else if (opr2->is_address()) {
1904       __ ldr(rscratch2, as_Address(opr2->as_address_ptr(), rscratch1, Address::IDT_INT));
1905       __ cmp(reg1, rscratch2);
1906     } else {
1907       ShouldNotReachHere();
1908     }
1909 
1910   } else if (opr1->is_double_cpu()) {
1911     assert(opr1->type() == T_LONG, "expect jlong type");
1912     assert(opr2->type() == T_LONG, "expect jlong type");
1913     Register xlo = opr1->as_register_lo();
1914     Register xhi = opr1->as_register_hi();
1915     if (opr2->is_double_cpu()) {
1916       // cpu register - cpu register
1917       Register ylo = opr2->as_register_lo();
1918       Register yhi = opr2->as_register_hi();
1919       switch (condition) {
1920         case lir_cond_equal:
1921         case lir_cond_notEqual:
1922         case lir_cond_belowEqual:
1923         case lir_cond_aboveEqual:
1924           // these need APSR.ZC. the ops below set them correctly (but not APSR.V)
1925           __ cmp(xhi, yhi);
1926           __ cmp(xlo, ylo, Assembler::EQ);
1927           break;
1928         case lir_cond_less:
1929         case lir_cond_greaterEqual:
1930           __ cmp(xlo, ylo);
1931           __ sbcs(rscratch1, xhi, yhi);
1932           break;
1933         case lir_cond_lessEqual:
1934         case lir_cond_greater:
1935           // here goes a trick: the below operations do not produce the valid
1936           // value for the APSR.Z flag and there is no easy way to set it. so
1937           // we exchange the order of arguments in the comparison and use the
1938           // opposite condition in the conditional statement that follows.
1939           // GE should be used instead of LE and LT in place of GT.
1940           // the comp_op() could only be followed by: emit_opBranch(), cmove() and
1941           // emit_assert(). these are patched to be aware of this trick
1942           __ cmp(ylo, xlo);
1943           __ sbcs(rscratch1, yhi, xhi);
1944           break;
1945       }
1946     } else if (opr2->is_constant()) {
1947       jlong y = opr2->as_jlong();
1948       assert(Assembler::operand_valid_for_add_sub_immediate(y), "immediate overflow");
1949       switch (condition) {
1950         case lir_cond_equal:
1951         case lir_cond_notEqual:
1952         case lir_cond_belowEqual:
1953         case lir_cond_aboveEqual:
1954           __ cmp(xhi, (int)(y >> 32));
1955           __ cmp(xlo, (int)y, Assembler::EQ);
1956           break;
1957         case lir_cond_less:
1958         case lir_cond_greaterEqual:
1959           __ cmp(xlo, (int)y);
1960           __ sbcs(rscratch1, xhi, (int)(y >> 32));
1961           break;
1962         case lir_cond_lessEqual:
1963         case lir_cond_greater:
1964           __ rsbs(rscratch1, xlo, (int)y);
1965           __ rscs(rscratch1, xhi, (int)(y >> 32));
1966           break;
1967       }
1968     } else {
1969       ShouldNotReachHere();
1970     }
1971   } else if (opr1->is_single_fpu()) {
1972     FloatRegister reg1 = opr1->as_float_reg();
1973     assert(opr2->is_single_fpu(), "expect single float register");
1974     FloatRegister reg2 = opr2->as_float_reg();
1975     __ vcmp_f32(reg1, reg2);
1976     __ get_fpsr();
1977   } else if (opr1->is_double_fpu()) {
1978     FloatRegister reg1 = opr1->as_double_reg();
1979     assert(opr2->is_double_fpu(), "expect double float register");
1980     FloatRegister reg2 = opr2->as_double_reg();
1981     __ vcmp_f64(reg1, reg2);
1982     __ get_fpsr();
1983   } else {
1984     ShouldNotReachHere();
1985   }
1986 }
1987 
1988 void LIR_Assembler::comp_fl2i(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dst, LIR_Op2* op){
1989   if (code == lir_cmp_fd2i || code == lir_ucmp_fd2i) {
1990     bool is_unordered_less = (code == lir_ucmp_fd2i);
1991     if (left->is_single_fpu()) {
1992       __ float_cmp(true, is_unordered_less ? -1 : 1, left->as_float_reg(), right->as_float_reg(), dst->as_register());
1993     } else if (left->is_double_fpu()) {
1994       __ float_cmp(false, is_unordered_less ? -1 : 1, left->as_double_reg(), right->as_double_reg(), dst->as_register());
1995     } else {
1996       ShouldNotReachHere();
1997     }
1998   } else if (code == lir_cmp_l2i) {
1999     __ mov(dst->as_register(), 1);
2000     __ subs(rscratch1, left->as_register_lo(), right->as_register_lo());
2001     __ sbc(rscratch2, left->as_register_hi(), right->as_register_hi());
2002     __ orrs(rscratch1, rscratch1, rscratch2);
2003     __ mov(dst->as_register(), -1, Assembler::MI);
2004     __ mov(dst->as_register(), 0, Assembler::EQ);
2005   } else {
2006     ShouldNotReachHere();
2007   }
2008 }
2009 
2010 
2011 void LIR_Assembler::align_call(LIR_Code code) {  }
2012 
2013 
2014 void LIR_Assembler::call(LIR_OpJavaCall* op, relocInfo::relocType rtype) {
2015   __ trampoline_call(Address(op->addr(), rtype));
2016   add_call_info(code_offset(), op->info());
2017 }
2018 
2019 
2020 void LIR_Assembler::ic_call(LIR_OpJavaCall* op) {
2021   __ ic_call(op->addr());
2022   add_call_info(code_offset(), op->info());
2023 }
2024 
2025 
2026 /* Currently, vtable-dispatch is only enabled for sparc platforms */
2027 void LIR_Assembler::vtable_call(LIR_OpJavaCall* op) {
2028   ShouldNotReachHere();
2029 }
2030 
2031 
2032 void LIR_Assembler::emit_static_call_stub() {
2033   address call_pc = __ pc();
2034   address stub = __ start_a_stub(call_stub_size());
2035   if (stub == NULL) {
2036     bailout("static call stub overflow");
2037     return;
2038   }
2039 
2040   int start = __ offset();
2041 
2042   __ relocate(static_stub_Relocation::spec(call_pc));
2043   __ mov_metadata(rmethod, (Metadata*)NULL);
2044   __ movptr(rscratch1, 0);
2045   __ b(rscratch1);
2046 
2047   assert(__ offset() - start <= call_stub_size(), "stub too big");
2048   __ end_a_stub();
2049 }
2050 
2051 
2052 void LIR_Assembler::throw_op(LIR_Opr exceptionPC, LIR_Opr exceptionOop, CodeEmitInfo* info) {
2053   assert(exceptionOop->as_register() == r0, "must match");
2054   assert(exceptionPC->as_register() == r3, "must match");
2055 
2056   // exception object is not added to oop map by LinearScan
2057   // (LinearScan assumes that no oops are in fixed registers)
2058   info->add_register_oop(exceptionOop);
2059   Runtime1::StubID unwind_id;
2060 
2061   // get current pc information
2062   // pc is only needed if the method has an exception handler, the unwind code does not need it.
2063   int pc_for_athrow_offset = __ offset();
2064   __ add(exceptionPC->as_register(), r15_pc, -8);
2065   add_call_info(pc_for_athrow_offset, info); // for exception handler
2066 
2067   __ verify_not_null_oop(r0);
2068   // search an exception handler (r0: exception oop, r3: throwing pc)
2069   if (compilation()->has_fpu_code()) {
2070     unwind_id = Runtime1::handle_exception_id;
2071   } else {
2072     unwind_id = Runtime1::handle_exception_nofpu_id;
2073   }
2074   __ far_call(RuntimeAddress(Runtime1::entry_for(unwind_id)));
2075 
2076   // FIXME: enough room for two byte trap   ????
2077   __ nop();
2078 }
2079 
2080 
2081 void LIR_Assembler::unwind_op(LIR_Opr exceptionOop) {
2082   assert(exceptionOop->as_register() == r0, "must match");
2083 
2084   __ b(_unwind_handler_entry);
2085 }
2086 
2087 
2088 void LIR_Assembler::shift_op(LIR_Code code, LIR_Opr left, LIR_Opr count, LIR_Opr dest, LIR_Opr tmp) {
2089   Register lreg = left->is_single_cpu() ? left->as_register() : left->as_register_lo();
2090   Register dreg = dest->is_single_cpu() ? dest->as_register() : dest->as_register_lo();
2091 
2092   switch (left->type()) {
2093     case T_INT:
2094     case T_ADDRESS:
2095     case T_OBJECT:
2096       __ andr(rscratch1, count->as_register(), 0x1f);
2097       switch (code) {
2098         case lir_shl: __ lsl(dreg, lreg, rscratch1); break;
2099         case lir_shr: __ asr(dreg, lreg, rscratch1); break;
2100         case lir_ushr: __ lsr(dreg, lreg, rscratch1); break;
2101         default:
2102           ShouldNotReachHere();
2103           break;
2104       }
2105       break;
2106     case T_LONG:
2107     {
2108       Register lreg_hi = left->as_register_hi();
2109       Register dreg_hi = dest->as_register_hi();
2110       const int word_bits = 8 * wordSize;
2111 
2112       if (code == lir_shl || code == lir_ushr) {
2113         check_register_collision(dreg, &lreg, &lreg_hi, rscratch1);
2114         check_register_collision(dreg_hi, &lreg, &lreg_hi, rscratch2);
2115       }
2116 
2117       switch (code) {
2118         case lir_shl:
2119           __ andr(dreg, count->as_register(), 0x3f);
2120           __ sub(dreg_hi, dreg, word_bits);
2121           __ lsl(lreg_hi, lreg_hi, dreg);
2122           __ orr(lreg_hi, lreg_hi, lreg, lsl(dreg_hi));
2123           __ rsb(dreg_hi, dreg, word_bits);
2124           __ orr(dreg_hi, lreg_hi, lreg, lsr(dreg_hi));
2125           __ lsl(dreg, lreg, dreg);
2126           break;
2127         case lir_shr: {
2128           __ mov(rscratch2, lreg_hi);
2129           __ andr(rscratch1, count->as_register(), 0x3f);
2130           __ lsr(dreg, lreg, rscratch1);
2131           __ rsb(dreg_hi, rscratch1, word_bits);
2132           __ orr(dreg, dreg, rscratch2, lsl(dreg_hi));
2133           __ asr(dreg_hi, rscratch2, rscratch1);
2134           __ subs(rscratch1, rscratch1, word_bits);
2135           __ mov(dreg, rscratch2, asr(rscratch1), Assembler::GT);
2136         }
2137           break;
2138         case lir_ushr:
2139           __ andr(dreg, count->as_register(), 0x3f);
2140           __ lsr(lreg, lreg, dreg);
2141           __ rsb(dreg_hi, dreg, word_bits);
2142           __ orr(lreg, lreg, lreg_hi, lsl(dreg_hi));
2143           __ lsr(dreg_hi, lreg_hi, dreg);
2144           __ sub(dreg, dreg, word_bits);
2145           __ orr(dreg, lreg, lreg_hi, lsr(dreg));
2146           break;
2147         default:
2148           ShouldNotReachHere();
2149           break;
2150       }
2151     }
2152       break;
2153     default:
2154       ShouldNotReachHere();
2155       break;
2156   }
2157 }
2158 
2159 
2160 void LIR_Assembler::shift_op(LIR_Code code, LIR_Opr left, jint count, LIR_Opr dest) {
2161   Register dreg = dest->is_single_cpu() ? dest->as_register() : dest->as_register_lo();
2162   Register lreg = left->is_single_cpu() ? left->as_register() : left->as_register_lo();
2163 
2164   if (!count) {
2165       reg2reg(left, dest);
2166       return;
2167    }
2168 
2169   switch (left->type()) {
2170     case T_INT:
2171     case T_ADDRESS:
2172     case T_OBJECT:
2173       switch (code) {
2174         case lir_shl: __ lsl(dreg, lreg, count); break;
2175         case lir_shr: __ asr(dreg, lreg, count); break;
2176         case lir_ushr: __ lsr(dreg, lreg, count); break;
2177         default:
2178           ShouldNotReachHere();
2179           break;
2180       }
2181       break;
2182     case T_LONG: {
2183       Register lreg_hi = left->as_register_hi();
2184       Register dreg_hi = dest->as_register_hi();
2185       const int word_bits = 8 * wordSize;
2186 
2187       switch (code) {
2188         case lir_shl:
2189           if (count >= word_bits) {
2190             __ lsl(dreg_hi, lreg, count - word_bits);
2191             __ mov(dreg, 0);
2192           } else {
2193             check_register_collision(dreg_hi, &lreg);
2194             __ lsl(dreg_hi, lreg_hi, count);
2195             __ orr(dreg_hi, dreg_hi, lreg, lsr(word_bits - count));
2196             __ lsl(dreg, lreg, count);
2197           }
2198           break;
2199         case lir_shr:
2200           if (count >= word_bits) {
2201             __ asr(dreg, lreg_hi, count - word_bits);
2202             __ asr(dreg_hi, lreg_hi, word_bits);
2203           } else {
2204             check_register_collision(dreg, &lreg_hi);
2205             __ lsr(dreg, lreg, count);
2206             __ orr(dreg, dreg, lreg_hi, lsl(word_bits - count));
2207             __ asr(dreg_hi, lreg_hi, count);
2208           }
2209           break;
2210         case lir_ushr:
2211           if (count >= word_bits) {
2212             __ lsr(dreg, lreg_hi, count - word_bits);
2213             __ mov(dreg_hi, 0);
2214           } else {
2215             check_register_collision(dreg, &lreg_hi);
2216             __ lsr(dreg, lreg, count);
2217             __ orr(dreg, dreg, lreg_hi, lsl(word_bits - count));
2218             __ lsr(dreg_hi, lreg_hi, count);
2219           }
2220           break;
2221         default:
2222           ShouldNotReachHere();
2223           break;
2224       }
2225     }
2226       break;
2227     default:
2228       ShouldNotReachHere();
2229       break;
2230   }
2231 }
2232 
2233 
2234 void LIR_Assembler::store_parameter(Register r, int offset_from_sp_in_words) {
2235   assert(offset_from_sp_in_words >= 0, "invalid offset from sp");
2236   int offset_from_sp_in_bytes = offset_from_sp_in_words * BytesPerWord;
2237   assert(offset_from_sp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2238   __ str (r, Address(sp, offset_from_sp_in_bytes));
2239 }
2240 
2241 
2242 void LIR_Assembler::store_parameter(jint c,     int offset_from_sp_in_words) {
2243   assert(offset_from_sp_in_words >= 0, "invalid offset from sp");
2244   int offset_from_sp_in_bytes = offset_from_sp_in_words * BytesPerWord;
2245   assert(offset_from_sp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2246   __ mov (rscratch1, c);
2247   __ str (rscratch1, Address(sp, offset_from_sp_in_bytes));
2248 }
2249 
2250 // This code replaces a call to arraycopy; no exception may
2251 // be thrown in this code, they must be thrown in the System.arraycopy
2252 // activation frame; we could save some checks if this would not be the case
2253 void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) {
2254   ciArrayKlass* default_type = op->expected_type();
2255   Register src = op->src()->as_register();
2256   Register dst = op->dst()->as_register();
2257   Register src_pos = op->src_pos()->as_register();
2258   Register dst_pos = op->dst_pos()->as_register();
2259   Register length  = op->length()->as_register();
2260   Register tmp = op->tmp()->as_register();
2261   // due to limited number of registers available and in order to simplify
2262   // the code we fix the registers used by the arguments to this intrinsic.
2263   // see the comment in LIRGenerator::do_ArrayCopy
2264   assert(src == j_rarg0, "assumed by implementation");
2265   assert(src_pos == j_rarg1, "assumed by implementation");
2266   assert(dst == j_rarg2, "assumed by implementation");
2267   assert(dst_pos == j_rarg3, "assumed by implementation");
2268   assert(length == r4, "assumed by implementation");
2269   assert(tmp == r5, "assumed by implementation");
2270   const int dst_spill_offset = 2*BytesPerWord;
2271   const int dst_pos_spill_offset = 1*BytesPerWord;
2272   const int length_spill_offset = 0*BytesPerWord;
2273   const int src_pos_spill_offset = 3*BytesPerWord;
2274   const int src_spill_offset = 4*BytesPerWord;
2275 
2276   CodeStub* stub = op->stub();
2277   int flags = op->flags();
2278   BasicType basic_type = default_type != NULL ? default_type->element_type()->basic_type() : T_ILLEGAL;
2279   if (basic_type == T_ARRAY) basic_type = T_OBJECT;
2280 
2281   // if we don't know anything, just go through the generic arraycopy
2282   if (default_type == NULL // || basic_type == T_OBJECT
2283       ) {
2284     Label done;
2285     assert(src == r0 && src_pos == r1, "mismatch in calling convention");
2286 
2287     // Save the arguments in case the generic arraycopy fails and we
2288     // have to fall back to the JNI stub. spill all but length since it's
2289     // in the callee save register
2290     __ str(dst,     Address(sp, dst_spill_offset));
2291     __ str(dst_pos, Address(sp, dst_pos_spill_offset));
2292     __ str(src_pos, Address(sp, src_pos_spill_offset));
2293     __ str(src,     Address(sp, src_spill_offset));
2294 
2295     address copyfunc_addr = StubRoutines::generic_arraycopy();
2296     assert(copyfunc_addr != NULL, "generic arraycopy stub required");
2297 
2298     // The arguments are in java calling convention so we shift them
2299     // to C convention
2300     assert(c_rarg0 == j_rarg0, "assumed in the code below");
2301     // the below C function follows C calling convention,
2302     // so should put 5th arg to stack
2303     assert(length_spill_offset == 0, "assumed in the code below");
2304     __ str(length,  Address(sp));
2305 
2306 #ifndef PRODUCT
2307     if (PrintC1Statistics) {
2308       __ increment(ExternalAddress((address)&Runtime1::_generic_arraycopystub_cnt));
2309     }
2310 #endif
2311     __ far_call(RuntimeAddress(copyfunc_addr));
2312 
2313     __ cbz(r0, *stub->continuation());
2314 
2315     // r0 is -1^K where K == partial copied count
2316     __ inv(rscratch1, r0);
2317 
2318     // Reload values from the stack so they are where the stub
2319     // expects them. don't reload length since it's in the callee-save register
2320     // and the value on the stack might have been modified by the C function
2321     __ ldr(dst,     Address(sp, dst_spill_offset));
2322     __ ldr(dst_pos, Address(sp, dst_pos_spill_offset));
2323     __ ldr(src_pos, Address(sp, src_pos_spill_offset));
2324     __ ldr(src,     Address(sp, src_spill_offset));
2325 
2326     // adjust length down and src/end pos up by partial copied count
2327     __ sub(length, length, rscratch1);
2328     __ add(src_pos, src_pos, rscratch1);
2329     __ add(dst_pos, dst_pos, rscratch1);
2330 
2331     __ b(*stub->entry());
2332 
2333     __ bind(*stub->continuation());
2334     return;
2335   }
2336 
2337   assert(default_type != NULL && default_type->is_array_klass() && default_type->is_loaded(), "must be true at this point");
2338 
2339   int elem_size = type2aelembytes(basic_type);
2340   int scale = exact_log2(elem_size);
2341 
2342   Address src_length_addr = Address(src, arrayOopDesc::length_offset_in_bytes());
2343   Address dst_length_addr = Address(dst, arrayOopDesc::length_offset_in_bytes());
2344   Address src_klass_addr = Address(src, oopDesc::klass_offset_in_bytes());
2345   Address dst_klass_addr = Address(dst, oopDesc::klass_offset_in_bytes());
2346 
2347   // test for NULL
2348   if (flags & LIR_OpArrayCopy::src_null_check) {
2349     __ cbz(src, *stub->entry());
2350   }
2351   if (flags & LIR_OpArrayCopy::dst_null_check) {
2352     __ cbz(dst, *stub->entry());
2353   }
2354 
2355   // If the compiler was not able to prove that exact type of the source or the destination
2356   // of the arraycopy is an array type, check at runtime if the source or the destination is
2357   // an instance type.
2358   if (flags & LIR_OpArrayCopy::type_check) {
2359     if (!(flags & LIR_OpArrayCopy::LIR_OpArrayCopy::dst_objarray)) {
2360       __ load_klass(tmp, dst);
2361       __ ldr(rscratch1, Address(tmp, in_bytes(Klass::layout_helper_offset())));
2362       __ cmp(rscratch1, Klass::_lh_neutral_value);
2363       __ b(*stub->entry(), Assembler::GE);
2364     }
2365 
2366     if (!(flags & LIR_OpArrayCopy::LIR_OpArrayCopy::src_objarray)) {
2367       __ load_klass(tmp, src);
2368       __ ldr(rscratch1, Address(tmp, in_bytes(Klass::layout_helper_offset())));
2369       __ cmp(rscratch1, Klass::_lh_neutral_value);
2370       __ b(*stub->entry(), Assembler::GE);
2371     }
2372   }
2373 
2374 
2375   // check if negative
2376   if (flags & LIR_OpArrayCopy::src_pos_positive_check) {
2377     __ cmp(src_pos, 0);
2378     __ b(*stub->entry(), Assembler::LT);
2379   }
2380   if (flags & LIR_OpArrayCopy::dst_pos_positive_check) {
2381     __ cmp(dst_pos, 0);
2382     __ b(*stub->entry(), Assembler::LT);
2383   }
2384 
2385   if (flags & LIR_OpArrayCopy::length_positive_check) {
2386     __ cmp(length, 0);
2387     __ b(*stub->entry(), Assembler::LT);
2388   }
2389 
2390   if (flags & LIR_OpArrayCopy::src_range_check) {
2391     __ add(tmp, src_pos, length);
2392     __ ldr(rscratch1, src_length_addr);
2393     __ cmp(tmp, rscratch1);
2394     __ b(*stub->entry(), Assembler::HI);
2395   }
2396   if (flags & LIR_OpArrayCopy::dst_range_check) {
2397     __ add(tmp, dst_pos, length);
2398     __ ldr(rscratch1, dst_length_addr);
2399     __ cmp(tmp, rscratch1);
2400     __ b(*stub->entry(), Assembler::HI);
2401   }
2402 
2403   // FIXME: The logic in LIRGenerator::arraycopy_helper clears
2404   // length_positive_check if the source of our length operand is an
2405   // arraylength.  However, that arraylength might be zero, and the
2406   // stub that we're about to call contains an assertion that count !=
2407   // 0 .  So we make this check purely in order not to trigger an
2408   // assertion failure.
2409   __ cbz(length, *stub->continuation());
2410 
2411   if (flags & LIR_OpArrayCopy::type_check) {
2412     // We don't know the array types are compatible
2413     if (basic_type != T_OBJECT) {
2414       // Simple test for basic type arrays
2415       __ ldr(tmp, src_klass_addr);
2416       __ ldr(rscratch1, dst_klass_addr);
2417       __ cmp(tmp, rscratch1);
2418       __ b(*stub->entry(), Assembler::NE);
2419     } else {
2420       // For object arrays, if src is a sub class of dst then we can
2421       // safely do the copy.
2422       Label cont, slow;
2423 
2424       __ push(RegSet::of(src, dst), sp);
2425 
2426       __ load_klass(src, src);
2427       __ load_klass(dst, dst);
2428 
2429       __ check_klass_subtype_fast_path(src, dst, tmp, &cont, &slow, NULL);
2430 
2431       __ push(src); // sub
2432       __ push(dst); // super
2433       __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
2434       // result on TOS
2435       __ pop(src); // result
2436       __ pop(dst);
2437 
2438       __ cbnz(src, cont);
2439 
2440       __ bind(slow);
2441       __ pop(RegSet::of(src, dst), sp);
2442 
2443       address copyfunc_addr = StubRoutines::checkcast_arraycopy();
2444       if (copyfunc_addr != NULL) { // use stub if available
2445         // src is not a sub class of dst so we have to do a
2446         // per-element check.
2447 
2448         int mask = LIR_OpArrayCopy::src_objarray|LIR_OpArrayCopy::dst_objarray;
2449         if ((flags & mask) != mask) {
2450           // Check that at least both of them object arrays.
2451           assert(flags & mask, "one of the two should be known to be an object array");
2452 
2453           if (!(flags & LIR_OpArrayCopy::src_objarray)) {
2454             __ load_klass(tmp, src);
2455           } else if (!(flags & LIR_OpArrayCopy::dst_objarray)) {
2456             __ load_klass(tmp, dst);
2457           }
2458           int lh_offset = in_bytes(Klass::layout_helper_offset());
2459           Address klass_lh_addr(tmp, lh_offset);
2460           jint objArray_lh = Klass::array_layout_helper(T_OBJECT);
2461           __ ldr(rscratch1, klass_lh_addr);
2462           __ mov(rscratch2, objArray_lh);
2463           __ eor(rscratch1, rscratch1, rscratch2);
2464           __ cbnz(rscratch1, *stub->entry());
2465         }
2466 
2467         // Spill because stub destroys r0-r3.
2468         assert(length_spill_offset == 0, "assumed in the code below");
2469         assert(length == r4, "shall not be in r0-r3");
2470         __ str(dst_pos, Address(sp, dst_pos_spill_offset));
2471         __ str(dst,     Address(sp, dst_spill_offset));
2472         __ str(src_pos, Address(sp, src_pos_spill_offset));
2473         __ str(src,     Address(sp, src_spill_offset));
2474 
2475         __ lea(c_rarg0, Address(src, src_pos, lsl(scale)));
2476         __ add(c_rarg0, c_rarg0, arrayOopDesc::base_offset_in_bytes(basic_type));
2477         assert_different_registers(c_rarg0, dst, dst_pos, length);
2478         __ lea(c_rarg1, Address(dst, dst_pos, lsl(scale)));
2479         __ add(c_rarg1, c_rarg1, arrayOopDesc::base_offset_in_bytes(basic_type));
2480         assert_different_registers(c_rarg1, dst, dst_pos, length);
2481 
2482         __ load_klass(c_rarg2, dst);
2483         __ ldr(c_rarg2, Address(c_rarg2, ObjArrayKlass::element_klass_offset()));
2484         __ ldr(c_rarg3, Address(c_rarg2, Klass::super_check_offset_offset()));
2485         __ str(c_rarg2, Address(sp));
2486         __ mov(c_rarg2, length);
2487         __ far_call(RuntimeAddress(copyfunc_addr));
2488 
2489 #ifndef PRODUCT
2490         if (PrintC1Statistics) {
2491           Label failed;
2492           __ cbnz(r0, failed);
2493           __ increment(ExternalAddress((address)&Runtime1::_arraycopy_checkcast_cnt));
2494           __ bind(failed);
2495         }
2496 #endif
2497 
2498         __ cbz(r0, *stub->continuation());
2499 
2500 #ifndef PRODUCT
2501         if (PrintC1Statistics) {
2502           __ increment(ExternalAddress((address)&Runtime1::_arraycopy_checkcast_attempt_cnt));
2503         }
2504 #endif
2505         assert_different_registers(dst, dst_pos, length, src_pos, src, rscratch1);
2506         // return value is -1^K where K is partial copied count
2507         __ mvn(rscratch1, r0);
2508 
2509         // Restore previously spilled arguments
2510         __ ldr(dst_pos, Address(sp, dst_pos_spill_offset));
2511         __ ldr(dst,     Address(sp, dst_spill_offset));
2512         __ ldr(src_pos, Address(sp, src_pos_spill_offset));
2513         __ ldr(src,     Address(sp, src_spill_offset));
2514 
2515         // adjust length down and src/end pos up by partial copied count
2516         __ sub(length, length, rscratch1);
2517         __ add(src_pos, src_pos, rscratch1);
2518         __ add(dst_pos, dst_pos, rscratch1);
2519       }
2520 
2521       __ b(*stub->entry());
2522 
2523       __ bind(cont);
2524       __ pop(RegSet::of(src, dst), sp);
2525     }
2526   }
2527 
2528 #ifdef ASSERT
2529   if (basic_type != T_OBJECT || !(flags & LIR_OpArrayCopy::type_check)) {
2530     // Sanity check the known type with the incoming class.  For the
2531     // primitive case the types must match exactly with src.klass and
2532     // dst.klass each exactly matching the default type.  For the
2533     // object array case, if no type check is needed then either the
2534     // dst type is exactly the expected type and the src type is a
2535     // subtype which we can't check or src is the same array as dst
2536     // but not necessarily exactly of type default_type.
2537     Label known_ok, halt;
2538     __ mov_metadata(tmp, default_type->constant_encoding());
2539 
2540     if (basic_type != T_OBJECT) {
2541 
2542       __ ldr(rscratch1, dst_klass_addr);
2543       __ cmp(tmp, rscratch1);
2544       __ b(halt, Assembler::NE);
2545       __ ldr(rscratch1, src_klass_addr);
2546       __ cmp(tmp, rscratch1);
2547       __ b(known_ok, Assembler::EQ);
2548     } else {
2549       __ ldr(rscratch1, dst_klass_addr);
2550       __ cmp(tmp, rscratch1);
2551       __ b(known_ok, Assembler::EQ);
2552       __ cmp(src, dst);
2553       __ b(known_ok, Assembler::EQ);
2554     }
2555     __ bind(halt);
2556     __ stop("incorrect type information in arraycopy");
2557     __ bind(known_ok);
2558   }
2559 #endif
2560 
2561   __ lea(c_rarg0, Address(src, src_pos, lsl(scale)));
2562   __ add(c_rarg0, c_rarg0, arrayOopDesc::base_offset_in_bytes(basic_type));
2563   assert_different_registers(c_rarg0, dst, dst_pos, length);
2564   __ lea(c_rarg1, Address(dst, dst_pos, lsl(scale)));
2565   __ add(c_rarg1, c_rarg1, arrayOopDesc::base_offset_in_bytes(basic_type));
2566   assert_different_registers(c_rarg1, dst, dst_pos, length);
2567   __ mov(c_rarg2, length);
2568 
2569   bool disjoint = (flags & LIR_OpArrayCopy::overlapping) == 0;
2570   bool aligned = (flags & LIR_OpArrayCopy::unaligned) == 0;
2571   const char *name;
2572   address entry = StubRoutines::select_arraycopy_function(basic_type, aligned, disjoint, name, false);
2573 
2574   CodeBlob *cb = CodeCache::find_blob(entry);
2575   if (cb) {
2576     __ far_call(RuntimeAddress(entry));
2577   } else {
2578     __ call_VM_leaf(entry, 3);
2579   }
2580 
2581   __ bind(*stub->continuation());
2582 }
2583 
2584 void LIR_Assembler::emit_lock(LIR_OpLock* op) {
2585   Register obj = op->obj_opr()->as_register();  // may not be an oop
2586   Register hdr = op->hdr_opr()->as_register();
2587   Register lock = op->lock_opr()->as_register();
2588   if (!UseFastLocking) {
2589     __ b(*op->stub()->entry());
2590   } else if (op->code() == lir_lock) {
2591     Register scratch = noreg;
2592     if (UseBiasedLocking) {
2593       scratch = op->scratch_opr()->as_register();
2594     }
2595     assert(BasicLock::displaced_header_offset_in_bytes() == 0, "lock_reg must point to the displaced header");
2596     // add debug info for NullPointerException only if one is possible
2597     int null_check_offset = __ lock_object(hdr, obj, lock, scratch, *op->stub()->entry());
2598     if (op->info() != NULL) {
2599       add_debug_info_for_null_check(null_check_offset, op->info());
2600     }
2601     // done
2602   } else if (op->code() == lir_unlock) {
2603     assert(BasicLock::displaced_header_offset_in_bytes() == 0, "lock_reg must point to the displaced header");
2604     __ unlock_object(hdr, obj, lock, *op->stub()->entry());
2605   } else {
2606     Unimplemented();
2607   }
2608   __ bind(*op->stub()->continuation());
2609 }
2610 
2611 
2612 void LIR_Assembler::emit_profile_call(LIR_OpProfileCall* op) {
2613   ciMethod* method = op->profiled_method();
2614   int bci          = op->profiled_bci();
2615   ciMethod* callee = op->profiled_callee();
2616 
2617   // Update counter for all call types
2618   ciMethodData* md = method->method_data_or_null();
2619   assert(md != NULL, "Sanity");
2620   ciProfileData* data = md->bci_to_data(bci);
2621   assert(data != NULL && data->is_CounterData(), "need CounterData for calls");
2622   assert(op->mdo()->is_single_cpu(),  "mdo must be allocated");
2623   Register mdo  = op->mdo()->as_register();
2624   __ mov_metadata(mdo, md->constant_encoding());
2625   Address counter_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
2626   // Perform additional virtual call profiling for invokevirtual and
2627   // invokeinterface bytecodes
2628   if (op->should_profile_receiver_type()) {
2629     assert(op->recv()->is_single_cpu(), "recv must be allocated");
2630     Register recv = op->recv()->as_register();
2631     assert_different_registers(mdo, recv);
2632     assert(data->is_VirtualCallData(), "need VirtualCallData for virtual calls");
2633     ciKlass* known_klass = op->known_holder();
2634     if (C1OptimizeVirtualCallProfiling && known_klass != NULL) {
2635       // We know the type that will be seen at this call site; we can
2636       // statically update the MethodData* rather than needing to do
2637       // dynamic tests on the receiver type
2638 
2639       // NOTE: we should probably put a lock around this search to
2640       // avoid collisions by concurrent compilations
2641       ciVirtualCallData* vc_data = (ciVirtualCallData*) data;
2642       uint i;
2643       for (i = 0; i < VirtualCallData::row_limit(); i++) {
2644         ciKlass* receiver = vc_data->receiver(i);
2645         if (known_klass->equals(receiver)) {
2646           Address data_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_count_offset(i)));
2647           __ addptr(data_addr, DataLayout::counter_increment);
2648           return;
2649         }
2650       }
2651 
2652       // Receiver type not found in profile data; select an empty slot
2653 
2654       // Note that this is less efficient than it should be because it
2655       // always does a write to the receiver part of the
2656       // VirtualCallData rather than just the first time
2657       for (i = 0; i < VirtualCallData::row_limit(); i++) {
2658         ciKlass* receiver = vc_data->receiver(i);
2659         if (receiver == NULL) {
2660           Address recv_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_offset(i)));
2661           __ mov_metadata(rscratch1, known_klass->constant_encoding());
2662           __ lea(rscratch2, recv_addr);
2663           __ str(rscratch1, Address(rscratch2));
2664           Address data_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_count_offset(i)));
2665           __ addptr(data_addr, DataLayout::counter_increment);
2666           return;
2667         }
2668       }
2669     } else {
2670       __ load_klass(recv, recv);
2671       Label update_done;
2672       type_profile_helper(mdo, md, data, recv, &update_done);
2673       // Receiver did not match any saved receiver and there is no empty row for it.
2674       // Increment total counter to indicate polymorphic case.
2675       __ addptr(counter_addr, DataLayout::counter_increment);
2676 
2677       __ bind(update_done);
2678     }
2679   } else {
2680     // Static call
2681     __ addptr(counter_addr, DataLayout::counter_increment);
2682   }
2683 }
2684 
2685 
2686 void LIR_Assembler::emit_delay(LIR_OpDelay*) {
2687   Unimplemented();
2688 }
2689 
2690 
2691 void LIR_Assembler::monitor_address(int monitor_no, LIR_Opr dst) {
2692   __ lea(dst->as_register(), frame_map()->address_for_monitor_lock(monitor_no));
2693 }
2694 
2695 void LIR_Assembler::emit_updatecrc32(LIR_OpUpdateCRC32* op) {
2696   assert(op->crc()->is_single_cpu(),  "crc must be register");
2697   assert(op->val()->is_single_cpu(),  "byte value must be register");
2698   assert(op->result_opr()->is_single_cpu(), "result must be register");
2699   Register crc = op->crc()->as_register();
2700   Register val = op->val()->as_register();
2701   Register res = op->result_opr()->as_register();
2702 
2703   assert_different_registers(val, crc, res);
2704   __ lea(res, ExternalAddress(StubRoutines::crc_table_addr()));
2705 
2706   __ inv(crc, crc);
2707   __ update_byte_crc32(crc, val, res);
2708   __ inv(res, crc);
2709 }
2710 
2711 void LIR_Assembler::emit_profile_type(LIR_OpProfileType* op) {
2712   COMMENT("emit_profile_type {");
2713   Register obj = op->obj()->as_register();
2714   Register tmp = op->tmp()->as_pointer_register();
2715   Address mdo_addr = as_Address(op->mdp()->as_address_ptr(), noreg, Address::IDT_INT);
2716   ciKlass* exact_klass = op->exact_klass();
2717   intptr_t current_klass = op->current_klass();
2718   bool not_null = op->not_null();
2719   bool no_conflict = op->no_conflict();
2720 
2721   Label update, next, none;
2722 
2723   bool do_null = !not_null;
2724   bool exact_klass_set = exact_klass != NULL && ciTypeEntries::valid_ciklass(current_klass) == exact_klass;
2725   bool do_update = !TypeEntries::is_type_unknown(current_klass) && !exact_klass_set;
2726 
2727   assert(do_null || do_update, "why are we here?");
2728   assert(!TypeEntries::was_null_seen(current_klass) || do_update, "why are we here?");
2729   assert(mdo_addr.base() != rscratch1, "wrong register");
2730 
2731   __ verify_oop(obj);
2732 
2733   if (tmp != obj) {
2734     __ mov(tmp, obj);
2735   }
2736   if (do_null) {
2737     __ cbnz(tmp, update);
2738     if (!TypeEntries::was_null_seen(current_klass)) {
2739       __ ldr(rscratch2, mdo_addr);
2740       __ orr(rscratch2, rscratch2, TypeEntries::null_seen);
2741       __ str(rscratch2, mdo_addr);
2742     }
2743     if (do_update) {
2744 #ifndef ASSERT
2745       __ b(next);
2746     }
2747 #else
2748       __ b(next);
2749     }
2750   } else {
2751     __ cbnz(tmp, update);
2752     __ stop("unexpected null obj");
2753 #endif
2754   }
2755 
2756   __ bind(update);
2757 
2758   if (do_update) {
2759 #ifdef ASSERT
2760     if (exact_klass != NULL) {
2761       Label ok;
2762       __ load_klass(tmp, tmp);
2763       __ mov_metadata(rscratch1, exact_klass->constant_encoding());
2764       __ eor(rscratch1, tmp, rscratch1);
2765       __ cbz(rscratch1, ok);
2766       __ stop("exact klass and actual klass differ");
2767       __ bind(ok);
2768     }
2769 #endif
2770     if (!no_conflict) {
2771       if (exact_klass == NULL || TypeEntries::is_type_none(current_klass)) {
2772         if (exact_klass != NULL) {
2773           __ mov_metadata(tmp, exact_klass->constant_encoding());
2774         } else {
2775           __ load_klass(tmp, tmp);
2776         }
2777 
2778         __ ldr(rscratch2, mdo_addr);
2779         __ eor(tmp, tmp, rscratch2);
2780         __ bics(rscratch1, tmp, ~TypeEntries::type_klass_mask);
2781         // klass seen before, nothing to do. The unknown bit may have been
2782         // set already but no need to check.
2783         __ b(next, Assembler::EQ);
2784 
2785         __ ands(rscratch1, tmp, TypeEntries::type_unknown);
2786         __ b(next, Assembler::NE); // already unknown. Nothing to do anymore.
2787 
2788         if (TypeEntries::is_type_none(current_klass)) {
2789           __ cbz(rscratch2, none);
2790           __ cmp(rscratch2, TypeEntries::null_seen);
2791           __ b(none, Assembler::EQ);
2792           // There is a chance that the checks above (re-reading profiling
2793           // data from memory) fail if another thread has just set the
2794           // profiling to this obj's klass
2795           __ dmb(Assembler::ISH);
2796           __ ldr(rscratch2, mdo_addr);
2797           __ eor(tmp, tmp, rscratch2);
2798           __ bics(rscratch1, tmp, ~TypeEntries::type_klass_mask);
2799           __ b(next, Assembler::EQ);
2800         }
2801       } else {
2802         assert(ciTypeEntries::valid_ciklass(current_klass) != NULL &&
2803                ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "conflict only");
2804 
2805         __ ldr(tmp, mdo_addr);
2806         __ ands(rscratch1, tmp, TypeEntries::type_unknown);
2807         __ b(next, Assembler::NE); // already unknown. Nothing to do anymore.
2808       }
2809 
2810       // different than before. Cannot keep accurate profile.
2811       __ ldr(rscratch2, mdo_addr);
2812       __ orr(rscratch2, rscratch2, TypeEntries::type_unknown);
2813       __ str(rscratch2, mdo_addr);
2814 
2815       if (TypeEntries::is_type_none(current_klass)) {
2816         __ b(next);
2817 
2818         __ bind(none);
2819         // first time here. Set profile type.
2820         __ str(tmp, mdo_addr);
2821       }
2822     } else {
2823       // There's a single possible klass at this profile point
2824       assert(exact_klass != NULL, "should be");
2825       if (TypeEntries::is_type_none(current_klass)) {
2826         __ mov_metadata(tmp, exact_klass->constant_encoding());
2827         __ ldr(rscratch2, mdo_addr);
2828         __ eor(tmp, tmp, rscratch2);
2829         __ bics(rscratch1, tmp, ~TypeEntries::type_klass_mask);
2830         __ b(next, Assembler::EQ);
2831 #ifdef ASSERT
2832         {
2833           Label ok;
2834           __ ldr(rscratch1, mdo_addr);
2835           __ cbz(rscratch1, ok);
2836           __ cmp(rscratch1, TypeEntries::null_seen);
2837           __ b(ok, Assembler::EQ);
2838           // may have been set by another thread
2839           __ dmb(Assembler::ISH);
2840           __ mov_metadata(rscratch1, exact_klass->constant_encoding());
2841           __ ldr(rscratch2, mdo_addr);
2842           __ eor(rscratch2, rscratch1, rscratch2);
2843           __ bics(rscratch2, rscratch2, ~TypeEntries::type_mask);
2844           __ b(ok, Assembler::EQ);
2845 
2846           __ stop("unexpected profiling mismatch");
2847           __ bind(ok);
2848         }
2849 #endif
2850         // first time here. Set profile type.
2851         __ ldr(tmp, mdo_addr);
2852       } else {
2853         assert(ciTypeEntries::valid_ciklass(current_klass) != NULL &&
2854                ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "inconsistent");
2855 
2856         __ ldr(tmp, mdo_addr);
2857         __ ands(rscratch1, tmp, TypeEntries::type_unknown);
2858         __ b(next, Assembler::NE); // already unknown. Nothing to do anymore.
2859 
2860         __ orr(tmp, tmp, TypeEntries::type_unknown);
2861         __ str(tmp, mdo_addr);
2862         // FIXME: Write barrier needed here?
2863       }
2864     }
2865 
2866     __ bind(next);
2867   }
2868   COMMENT("} emit_profile_type");
2869 }
2870 
2871 
2872 void LIR_Assembler::align_backward_branch_target() {
2873 }
2874 
2875 
2876 void LIR_Assembler::negate(LIR_Opr left, LIR_Opr dest) {
2877   if (left->is_single_cpu()) {
2878     assert(left->type() != T_FLOAT, "expect integer type");
2879     assert(dest->type() != T_FLOAT, "expect integer type");
2880     assert(dest->is_single_cpu(), "expect single result reg");
2881     __ neg(dest->as_register(), left->as_register());
2882   } else if (left->is_double_cpu()) {
2883     assert(left->type() != T_DOUBLE, "expect integer type");
2884     assert(dest->type() != T_DOUBLE, "expect integer type");
2885     assert(dest->is_double_cpu(), "expect double result reg");
2886     const Register l_lo = left->as_register_lo();
2887     Register l_hi = left->as_register_hi();
2888     check_register_collision(dest->as_register_lo(), &l_hi);
2889     __ rsbs(dest->as_register_lo(), l_lo, 0);
2890     __ rsc(dest->as_register_hi(), l_hi, 0);
2891   } else if (left->is_single_fpu()) {
2892     assert(dest->is_single_fpu(), "expect single float result reg");
2893     __ vneg_f32(dest->as_float_reg(), left->as_float_reg());
2894   } else if (left->is_double_fpu()) {
2895     assert(left->is_double_fpu(), "expect double float operand reg");
2896     assert(dest->is_double_fpu(), "expect double float result reg");
2897     __ vneg_f64(dest->as_double_reg(), left->as_double_reg());
2898   } else {
2899       ShouldNotReachHere();
2900   }
2901 }
2902 
2903 
2904 void LIR_Assembler::leal(LIR_Opr addr, LIR_Opr dest, LIR_PatchCode patch_code, CodeEmitInfo* info) {
2905   assert(patch_code == lir_patch_none, "Patch code not supported");
2906   __ lea(dest->as_register(), as_Address(addr->as_address_ptr(), noreg, Address::IDT_LEA));
2907 }
2908 
2909 
2910 void LIR_Assembler::rt_call(LIR_Opr result, address dest, const LIR_OprList* args, LIR_Opr tmp, CodeEmitInfo* info) {
2911   assert(!tmp->is_valid(), "don't need temporary");
2912   CodeBlob *cb = CodeCache::find_blob(dest);
2913   if (cb) {
2914     __ far_call(RuntimeAddress(dest));
2915   } else {
2916     __ lea(rscratch1, RuntimeAddress(dest));
2917     __ bl(rscratch1);
2918   }
2919   if (info != NULL) {
2920     add_call_info_here(info);
2921   }
2922   __ maybe_isb();
2923 }
2924 
2925 void LIR_Assembler::volatile_move_op(LIR_Opr src, LIR_Opr dest, BasicType type, CodeEmitInfo* info) {
2926   if (type == T_LONG || type == T_DOUBLE) {
2927     const LIR_Opr long_val = FrameMap::long0_opr;
2928 
2929     int null_check_offset = -1;
2930 
2931     if (src->is_register() && dest->is_address()) {
2932       // long1 reserved as temp by LinearScan::pd_add_temps
2933       const LIR_Opr long_tmp = FrameMap::long1_opr;
2934       __ lea(rscratch1, as_Address_lo(dest->as_address_ptr(), Address::IDT_LEA));
2935 
2936 
2937       if (src->is_double_fpu()) {
2938         assert(type == T_DOUBLE, "invalid register allocation");
2939         // long0 reserved as temp by LinearScan::pd_add_temps
2940         __ vmov_f64(long_val->as_register_lo(), long_val->as_register_hi(), src->as_double_reg());
2941       } else {
2942         assert(type == T_LONG && src->is_same_register(long_val), "T_LONG src should be in long0 (by LIRGenerator)");
2943       }
2944 
2945       null_check_offset = __ offset();
2946       __ atomic_strd(long_val->as_register_lo(), long_val->as_register_hi(), rscratch1,
2947           long_tmp->as_register_lo(), long_tmp->as_register_hi());
2948 
2949     } else if (src->is_address() && dest->is_register()) {
2950       __ lea(rscratch1, as_Address_lo(src->as_address_ptr(), Address::IDT_LEA));
2951 
2952       null_check_offset = __ offset();
2953       __ atomic_ldrd(long_val->as_register_lo(), long_val->as_register_hi(), rscratch1);
2954 
2955       if (dest->is_double_fpu()) {
2956         __ vmov_f64(dest->as_double_reg(), long_val->as_register_lo(), long_val->as_register_hi());
2957       } else {
2958         assert(type != T_LONG || dest->is_same_register(long_val), "T_LONG dest should be in long0 (by LIRGenerator)");
2959       }
2960     } else {
2961       Unimplemented();
2962     }
2963 
2964     if (info != NULL) {
2965       add_debug_info_for_null_check(null_check_offset, info);
2966     }
2967 
2968   } else {
2969     move_op(src, dest, type, lir_patch_none, info,
2970             /*pop_fpu_stack*/false, /*unaligned*/false, /*wide*/false);
2971   }
2972 }
2973 
2974 #ifdef ASSERT
2975 // emit run-time assertion
2976 void LIR_Assembler::emit_assert(LIR_OpAssert* op) {
2977   assert(op->code() == lir_assert, "must be");
2978 
2979   if (op->in_opr1()->is_valid()) {
2980     assert(op->in_opr2()->is_valid(), "both operands must be valid");
2981     comp_op(op->condition(), op->in_opr1(), op->in_opr2(), op);
2982   } else {
2983     assert(op->in_opr2()->is_illegal(), "both operands must be illegal");
2984     assert(op->condition() == lir_cond_always, "no other conditions allowed");
2985   }
2986 
2987   Label ok;
2988   if (op->condition() != lir_cond_always) {
2989     Assembler::Condition acond = Assembler::AL;
2990     switch (op->condition()) {
2991       case lir_cond_equal:        acond = Assembler::EQ;  break;
2992       case lir_cond_notEqual:     acond = Assembler::NE;  break;
2993       case lir_cond_less:         acond = Assembler::LT;  break;
2994       case lir_cond_greaterEqual: acond = Assembler::GE;  break;
2995       case lir_cond_lessEqual:    acond = Assembler::LE;  break;
2996       case lir_cond_greater:      acond = Assembler::GT;  break;
2997       case lir_cond_belowEqual:   acond = Assembler::LS;  break;
2998       case lir_cond_aboveEqual:   acond = Assembler::HS;  break;
2999       default:                    ShouldNotReachHere();
3000     }
3001     if (op->in_opr1()->type() == T_LONG) {
3002       // a special trick here to be able to effectively compare jlongs
3003       // for the lessEqual and greater conditions the jlong operands are swapped
3004       // during comparison and hence should use mirror condition in conditional
3005       // instruction
3006       // see LIR_Assembler::comp_op and LIR_Assembler::cmove
3007       switch (op->condition()) {
3008         case lir_cond_lessEqual:    acond = Assembler::GE;  break;
3009         case lir_cond_greater:      acond = Assembler::LT;  break;
3010       }
3011     }
3012     __ b(ok, acond);
3013   }
3014   if (op->halt()) {
3015     const char* str = __ code_string(op->msg());
3016     __ stop(str);
3017   } else {
3018     breakpoint();
3019   }
3020   __ bind(ok);
3021 }
3022 #endif
3023 
3024 #ifndef PRODUCT
3025 #define COMMENT(x)   do { __ block_comment(x); } while (0)
3026 #else
3027 #define COMMENT(x)
3028 #endif
3029 
3030 void LIR_Assembler::membar() {
3031   COMMENT("membar");
3032   __ membar(MacroAssembler::AnyAny);
3033 }
3034 
3035 void LIR_Assembler::membar_acquire() {
3036   __ membar(Assembler::LoadLoad|Assembler::LoadStore);
3037 }
3038 
3039 void LIR_Assembler::membar_release() {
3040   __ membar(Assembler::LoadStore|Assembler::StoreStore);
3041 }
3042 
3043 void LIR_Assembler::membar_loadload() {
3044   __ membar(Assembler::LoadLoad);
3045 }
3046 
3047 void LIR_Assembler::membar_storestore() {
3048   __ membar(MacroAssembler::StoreStore);
3049 }
3050 
3051 void LIR_Assembler::membar_loadstore() { __ membar(MacroAssembler::LoadStore); }
3052 
3053 void LIR_Assembler::membar_storeload() { __ membar(MacroAssembler::StoreLoad); }
3054 
3055 void LIR_Assembler::on_spin_wait() {
3056   Unimplemented();
3057 }
3058 
3059 void LIR_Assembler::get_thread(LIR_Opr result_reg) {
3060   __ mov(result_reg->as_register(), rthread);
3061 }
3062 
3063 
3064 void LIR_Assembler::peephole(LIR_List *lir) {
3065 #if 0
3066   if (tableswitch_count >= max_tableswitches)
3067     return;
3068 
3069   /*
3070     This finite-state automaton recognizes sequences of compare-and-
3071     branch instructions.  We will turn them into a tableswitch.  You
3072     could argue that C1 really shouldn't be doing this sort of
3073     optimization, but without it the code is really horrible.
3074   */
3075 
3076   enum { start_s, cmp1_s, beq_s, cmp_s } state;
3077   int first_key, last_key = -2147483648;
3078   int next_key = 0;
3079   int start_insn = -1;
3080   int last_insn = -1;
3081   Register reg = noreg;
3082   LIR_Opr reg_opr;
3083   state = start_s;
3084 
3085   LIR_OpList* inst = lir->instructions_list();
3086   for (int i = 0; i < inst->length(); i++) {
3087     LIR_Op* op = inst->at(i);
3088     switch (state) {
3089     case start_s:
3090       first_key = -1;
3091       start_insn = i;
3092       switch (op->code()) {
3093       case lir_cmp:
3094         LIR_Opr opr1 = op->as_Op2()->in_opr1();
3095         LIR_Opr opr2 = op->as_Op2()->in_opr2();
3096         if (opr1->is_cpu_register() && opr1->is_single_cpu()
3097             && opr2->is_constant()
3098             && opr2->type() == T_INT) {
3099           reg_opr = opr1;
3100           reg = opr1->as_register();
3101           first_key = opr2->as_constant_ptr()->as_jint();
3102           next_key = first_key + 1;
3103           state = cmp_s;
3104           goto next_state;
3105         }
3106         break;
3107       }
3108       break;
3109     case cmp_s:
3110       switch (op->code()) {
3111       case lir_branch:
3112         if (op->as_OpBranch()->cond() == lir_cond_equal) {
3113           state = beq_s;
3114           last_insn = i;
3115           goto next_state;
3116         }
3117       }
3118       state = start_s;
3119       break;
3120     case beq_s:
3121       switch (op->code()) {
3122       case lir_cmp: {
3123         LIR_Opr opr1 = op->as_Op2()->in_opr1();
3124         LIR_Opr opr2 = op->as_Op2()->in_opr2();
3125         if (opr1->is_cpu_register() && opr1->is_single_cpu()
3126             && opr1->as_register() == reg
3127             && opr2->is_constant()
3128             && opr2->type() == T_INT
3129             && opr2->as_constant_ptr()->as_jint() == next_key) {
3130           last_key = next_key;
3131           next_key++;
3132           state = cmp_s;
3133           goto next_state;
3134         }
3135       }
3136       }
3137       last_key = next_key;
3138       state = start_s;
3139       break;
3140     default:
3141       assert(false, "impossible state");
3142     }
3143     if (state == start_s) {
3144       if (first_key < last_key - 5L && reg != noreg) {
3145         {
3146           // printf("found run register %d starting at insn %d low value %d high value %d\n",
3147           //        reg->encoding(),
3148           //        start_insn, first_key, last_key);
3149           //   for (int i = 0; i < inst->length(); i++) {
3150           //     inst->at(i)->print();
3151           //     tty->print("\n");
3152           //   }
3153           //   tty->print("\n");
3154         }
3155 
3156         struct tableswitch *sw = &switches[tableswitch_count];
3157         sw->_insn_index = start_insn, sw->_first_key = first_key,
3158           sw->_last_key = last_key, sw->_reg = reg;
3159         inst->insert_before(last_insn + 1, new LIR_OpLabel(&sw->_after));
3160         {
3161           // Insert the new table of branches
3162           int offset = last_insn;
3163           for (int n = first_key; n < last_key; n++) {
3164             inst->insert_before
3165               (last_insn + 1,
3166                new LIR_OpBranch(lir_cond_always, T_ILLEGAL,
3167                                 inst->at(offset)->as_OpBranch()->label()));
3168             offset -= 2, i++;
3169           }
3170         }
3171         // Delete all the old compare-and-branch instructions
3172         for (int n = first_key; n < last_key; n++) {
3173           inst->remove_at(start_insn);
3174           inst->remove_at(start_insn);
3175         }
3176         // Insert the tableswitch instruction
3177         inst->insert_before(start_insn,
3178                             new LIR_Op2(lir_cmp, lir_cond_always,
3179                                         LIR_OprFact::intConst(tableswitch_count),
3180                                         reg_opr));
3181         inst->insert_before(start_insn + 1, new LIR_OpLabel(&sw->_branches));
3182         tableswitch_count++;
3183       }
3184       reg = noreg;
3185       last_key = -2147483648;
3186     }
3187   next_state:
3188     ;
3189   }
3190 #endif
3191 }
3192 
3193 void LIR_Assembler::atomic_op(LIR_Code code, LIR_Opr src, LIR_Opr data, LIR_Opr dest, LIR_Opr tmp_op) {
3194   BasicType type = src->type();
3195   Address addr = as_Address(src->as_address_ptr(), Address::toInsnDataType(type));
3196 
3197   bool is_long = false;
3198 
3199   switch(type) {
3200   case T_INT:
3201   case T_OBJECT:
3202   case T_ARRAY:
3203     break;
3204   case T_LONG:
3205     is_long = true;
3206     break;
3207   default:
3208     ShouldNotReachHere();
3209   }
3210 
3211   switch (code) {
3212   case lir_xadd:
3213     {
3214       Register tmp = tmp_op->as_register();
3215       Register dst = as_reg(dest);
3216       Label again;
3217       __ lea(tmp, addr);
3218       __ bind(again);
3219       if(is_long) {
3220           assert(dest->as_register_lo()->successor() == dest->as_register_hi(), "must be contiguous");
3221           assert((dest->as_register_lo()->encoding() & 1) == 0, "must be even");
3222           _masm->ldrexd(dst, tmp);
3223       } else {
3224           _masm->ldrex(dst, tmp);
3225       }
3226       arith_op(lir_add, dest, data, dest, NULL, false);
3227       if (is_long) {
3228         _masm->strexd(rscratch1, dst, tmp);
3229       } else {
3230         _masm->strex(rscratch1, dst, tmp);
3231       }
3232       __ cbnz(rscratch1, again);
3233       arith_op(lir_sub, dest, data, dest, NULL, false);
3234       break;
3235     }
3236   case lir_xchg:
3237     {
3238       Register tmp = tmp_op->as_register();
3239       Register obj = as_reg(data);
3240       Register dst = as_reg(dest);
3241       assert_different_registers(obj, addr.base(), tmp, rscratch1, dst);
3242       Label again;
3243       __ lea(tmp, addr);
3244       __ bind(again);
3245       if(is_long) {
3246           assert(dest->as_register_lo()->successor() == dest->as_register_hi(), "must be contiguous");
3247           assert((dest->as_register_lo()->encoding() & 1) == 0, "must be even");
3248 
3249           assert(data->is_double_cpu(), "should be double register");
3250           assert(data->as_register_lo()->successor() == data->as_register_hi(), "must be contiguous");
3251           assert((data->as_register_lo()->encoding() & 1) == 0, "must be even");
3252 
3253           _masm->ldrexd(dst, tmp);
3254           _masm->strexd(rscratch1, obj, tmp);
3255       } else {
3256          _masm->ldrex(dst, tmp);
3257          _masm->strex(rscratch1, obj, tmp);
3258       }
3259       __ cbnz(rscratch1, again);
3260     }
3261     break;
3262   default:
3263     ShouldNotReachHere();
3264   }
3265   __ membar(__ AnyAny);
3266 }
3267 
3268 void LIR_Assembler::check_register_collision(Register d, Register *s1, Register *s2, Register tmp) {
3269   // use a temp if any of the registers used as a source of operation
3270   // collide with result register of the prerequisite operation
3271   if (d == *s1) {
3272     __ mov(tmp, d);
3273     *s1 = tmp;
3274   } else if (s2 && d == *s2) {
3275     __ mov(tmp, d);
3276     *s2 = tmp;
3277   }
3278 }
3279 
3280 #undef __