1 /*
   2  * Copyright (c) 2000, 2015, Oracle and/or its affiliates. All rights reserved.
   3  * Copyright (c) 2014, Red Hat Inc. All rights reserved.
   4  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   5  *
   6  * This code is free software; you can redistribute it and/or modify it
   7  * under the terms of the GNU General Public License version 2 only, as
   8  * published by the Free Software Foundation.
   9  *
  10  * This code is distributed in the hope that it will be useful, but WITHOUT
  11  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  12  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  13  * version 2 for more details (a copy is included in the LICENSE file that
  14  * accompanied this code).
  15  *
  16  * You should have received a copy of the GNU General Public License version
  17  * 2 along with this work; if not, write to the Free Software Foundation,
  18  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  19  *
  20  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  21  * or visit www.oracle.com if you need additional information or have any
  22  * questions.
  23  *
  24  */
  25 // This file is a derivative work resulting from (and including) modifications
  26 // made by Azul Systems, Inc.  The dates of such changes are 2013-2016.
  27 // Copyright 2013-2016 Azul Systems, Inc.  All Rights Reserved.
  28 //
  29 // Please contact Azul Systems, 385 Moffett Park Drive, Suite 115, Sunnyvale,
  30 // CA 94089 USA or visit www.azul.com if you need additional information or
  31 // have any questions.
  32 
  33 #include "precompiled.hpp"
  34 #include "asm/assembler.hpp"
  35 #include "c1/c1_CodeStubs.hpp"
  36 #include "c1/c1_Compilation.hpp"
  37 #include "c1/c1_LIRAssembler.hpp"
  38 #include "c1/c1_MacroAssembler.hpp"
  39 #include "c1/c1_Runtime1.hpp"
  40 #include "c1/c1_ValueStack.hpp"
  41 #include "ci/ciArrayKlass.hpp"
  42 #include "ci/ciInstance.hpp"
  43 #include "gc_interface/collectedHeap.hpp"
  44 #include "memory/barrierSet.hpp"
  45 #include "memory/cardTableModRefBS.hpp"
  46 #include "nativeInst_aarch32.hpp"
  47 #include "oops/objArrayKlass.hpp"
  48 #include "runtime/sharedRuntime.hpp"
  49 #include "vmreg_aarch32.inline.hpp"
  50 
  51 #ifndef PRODUCT
  52 #define COMMENT(x)   do { __ block_comment(x); } while (0)
  53 #else
  54 #define COMMENT(x)
  55 #endif
  56 
  57 NEEDS_CLEANUP // remove this definitions ?
  58 const Register IC_Klass    = rscratch2;   // where the IC klass is cached
  59 const Register SYNC_header = r0;   // synchronization header
  60 const Register SHIFT_count = r0;   // where count for shift operations must be
  61 
  62 #define __ _masm->
  63 
  64 
  65 static void select_different_registers(Register preserve,
  66                                        Register extra,
  67                                        Register &tmp1,
  68                                        Register &tmp2) {
  69   if (tmp1 == preserve) {
  70     assert_different_registers(tmp1, tmp2, extra);
  71     tmp1 = extra;
  72   } else if (tmp2 == preserve) {
  73     assert_different_registers(tmp1, tmp2, extra);
  74     tmp2 = extra;
  75   }
  76   assert_different_registers(preserve, tmp1, tmp2);
  77 }
  78 
  79 
  80 
  81 static void select_different_registers(Register preserve,
  82                                        Register extra,
  83                                        Register &tmp1,
  84                                        Register &tmp2,
  85                                        Register &tmp3) {
  86   if (tmp1 == preserve) {
  87     assert_different_registers(tmp1, tmp2, tmp3, extra);
  88     tmp1 = extra;
  89   } else if (tmp2 == preserve) {
  90     assert_different_registers(tmp1, tmp2, tmp3, extra);
  91     tmp2 = extra;
  92   } else if (tmp3 == preserve) {
  93     assert_different_registers(tmp1, tmp2, tmp3, extra);
  94     tmp3 = extra;
  95   }
  96   assert_different_registers(preserve, tmp1, tmp2, tmp3);
  97 }
  98 
  99 bool LIR_Assembler::is_small_constant(LIR_Opr opr) { Unimplemented(); return false; }
 100 
 101 
 102 LIR_Opr LIR_Assembler::receiverOpr() {
 103   return FrameMap::receiver_opr;
 104 }
 105 
 106 LIR_Opr LIR_Assembler::osrBufferPointer() {
 107   return FrameMap::as_pointer_opr(receiverOpr()->as_register());
 108 }
 109 
 110 //--------------fpu register translations-----------------------
 111 
 112 
 113 address LIR_Assembler::float_constant(float f) {
 114   address const_addr = __ float_constant(f);
 115   if (const_addr == NULL) {
 116     bailout("const section overflow");
 117     return __ code()->consts()->start();
 118   } else {
 119     return const_addr;
 120   }
 121 }
 122 
 123 
 124 address LIR_Assembler::double_constant(double d) {
 125   address const_addr = __ double_constant(d);
 126   if (const_addr == NULL) {
 127     bailout("const section overflow");
 128     return __ code()->consts()->start();
 129   } else {
 130     return const_addr;
 131   }
 132 }
 133 
 134 address LIR_Assembler::int_constant(jlong n) {
 135   address const_addr = __ long_constant(n);
 136   if (const_addr == NULL) {
 137     bailout("const section overflow");
 138     return __ code()->consts()->start();
 139   } else {
 140     return const_addr;
 141   }
 142 }
 143 
 144 void LIR_Assembler::set_24bit_FPU() { Unimplemented(); }
 145 
 146 void LIR_Assembler::reset_FPU() { Unimplemented(); }
 147 
 148 void LIR_Assembler::fpop() { Unimplemented(); }
 149 
 150 void LIR_Assembler::fxch(int i) { Unimplemented(); }
 151 
 152 void LIR_Assembler::fld(int i) { Unimplemented(); }
 153 
 154 void LIR_Assembler::ffree(int i) { Unimplemented(); }
 155 
 156 void LIR_Assembler::breakpoint() { __ bkpt(0); }
 157 
 158 void LIR_Assembler::push(LIR_Opr opr) { Unimplemented(); }
 159 
 160 void LIR_Assembler::pop(LIR_Opr opr) { Unimplemented(); }
 161 
 162 //-------------------------------------------
 163 
 164 static Register as_reg(LIR_Opr op) {
 165   return op->is_double_cpu() ? op->as_register_lo() : op->as_register();
 166 }
 167 
 168 Address LIR_Assembler::as_Address(LIR_Address* addr) {
 169   // as_Address(LIR_Address*, Address::InsnDataType) should be used instead
 170   ShouldNotCallThis();
 171 }
 172 
 173 Address LIR_Assembler::as_Address_hi(LIR_Address* addr) {
 174   // as_Address_hi(LIR_Address*, Address::InsnDataType) should be used instead
 175   ShouldNotCallThis();
 176 }
 177 
 178 Address LIR_Assembler::as_Address_lo(LIR_Address* addr) {
 179   // as_Address_lo(LIR_Address*, Address::InsnDataType) should be used instead
 180   ShouldNotCallThis();
 181 }
 182 
 183 Address LIR_Assembler::as_Address(LIR_Address* addr, Register tmp, Address::InsnDataType type) {
 184   if (addr->base()->is_illegal()) {
 185     assert(addr->index()->is_illegal(), "must be illegal too");
 186     __ mov(tmp, addr->disp());
 187     return Address(tmp); // encoding is ok for any data type
 188   }
 189 
 190   Register base = addr->base()->as_pointer_register();
 191 
 192   if (addr->index()->is_illegal()) {
 193     return Address(base, addr->disp()).safe_for(type, _masm, tmp);
 194   } else if (addr->index()->is_cpu_register()) {
 195     assert(addr->disp() == 0, "must be");
 196     Register index = addr->index()->as_pointer_register();
 197     return Address(base, index, lsl(addr->scale())).safe_for(type, _masm, tmp);
 198   } else if (addr->index()->is_constant()) {
 199     intptr_t addr_offset = (addr->index()->as_constant_ptr()->as_jint() << addr->scale()) + addr->disp();
 200     return Address(base, addr_offset).safe_for(type, _masm, tmp);
 201   }
 202 
 203   Unimplemented();
 204   return Address();
 205 }
 206 
 207 Address LIR_Assembler::as_Address_hi(LIR_Address* addr, Address::InsnDataType type) {
 208   assert(type == Address::IDT_INT, "only to be used for accessing high word of jlong");
 209 
 210   if (addr->base()->is_illegal()) {
 211     assert(addr->index()->is_illegal(), "must be illegal too");
 212     __ mov(rscratch1, addr->disp() + wordSize);
 213     return Address(rscratch1); // encoding is ok for IDR_INT
 214   }
 215 
 216   Register base = addr->base()->as_pointer_register();
 217 
 218   if (addr->index()->is_illegal()) {
 219     return Address(base, addr->disp() + wordSize).safe_for(Address::IDT_INT, _masm, rscratch1);
 220   } else if (addr->index()->is_cpu_register()) {
 221     assert(addr->disp() == 0, "must be");
 222     Register index = addr->index()->as_pointer_register();
 223     __ add(rscratch1, base, wordSize);
 224     return Address(rscratch1, index, lsl(addr->scale())); // encoding is ok for IDT_INT
 225   } else if (addr->index()->is_constant()) {
 226     intptr_t addr_offset = (addr->index()->as_constant_ptr()->as_jint() << addr->scale()) + addr->disp() + wordSize;
 227     return Address(base, addr_offset).safe_for(Address::IDT_INT, _masm, rscratch1);
 228   }
 229 
 230   Unimplemented();
 231   return Address();
 232 }
 233 
 234 Address LIR_Assembler::as_Address_lo(LIR_Address* addr, Address::InsnDataType type) {
 235   return as_Address(addr, rscratch1, type);
 236 }
 237 
 238 
 239 void LIR_Assembler::osr_entry() {
 240   offsets()->set_value(CodeOffsets::OSR_Entry, code_offset());
 241   BlockBegin* osr_entry = compilation()->hir()->osr_entry();
 242   ValueStack* entry_state = osr_entry->state();
 243   int number_of_locks = entry_state->locks_size();
 244 
 245   // we jump here if osr happens with the interpreter
 246   // state set up to continue at the beginning of the
 247   // loop that triggered osr - in particular, we have
 248   // the following registers setup:
 249   //
 250   // r1: osr buffer
 251   //
 252 
 253   // build frame
 254   ciMethod* m = compilation()->method();
 255   __ build_frame(initial_frame_size_in_bytes(), bang_size_in_bytes());
 256 
 257   // OSR buffer is
 258   //
 259   // locals[nlocals-1..0]
 260   // monitors[0..number_of_locks]
 261   //
 262   // locals is a direct copy of the interpreter frame so in the osr buffer
 263   // so first slot in the local array is the last local from the interpreter
 264   // and last slot is local[0] (receiver) from the interpreter
 265   //
 266   // Similarly with locks. The first lock slot in the osr buffer is the nth lock
 267   // from the interpreter frame, the nth lock slot in the osr buffer is 0th lock
 268   // in the interpreter frame (the method lock if a sync method)
 269 
 270   // Initialize monitors in the compiled activation.
 271   //   r1: pointer to osr buffer
 272   //
 273   // All other registers are dead at this point and the locals will be
 274   // copied into place by code emitted in the IR.
 275 
 276   Register OSR_buf = osrBufferPointer()->as_pointer_register();
 277   { assert(frame::interpreter_frame_monitor_size() == BasicObjectLock::size(), "adjust code below");
 278     int monitor_offset = BytesPerWord * method()->max_locals() +
 279       (2 * BytesPerWord) * (number_of_locks - 1);
 280     // SharedRuntime::OSR_migration_begin() packs BasicObjectLocks in
 281     // the OSR buffer using 2 word entries: first the lock and then
 282     // the oop.
 283     for (int i = 0; i < number_of_locks; i++) {
 284       int slot_offset = monitor_offset - ((i * 2) * BytesPerWord);
 285 #ifdef ASSERT
 286       // verify the interpreter's monitor has a non-null object
 287       {
 288         Label L;
 289         __ ldr(rscratch1, Address(OSR_buf, slot_offset + 1*BytesPerWord));
 290         __ cbnz(rscratch1, L);
 291         __ stop("locked object is NULL");
 292         __ bind(L);
 293       }
 294 #endif
 295       __ ldr(rscratch1, Address(OSR_buf, slot_offset + 0));
 296       __ str(rscratch1, frame_map()->address_for_monitor_lock(i));
 297       __ ldr(rscratch1, Address(OSR_buf, slot_offset + 1*BytesPerWord));
 298       __ str(rscratch1, frame_map()->address_for_monitor_object(i));
 299     }
 300   }
 301 }
 302 
 303 
 304 // inline cache check; done before the frame is built.
 305 int LIR_Assembler::check_icache() {
 306   Register receiver = FrameMap::receiver_opr->as_register();
 307   Register ic_klass = IC_Klass;
 308   int start_offset = __ offset();
 309   __ inline_cache_check(receiver, ic_klass);
 310 
 311   // if icache check fails, then jump to runtime routine
 312   // Note: RECEIVER must still contain the receiver!
 313   Label dont;
 314   __ b(dont, Assembler::EQ);
 315   __ far_jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
 316 
 317   // We align the verified entry point unless the method body
 318   // (including its inline cache check) will fit in a single 64-byte
 319   // icache line.
 320   if (! method()->is_accessor() || __ offset() - start_offset > 4 * 4) {
 321     // force alignment after the cache check.
 322     __ align(CodeEntryAlignment);
 323   }
 324 
 325   __ bind(dont);
 326   return start_offset;
 327 }
 328 
 329 
 330 void LIR_Assembler::jobject2reg(jobject o, Register reg) {
 331   if (o == NULL) {
 332     __ mov(reg, 0);
 333   } else {
 334     __ movoop(reg, o, /*immediate*/true);
 335   }
 336 }
 337 
 338 void LIR_Assembler::deoptimize_trap(CodeEmitInfo *info) {
 339   __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::deoptimize_id)));
 340   add_call_info_here(info);
 341 }
 342 
 343 void LIR_Assembler::jobject2reg_with_patching(Register reg, CodeEmitInfo *info) {
 344   jobject o = NULL;
 345   PatchingStub* patch = new PatchingStub(_masm, patching_id(info));
 346   __ movoop(reg, o, true);
 347   patching_epilog(patch, lir_patch_normal, reg, info);
 348 }
 349 
 350 // Return sp decrement needed to build a frame
 351 int LIR_Assembler::initial_frame_size_in_bytes() const {
 352   // We need to subtract two words to take into account saved lr and rfp.
 353   return in_bytes(frame_map()->framesize_in_bytes()) -
 354          FrameMap::frame_pad_in_bytes;
 355 }
 356 
 357 int LIR_Assembler::emit_exception_handler() {
 358   // if the last instruction is a call (typically to do a throw which
 359   // is coming at the end after block reordering) the return address
 360   // must still point into the code area in order to avoid assertion
 361   // failures when searching for the corresponding bci => add a nop
 362   // (was bug 5/14/1999 - gri)
 363   __ nop();
 364 
 365   // generate code for exception handler
 366   address handler_base = __ start_a_stub(exception_handler_size);
 367   if (handler_base == NULL) {
 368     // not enough space left for the handler
 369     bailout("exception handler overflow");
 370     return -1;
 371   }
 372 
 373   int offset = code_offset();
 374 
 375   // the exception oop and pc are in r0, and r3
 376   // no other registers need to be preserved, so invalidate them
 377   __ invalidate_registers(false, true, false);
 378 
 379   // check that there is really an exception
 380   __ verify_not_null_oop(r0);
 381 
 382   // search an exception handler (r0: exception oop, r3: throwing pc)
 383   __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::handle_exception_from_callee_id)));  __ should_not_reach_here();
 384   guarantee(code_offset() - offset <= exception_handler_size, "overflow");
 385   __ end_a_stub();
 386 
 387   return offset;
 388 }
 389 
 390 
 391 // Emit the code to remove the frame from the stack in the exception
 392 // unwind path.
 393 int LIR_Assembler::emit_unwind_handler() {
 394 #ifndef PRODUCT
 395   if (CommentedAssembly) {
 396     _masm->block_comment("Unwind handler");
 397   }
 398 #endif
 399 
 400   int offset = code_offset();
 401 
 402   // Fetch the exception from TLS and clear out exception related thread state
 403   __ ldr(r0, Address(rthread, JavaThread::exception_oop_offset()));
 404   __ mov(rscratch1, 0);
 405   __ str(rscratch1, Address(rthread, JavaThread::exception_oop_offset()));
 406   __ str(rscratch1, Address(rthread, JavaThread::exception_pc_offset()));
 407 
 408   __ bind(_unwind_handler_entry);
 409   __ verify_not_null_oop(r0);
 410 
 411   // Preform needed unlocking
 412   MonitorExitStub* stub = NULL;
 413   if (method()->is_synchronized()) {
 414     monitor_address(0, FrameMap::r1_opr);
 415     stub = new MonitorExitStub(FrameMap::r1_opr, true, 0);
 416     __ unlock_object(r5, r4, r1, *stub->entry());
 417     __ bind(*stub->continuation());
 418   }
 419 
 420   if (compilation()->env()->dtrace_method_probes()) {
 421     __ call_Unimplemented();
 422 #if 0
 423     // FIXME check exception_store is not clobbered below!
 424     __ movptr(Address(rsp, 0), rax);
 425     __ mov_metadata(Address(rsp, sizeof(void*)), method()->constant_encoding());
 426     __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_exit)));
 427 #endif
 428   }
 429 
 430   // remove the activation and dispatch to the unwind handler
 431   __ block_comment("remove_frame and dispatch to the unwind handler");
 432   __ remove_frame(initial_frame_size_in_bytes());
 433   __ far_jump(RuntimeAddress(Runtime1::entry_for(Runtime1::unwind_exception_id)));
 434 
 435   // Emit the slow path assembly
 436   if (stub != NULL) {
 437     stub->emit_code(this);
 438   }
 439 
 440   return offset;
 441 }
 442 
 443 
 444 int LIR_Assembler::emit_deopt_handler() {
 445   // if the last instruction is a call (typically to do a throw which
 446   // is coming at the end after block reordering) the return address
 447   // must still point into the code area in order to avoid assertion
 448   // failures when searching for the corresponding bci => add a nop
 449   // (was bug 5/14/1999 - gri)
 450   __ nop();
 451 
 452   // generate code for exception handler
 453   address handler_base = __ start_a_stub(deopt_handler_size);
 454   if (handler_base == NULL) {
 455     // not enough space left for the handler
 456     bailout("deopt handler overflow");
 457     return -1;
 458   }
 459 
 460   int offset = code_offset();
 461 
 462   __ adr(lr, pc());
 463   __ far_jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
 464   guarantee(code_offset() - offset <= deopt_handler_size, "overflow");
 465   __ end_a_stub();
 466 
 467   return offset;
 468 }
 469 
 470 
 471 // This is the fast version of java.lang.String.compare; it has not
 472 // OSR-entry and therefore, we generate a slow version for OSR's
 473 void LIR_Assembler::emit_string_compare(LIR_Opr arg0, LIR_Opr arg1, LIR_Opr dst, CodeEmitInfo* info)  {
 474   __ mov(r2, (address)__FUNCTION__);
 475   __ call_Unimplemented();
 476 }
 477 
 478 
 479 void LIR_Assembler::add_debug_info_for_branch(address adr, CodeEmitInfo* info) {
 480   _masm->code_section()->relocate(adr, relocInfo::poll_type);
 481   int pc_offset = code_offset();
 482   flush_debug_info(pc_offset);
 483   info->record_debug_info(compilation()->debug_info_recorder(), pc_offset);
 484   if (info->exception_handlers() != NULL) {
 485     compilation()->add_exception_handlers_for_pco(pc_offset, info->exception_handlers());
 486   }
 487 }
 488 
 489 void LIR_Assembler::return_op(LIR_Opr result) {
 490   assert(result->is_illegal() || !result->is_single_cpu() || result->as_register() == r0, "word returns are in r0,");
 491   // Pop the stack before the safepoint code
 492   __ remove_frame(initial_frame_size_in_bytes());
 493   address polling_page(os::get_polling_page());
 494   __ read_polling_page(rscratch1, polling_page, relocInfo::poll_return_type);
 495   __ ret(lr);
 496 }
 497 
 498 int LIR_Assembler::safepoint_poll(LIR_Opr tmp, CodeEmitInfo* info) {
 499   address polling_page(os::get_polling_page());
 500   guarantee(info != NULL, "Shouldn't be NULL");
 501   assert(os::is_poll_address(polling_page), "should be");
 502   __ mov(rscratch1, Address(polling_page, relocInfo::poll_type));
 503   add_debug_info_for_branch(info);  // This isn't just debug info:
 504   // it's the oop map
 505   __ read_polling_page(rscratch1, relocInfo::poll_type);
 506   return __ offset();
 507 }
 508 
 509 void LIR_Assembler::move_regs(Register from_reg, Register to_reg) {
 510   if (from_reg != to_reg) {
 511     __ mov(to_reg, from_reg);
 512   }
 513 }
 514 
 515 void LIR_Assembler::swap_reg(Register a, Register b) {
 516   Unimplemented();
 517 }
 518 
 519 void LIR_Assembler::const2reg(LIR_Opr src, LIR_Opr dest, LIR_PatchCode patch_code, CodeEmitInfo* info) {
 520   assert(src->is_constant(), "should not call otherwise");
 521   assert(dest->is_register(), "should not call otherwise");
 522   LIR_Const* c = src->as_constant_ptr();
 523 
 524   switch (c->type()) {
 525     case T_INT: {
 526       assert(patch_code == lir_patch_none, "no patching handled here");
 527       __ mov(dest->as_register(), c->as_jint());
 528       break;
 529     }
 530 
 531     case T_ADDRESS: {
 532       assert(patch_code == lir_patch_none, "no patching handled here");
 533       __ mov(dest->as_register(), c->as_jint());
 534       break;
 535     }
 536 
 537     case T_LONG: {
 538       assert(patch_code == lir_patch_none, "no patching handled here");
 539       __ mov(dest->as_register_lo(), c->as_jint_lo());
 540       __ mov(dest->as_register_hi(), c->as_jint_hi());
 541       break;
 542     }
 543 
 544     case T_OBJECT: {
 545         if (patch_code == lir_patch_none) {
 546           jobject2reg(c->as_jobject(), dest->as_register());
 547         } else {
 548           jobject2reg_with_patching(dest->as_register(), info);
 549         }
 550       break;
 551     }
 552 
 553     case T_METADATA: {
 554       if (patch_code != lir_patch_none) {
 555         klass2reg_with_patching(dest->as_register(), info);
 556       } else {
 557         __ mov_metadata(dest->as_register(), c->as_metadata());
 558       }
 559       break;
 560     }
 561 
 562     case T_FLOAT: {
 563 #ifdef __ARM_PCS_VFP
 564         if (__ operand_valid_for_float_immediate(c->as_jfloat())) {
 565             __ vmov_f32(dest->as_float_reg(), c->as_jfloat());
 566         } else {
 567             __ lea(rscratch1, InternalAddress(float_constant(c->as_jfloat())));
 568             __ vldr_f32(dest->as_float_reg(), Address(rscratch1));
 569         }
 570 #else
 571 #error "unimplemented"
 572 #endif
 573       break;
 574     }
 575 
 576     case T_DOUBLE: {
 577 #ifdef __ARM_PCS_VFP
 578         if (__ operand_valid_for_double_immediate(c->as_jdouble())) {
 579             __ vmov_f64(dest->as_double_reg(), c->as_jdouble());
 580         } else {
 581             __ lea(rscratch1, InternalAddress(double_constant(c->as_jdouble())));
 582             __ vldr_f64(dest->as_double_reg(), Address(rscratch1));
 583         }
 584 #else
 585 #error "unimplemented"
 586 #endif
 587       break;
 588     }
 589 
 590     default:
 591       ShouldNotReachHere();
 592   }
 593 }
 594 
 595 void LIR_Assembler::const2stack(LIR_Opr src, LIR_Opr dest) {
 596   LIR_Const* c = src->as_constant_ptr();
 597   switch (c->type()) {
 598   case T_OBJECT:
 599     {
 600       if (! c->as_jobject()) {
 601         __ mov(rscratch1, 0);
 602         __ str(rscratch1, frame_map()->address_for_slot(dest->single_stack_ix()));
 603       } else {
 604         const2reg(src, FrameMap::rscratch1_opr, lir_patch_none, NULL);
 605         reg2stack(FrameMap::rscratch1_opr, dest, c->type(), false);
 606       }
 607     }
 608     break;
 609   case T_ADDRESS:
 610     {
 611       const2reg(src, FrameMap::rscratch1_opr, lir_patch_none, NULL);
 612       reg2stack(FrameMap::rscratch1_opr, dest, c->type(), false);
 613     }
 614   case T_INT:
 615   case T_FLOAT:
 616     {
 617       __ mov(rscratch1, c->as_jint_bits());
 618       __ str(rscratch1, frame_map()->address_for_slot(dest->single_stack_ix()));
 619     }
 620     break;
 621   case T_LONG:
 622   case T_DOUBLE:
 623     {
 624         __ mov(rscratch1, c->as_jint_lo());
 625         __ str(rscratch1, frame_map()->address_for_slot(dest->double_stack_ix(),
 626                                                         lo_word_offset_in_bytes));
 627         if (c->as_jint_lo() != c->as_jint_hi())
 628             __ mov(rscratch1, c->as_jint_hi());
 629         __ str(rscratch1, frame_map()->address_for_slot(dest->double_stack_ix(),
 630                                                         hi_word_offset_in_bytes));
 631     }
 632     break;
 633   default:
 634     ShouldNotReachHere();
 635   }
 636 }
 637 
 638 /*
 639  * For now this code can load only zero constants as in aarch32.
 640  * It seems like this implementation can break some tests in future.
 641  * TODO: ensure, write test, and rewrite if need.
 642  */
 643 void LIR_Assembler::const2mem(LIR_Opr src, LIR_Opr dest, BasicType type, CodeEmitInfo* info, bool wide) {
 644   assert(src->is_constant(), "should not call otherwise");
 645   LIR_Const* c = src->as_constant_ptr();
 646   LIR_Address* to_addr = dest->as_address_ptr();
 647 
 648   void (Assembler::* insn)(Register Rt, const Address &adr, Assembler::Condition cnd);
 649 
 650   __ mov(rscratch2, 0);
 651 
 652   int null_check_here = code_offset();
 653 
 654   Address::InsnDataType idt = Address::toInsnDataType(type);
 655   switch (type) {
 656   case T_ADDRESS:
 657     assert(c->as_jint() == 0, "should be");
 658     insn = &Assembler::str;
 659     break;
 660   case T_LONG: {
 661     assert(c->as_jlong() == 0, "should be");
 662     insn = &Assembler::str;
 663     Address addr = as_Address_hi(to_addr, Address::IDT_INT);
 664     null_check_here = code_offset();
 665     __ str(rscratch2, addr);
 666     idt = Address::IDT_INT;
 667     break;
 668   }
 669   case T_INT:
 670     assert(c->as_jint() == 0, "should be");
 671     insn = &Assembler::str;
 672     break;
 673   case T_OBJECT:
 674   case T_ARRAY:
 675     assert(c->as_jobject() == 0, "should be");
 676     insn = &Assembler::str;
 677     break;
 678   case T_CHAR:
 679   case T_SHORT:
 680     assert(c->as_jint() == 0, "should be");
 681     insn = &Assembler::strh;
 682     break;
 683   case T_BOOLEAN:
 684   case T_BYTE:
 685     assert(c->as_jint() == 0, "should be");
 686     insn = &Assembler::strb;
 687     break;
 688   default:
 689     ShouldNotReachHere();
 690   }
 691 
 692   (_masm->*insn)(rscratch2, as_Address(to_addr, idt), Assembler::C_DFLT);
 693   if (info) add_debug_info_for_null_check(null_check_here, info);
 694 }
 695 
 696 void LIR_Assembler::reg2reg(LIR_Opr src, LIR_Opr dest) {
 697   assert(src->is_register(), "should not call otherwise");
 698   assert(dest->is_register(), "should not call otherwise");
 699 
 700   // move between cpu-registers
 701   if (dest->is_single_cpu()) {
 702     if (src->type() == T_LONG) {
 703       // Can do LONG -> OBJECT
 704       __ stop("investigate how \"LONG -> OBJECT\" works especially when high part is != 0");
 705       move_regs(src->as_register_lo(), dest->as_register());
 706       return;
 707     }
 708     assert(src->is_single_cpu(), "must match");
 709     if (src->type() == T_OBJECT) {
 710       __ verify_oop(src->as_register());
 711     }
 712     move_regs(src->as_register(), dest->as_register());
 713 
 714   } else if (dest->is_double_cpu()) {
 715     if (src->type() == T_OBJECT || src->type() == T_ARRAY) {
 716       // Surprising to me but we can see move of a long to t_object
 717       __ verify_oop(src->as_register());
 718       move_regs(src->as_register(), dest->as_register_lo());
 719       __ mov(dest->as_register_hi(), 0);
 720       return;
 721     }
 722     assert(src->is_double_cpu(), "must match");
 723     Register f_lo = src->as_register_lo();
 724     Register f_hi = src->as_register_hi();
 725     Register t_lo = dest->as_register_lo();
 726     Register t_hi = dest->as_register_hi();
 727     assert(f_hi != f_lo, "must be different");
 728     assert(t_hi != t_lo, "must be different");
 729     check_register_collision(t_lo, &f_hi);
 730     move_regs(f_lo, t_lo);
 731     move_regs(f_hi, t_hi);
 732   } else if (dest->is_single_fpu()) {
 733     __ vmov_f32(dest->as_float_reg(), src->as_float_reg());
 734 
 735   } else if (dest->is_double_fpu()) {
 736     __ vmov_f64(dest->as_double_reg(), src->as_double_reg());
 737 
 738   } else {
 739     ShouldNotReachHere();
 740   }
 741 }
 742 
 743 void LIR_Assembler::reg2stack(LIR_Opr src, LIR_Opr dest, BasicType type, bool pop_fpu_stack) {
 744   if (src->is_single_cpu()) {
 745     if (type == T_ARRAY || type == T_OBJECT) {
 746       __ str(src->as_register(), frame_map()->address_for_slot(dest->single_stack_ix()));
 747       __ verify_oop(src->as_register());
 748     } else {
 749       __ str(src->as_register(), frame_map()->address_for_slot(dest->single_stack_ix()));
 750     }
 751 
 752   } else if (src->is_double_cpu()) {
 753     Address dest_addr_LO = frame_map()->address_for_slot(dest->double_stack_ix(), lo_word_offset_in_bytes);
 754     __ strd(src->as_register_lo(), src->as_register_hi(), dest_addr_LO);
 755 
 756   } else if (src->is_single_fpu()) {
 757     Address dest_addr = frame_map()->address_for_slot(dest->single_stack_ix());
 758 #ifdef __ARM_PCS_VFP
 759     __ vstr_f32(src->as_float_reg(), dest_addr.safe_for(Address::IDT_FLOAT, _masm, rscratch1));
 760 #else
 761 #error "unimplemented"
 762 #endif
 763   } else if (src->is_double_fpu()) {
 764     Address dest_addr = frame_map()->address_for_slot(dest->double_stack_ix());
 765 #ifdef __ARM_PCS_VFP
 766     __ vstr_f64(src->as_double_reg(), dest_addr.safe_for(Address::IDT_DOUBLE, _masm, rscratch1));
 767 #else
 768 #error "unimplemented"
 769 #endif
 770   } else {
 771     ShouldNotReachHere();
 772   }
 773 
 774 }
 775 
 776 
 777 void LIR_Assembler::reg2mem(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_PatchCode patch_code, CodeEmitInfo* info, bool pop_fpu_stack, bool wide, bool /* unaligned */) {
 778   LIR_Address* to_addr = dest->as_address_ptr();
 779 
 780   if (type == T_ARRAY || type == T_OBJECT) {
 781     __ verify_oop(src->as_register());
 782   }
 783 
 784   PatchingStub* patch = NULL;
 785   if (patch_code != lir_patch_none) {
 786     assert(to_addr->disp() != 0, "must have");
 787 
 788     patch = new PatchingStub(_masm, PatchingStub::access_field_id);
 789     __ mov(rscratch1, (address) to_addr->disp());
 790     patching_epilog(patch, patch_code, to_addr->base()->as_register(), info);
 791 
 792     to_addr = new LIR_Address(to_addr->base(), FrameMap::rscratch1_opr, to_addr->type());
 793   }
 794 
 795 
 796   int null_check_here = code_offset();
 797   switch (type) {
 798     case T_FLOAT: {
 799 #ifdef __ARM_PCS_VFP
 800       Address addr = as_Address(to_addr, Address::IDT_FLOAT);
 801       null_check_here = code_offset();
 802       __ vstr_f32(src->as_float_reg(), addr);
 803 #else
 804 #error "unimplemented"
 805 #endif
 806       break;
 807     }
 808 
 809     case T_DOUBLE: {
 810 #ifdef __ARM_PCS_VFP
 811       Address addr = as_Address(to_addr, Address::IDT_DOUBLE);
 812       null_check_here = code_offset();
 813       __ vstr_f64(src->as_double_reg(), addr);
 814 #else
 815 #error "unimplemented"
 816 #endif
 817 
 818       break;
 819     }
 820 
 821     case T_ARRAY:   // fall through
 822     case T_OBJECT:  // fall through
 823     case T_ADDRESS: // fall though
 824     case T_INT: {
 825       Address addr = as_Address(to_addr, Address::toInsnDataType(type));
 826       null_check_here = code_offset();
 827       __ str(src->as_register(), addr);
 828       break;
 829     }
 830     case T_METADATA:
 831       // We get here to store a method pointer to the stack to pass to
 832       // a dtrace runtime call. This can't work on 64 bit with
 833       // compressed klass ptrs: T_METADATA can be a compressed klass
 834       // ptr or a 64 bit method pointer.
 835       ShouldNotReachHere();
 836 //      __ str(src->as_register(), as_Address(to_addr));
 837       break;
 838 
 839     case T_LONG: {
 840       Address addr = as_Address_lo(to_addr, Address::IDT_LONG);
 841       null_check_here = code_offset();
 842       null_check_here += __ strd(src->as_register_lo(), src->as_register_hi(), addr);
 843       break;
 844     }
 845 
 846     case T_BYTE:    // fall through
 847     case T_BOOLEAN: {
 848       Address addr = as_Address(to_addr, Address::toInsnDataType(type));
 849       null_check_here = code_offset();
 850       __ strb(src->as_register(), addr);
 851       break;
 852     }
 853     case T_CHAR:    // fall through
 854     case T_SHORT: {
 855       Address addr = as_Address(to_addr, Address::toInsnDataType(type));
 856       null_check_here = code_offset();
 857       __ strh(src->as_register(), addr);
 858       break;
 859     }
 860     default:
 861       ShouldNotReachHere();
 862   }
 863 
 864   if (info != NULL) {
 865     add_debug_info_for_null_check(null_check_here, info);
 866   }
 867 }
 868 
 869 
 870 void LIR_Assembler::stack2reg(LIR_Opr src, LIR_Opr dest, BasicType type) {
 871   assert(src->is_stack(), "should not call otherwise");
 872   assert(dest->is_register(), "should not call otherwise");
 873 
 874   if (dest->is_single_cpu()) {
 875     if (type == T_ARRAY || type == T_OBJECT) {
 876       __ ldr(dest->as_register(), frame_map()->address_for_slot(src->single_stack_ix()));
 877       __ verify_oop(dest->as_register());
 878     } else {
 879       __ ldr(dest->as_register(), frame_map()->address_for_slot(src->single_stack_ix()));
 880     }
 881 
 882   } else if (dest->is_double_cpu()) {
 883     Address src_addr_LO = frame_map()->address_for_slot(src->double_stack_ix(), lo_word_offset_in_bytes);
 884     __ ldrd(dest->as_register_lo(), dest->as_register_hi(), src_addr_LO);
 885 
 886   } else if (dest->is_single_fpu()) {
 887 #ifdef __ARM_PCS_VFP
 888     Address src_addr = frame_map()->address_for_slot(src->single_stack_ix());
 889     __ vldr_f32(dest->as_float_reg(), src_addr.safe_for(Address::IDT_FLOAT, _masm, rscratch1));
 890 #else
 891 #error "unimplemented"
 892 #endif
 893   } else if (dest->is_double_fpu()) {
 894 #ifdef __ARM_PCS_VFP
 895     Address src_addr = frame_map()->address_for_slot(src->double_stack_ix());
 896     __ vldr_f64(dest->as_double_reg(), src_addr.safe_for(Address::IDT_DOUBLE, _masm, rscratch1));
 897 #else
 898 #error "unimplemented"
 899 #endif
 900   } else {
 901     ShouldNotReachHere();
 902   }
 903 }
 904 
 905 
 906 void LIR_Assembler::klass2reg_with_patching(Register reg, CodeEmitInfo* info) {
 907   Metadata* o = NULL;
 908   PatchingStub* patch = new PatchingStub(_masm, PatchingStub::load_klass_id);
 909   __ mov_metadata(reg, o);
 910   patching_epilog(patch, lir_patch_normal, reg, info);
 911 }
 912 
 913 void LIR_Assembler::stack2stack(LIR_Opr src, LIR_Opr dest, BasicType type) {
 914 
 915   LIR_Opr temp;
 916   if (type == T_LONG || type == T_DOUBLE)
 917     temp = FrameMap::rscratch_long_opr;
 918   else
 919     temp = FrameMap::rscratch1_opr;
 920 
 921   stack2reg(src, temp, src->type());
 922   reg2stack(temp, dest, dest->type(), false);
 923 }
 924 
 925 
 926 void LIR_Assembler::mem2reg(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_PatchCode patch_code, CodeEmitInfo* info, bool wide, bool /* unaligned */) {
 927   LIR_Address* from_addr = src->as_address_ptr();
 928 
 929   if (from_addr->base()->type() == T_OBJECT) {
 930     __ verify_oop(from_addr->base()->as_pointer_register());
 931   }
 932 
 933   PatchingStub* patch = NULL;
 934   if (patch_code != lir_patch_none) {
 935     assert(from_addr->disp() != 0, "must have");
 936 
 937     patch = new PatchingStub(_masm, PatchingStub::access_field_id);
 938     __ mov(rscratch1, (address) from_addr->disp());
 939     patching_epilog(patch, patch_code, from_addr->base()->as_register(), info);
 940 
 941     from_addr = new LIR_Address(from_addr->base(), FrameMap::rscratch1_opr, from_addr->type());
 942   }
 943 
 944   int null_check_here = code_offset();
 945 
 946   switch (type) {
 947     case T_FLOAT: {
 948 #ifdef __ARM_PCS_VFP
 949     Address addr = as_Address(from_addr, Address::IDT_FLOAT);
 950     null_check_here = code_offset();
 951     __ vldr_f32(dest->as_float_reg(), addr);
 952 #else
 953 #error "unimplemented"
 954 #endif
 955       break;
 956     }
 957 
 958     case T_DOUBLE: {
 959 #ifdef __ARM_PCS_VFP
 960     Address addr = as_Address(from_addr, Address::IDT_DOUBLE);
 961     null_check_here = code_offset();
 962     __ vldr_f64(dest->as_double_reg(), addr);
 963 #else
 964 #error "unimplemented"
 965 #endif
 966       break;
 967     }
 968 
 969     case T_ARRAY:   // fall through
 970     case T_OBJECT:  // fall through
 971     case T_ADDRESS: // fall through
 972     case T_INT: {
 973       Address addr = as_Address(from_addr, Address::toInsnDataType(type));
 974       null_check_here = code_offset();
 975       __ ldr(dest->as_register(), addr);
 976       break;
 977     }
 978     case T_METADATA:
 979       // We get here to store a method pointer to the stack to pass to
 980       // a dtrace runtime call. This can't work on 64 bit with
 981       // compressed klass ptrs: T_METADATA can be a compressed klass
 982       // ptr or a 64 bit method pointer.
 983       ShouldNotReachHere();
 984 //      __ ldr(dest->as_register(), as_Address(from_addr));
 985       break;
 986 
 987     case T_LONG: {
 988       Address addr = as_Address_lo(from_addr, Address::IDT_LONG);
 989       null_check_here = code_offset();
 990       null_check_here += __ ldrd(dest->as_register_lo(), dest->as_register_hi(), addr);
 991       break;
 992     }
 993 
 994     case T_BYTE: {
 995       Address addr =  as_Address(from_addr, Address::IDT_BYTE);
 996       null_check_here = code_offset();
 997       __ ldrsb(dest->as_register(), addr);
 998       break;
 999     }
1000     case T_BOOLEAN: {
1001       Address addr = as_Address(from_addr, Address::IDT_BOOLEAN);
1002       null_check_here = code_offset();
1003       __ ldrb(dest->as_register(), addr);
1004       break;
1005     }
1006 
1007     case T_CHAR: {
1008       Address addr = as_Address(from_addr, Address::IDT_CHAR);
1009       null_check_here = code_offset();
1010       __ ldrh(dest->as_register(), addr);
1011       break;
1012     }
1013     case T_SHORT: {
1014       Address addr = as_Address(from_addr, Address::IDT_SHORT);
1015       null_check_here = code_offset();
1016       __ ldrsh(dest->as_register(), addr);
1017       break;
1018     }
1019 
1020     default:
1021       ShouldNotReachHere();
1022   }
1023 
1024   if (type == T_ARRAY || type == T_OBJECT) {
1025     __ verify_oop(dest->as_register());
1026   }
1027 
1028   if (info != NULL) {
1029     add_debug_info_for_null_check(null_check_here, info);
1030   }
1031 }
1032 
1033 void LIR_Assembler::prefetchr(LIR_Opr src) {
1034   Unimplemented();
1035 }
1036 
1037 void LIR_Assembler::prefetchw(LIR_Opr src) {
1038   Unimplemented();
1039 }
1040 
1041 int LIR_Assembler::array_element_size(BasicType type) const {
1042   int elem_size = type2aelembytes(type);
1043   return exact_log2(elem_size);
1044 }
1045 
1046 void LIR_Assembler::emit_op3(LIR_Op3* op) {
1047   Register Rdividend = op->in_opr1()->as_register();
1048   Register Rdivisor  = op->in_opr2()->as_register();
1049   Register Rscratch  = op->in_opr3()->as_register();
1050   Register Rresult   = op->result_opr()->as_register();
1051   int divisor = -1;
1052 
1053   /*
1054   TODO: For some reason, using the Rscratch that gets passed in is
1055   not possible because the register allocator does not see the tmp reg
1056   as used, and assignes it the same register as Rdividend. We use rscratch1
1057    instead.
1058 
1059   assert(Rdividend != Rscratch, "");
1060   assert(Rdivisor  != Rscratch, "");
1061   */
1062 
1063   if (Rdivisor == noreg && is_power_of_2(divisor)) {
1064     // convert division by a power of two into some shifts and logical operations
1065   }
1066 
1067   assert(op->code() == lir_irem || op->code() == lir_idiv, "should be irem or idiv");
1068   bool want_remainder = op->code() == lir_irem;
1069 
1070   __ divide(Rresult, Rdividend, Rdivisor, 32, want_remainder);
1071 }
1072 
1073 void LIR_Assembler::emit_opBranch(LIR_OpBranch* op) {
1074 #ifdef ASSERT
1075   assert(op->block() == NULL || op->block()->label() == op->label(), "wrong label");
1076   if (op->block() != NULL)  _branch_target_blocks.append(op->block());
1077   if (op->ublock() != NULL) _branch_target_blocks.append(op->ublock());
1078 #endif
1079 
1080   if (op->cond() == lir_cond_always) {
1081     if (op->info() != NULL) add_debug_info_for_branch(op->info());
1082     __ b(*(op->label()));
1083   } else {
1084     Assembler::Condition acond;
1085     if (op->code() == lir_cond_float_branch) {
1086       bool is_unordered = (op->ublock() == op->block());
1087       // Assembler::EQ does not permit unordered branches, so we add
1088       // another branch here.  Likewise, Assembler::NE does not permit
1089       // ordered branches.
1090       if (is_unordered && op->cond() == lir_cond_equal
1091           || !is_unordered && op->cond() == lir_cond_notEqual)
1092         __ b(*(op->ublock()->label()), Assembler::VS);
1093       switch(op->cond()) {
1094       case lir_cond_equal:        acond = Assembler::EQ; break;
1095       case lir_cond_notEqual:     acond = Assembler::NE; break;
1096       case lir_cond_less:         acond = (is_unordered ? Assembler::LT : Assembler::LO); break;
1097       case lir_cond_lessEqual:    acond = (is_unordered ? Assembler::LE : Assembler::LS); break;
1098       case lir_cond_greaterEqual: acond = (is_unordered ? Assembler::HS : Assembler::GE); break;
1099       case lir_cond_greater:      acond = (is_unordered ? Assembler::HI : Assembler::GT); break;
1100       default:                    ShouldNotReachHere();
1101       }
1102     } else {
1103       switch (op->cond()) {
1104         case lir_cond_equal:        acond = Assembler::EQ; break;
1105         case lir_cond_notEqual:     acond = Assembler::NE; break;
1106         case lir_cond_less:         acond = Assembler::LT; break;
1107         case lir_cond_greaterEqual: acond = Assembler::GE; break;
1108         case lir_cond_lessEqual:    acond = Assembler::LE; break;
1109         case lir_cond_greater:      acond = Assembler::GT; break;
1110         case lir_cond_belowEqual:   acond = Assembler::LS; break;
1111         case lir_cond_aboveEqual:   acond = Assembler::HS; break;
1112         default:                         ShouldNotReachHere();
1113       }
1114       if (op->type() == T_LONG) {
1115         // a special trick here to be able to effectively compare jlongs
1116         // for the lessEqual and greater conditions the jlong operands are swapped
1117         // during comparison and hence should use mirror condition in conditional
1118         // instruction
1119         // see LIR_Assembler::comp_op and LIR_Assembler::cmove
1120         switch (op->cond()) {
1121           case lir_cond_lessEqual: acond = Assembler::GE;  break;
1122           case lir_cond_greater: acond = Assembler::LT;    break;
1123         }
1124       }
1125     }
1126     __ b(*(op->label()), acond);
1127   }
1128 }
1129 
1130 FloatRegister LIR_Assembler::as_float_reg(LIR_Opr doubleReg) {
1131     assert(doubleReg->is_double_fpu(), "must be f64");
1132     return as_FloatRegister(doubleReg->fpu_regnrLo());
1133 }
1134 
1135 void LIR_Assembler::emit_opConvert(LIR_OpConvert* op) {
1136   LIR_Opr src  = op->in_opr();
1137   LIR_Opr dest = op->result_opr();
1138 
1139   switch (op->bytecode()) {
1140     case Bytecodes::_i2f:
1141       {
1142         __ vmov_f32(dest->as_float_reg(), src->as_register());
1143         __ vcvt_f32_s32(dest->as_float_reg(), dest->as_float_reg());
1144         break;
1145       }
1146     case Bytecodes::_i2d:
1147       {
1148         __ vmov_f32(as_float_reg(dest), src->as_register());
1149         __ vcvt_f64_s32(dest->as_double_reg(), as_float_reg(dest));
1150         break;
1151       }
1152     case Bytecodes::_f2d:
1153       {
1154         __ vcvt_f64_f32(dest->as_double_reg(), src->as_float_reg());
1155         break;
1156       }
1157     case Bytecodes::_d2f:
1158       {
1159         __ vcvt_f32_f64(dest->as_float_reg(), src->as_double_reg());
1160         break;
1161       }
1162     case Bytecodes::_i2c:
1163       {
1164         __ uxth(dest->as_register(), src->as_register());
1165         break;
1166       }
1167     case Bytecodes::_i2l:
1168       {
1169         const Register dst_hi = dest->as_register_hi();
1170         const Register dst_lo = dest->as_register_lo();
1171         const Register src_lo = as_reg(src);
1172         __ mov(dst_lo, src_lo);
1173         __ asr(dst_hi, src_lo, 31);
1174         break;
1175       }
1176     case Bytecodes::_i2s:
1177       {
1178         __ sxth(dest->as_register(), src->as_register());
1179         break;
1180       }
1181     case Bytecodes::_i2b:
1182       {
1183         __ sxtb(dest->as_register(), src->as_register());
1184         break;
1185       }
1186     case Bytecodes::_l2i:
1187       {
1188         assert(dest->is_single_cpu(), "must be single register");
1189         __ mov(dest->as_register(), src->as_register_lo());
1190         break;
1191       }
1192     case Bytecodes::_f2i:
1193       {
1194         __ vcvt_s32_f32(src->as_float_reg(), src->as_float_reg());
1195         __ vmov_f32(dest->as_register(), src->as_float_reg());
1196         break;
1197       }
1198     case Bytecodes::_d2i:
1199       {
1200         __ vcvt_s32_f64(as_float_reg(src), src->as_double_reg());
1201         __ vmov_f32(dest->as_register(), as_float_reg(src));
1202         break;
1203       }
1204     default: ShouldNotReachHere();
1205   }
1206 }
1207 
1208 void LIR_Assembler::emit_alloc_obj(LIR_OpAllocObj* op) {
1209   if (op->init_check()) {
1210     __ ldrb(rscratch1, Address(op->klass()->as_register(),
1211                                InstanceKlass::init_state_offset()));
1212     __ cmp(rscratch1, InstanceKlass::fully_initialized);
1213     add_debug_info_for_null_check_here(op->stub()->info());
1214     __ b(*op->stub()->entry(), Assembler::NE);
1215   }
1216   __ allocate_object(op->obj()->as_register(),
1217                      op->tmp1()->as_register(),
1218                      op->tmp2()->as_register(),
1219                      op->header_size(),
1220                      op->object_size(),
1221                      op->klass()->as_register(),
1222                      *op->stub()->entry());
1223   __ bind(*op->stub()->continuation());
1224 }
1225 
1226 void LIR_Assembler::emit_alloc_array(LIR_OpAllocArray* op) {
1227   Register len =  as_reg(op->len());
1228 
1229   if (UseSlowPath ||
1230       (!UseFastNewObjectArray && (op->type() == T_OBJECT || op->type() == T_ARRAY)) ||
1231       (!UseFastNewTypeArray   && (op->type() != T_OBJECT && op->type() != T_ARRAY))) {
1232     __ b(*op->stub()->entry());
1233   } else {
1234     Register tmp1 = op->tmp1()->as_register();
1235     Register tmp2 = op->tmp2()->as_register();
1236     Register tmp3 = op->tmp3()->as_register();
1237     if (len == tmp1) {
1238       tmp1 = tmp3;
1239     } else if (len == tmp2) {
1240       tmp2 = tmp3;
1241     } else if (len == tmp3) {
1242       // everything is ok
1243     } else {
1244       __ mov(tmp3, len);
1245     }
1246     __ allocate_array(op->obj()->as_register(),
1247                       len,
1248                       tmp1,
1249                       tmp2,
1250                       arrayOopDesc::header_size(op->type()),
1251                       array_element_size(op->type()),
1252                       op->klass()->as_register(),
1253                       *op->stub()->entry());
1254   }
1255   __ bind(*op->stub()->continuation());
1256 }
1257 
1258 void LIR_Assembler::type_profile_helper(Register mdo,
1259                                         ciMethodData *md, ciProfileData *data,
1260                                         Register recv, Label* update_done) {
1261   for (uint i = 0; i < ReceiverTypeData::row_limit(); i++) {
1262     Label next_test;
1263     // See if the receiver is receiver[n].
1264     __ lea(rscratch2, Address(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_offset(i))));
1265     __ ldr(rscratch1, Address(rscratch2));
1266     __ cmp(recv, rscratch1);
1267     __ b(next_test, Assembler::NE);
1268     Address data_addr(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_count_offset(i)));
1269     __ addptr(data_addr, DataLayout::counter_increment);
1270     __ b(*update_done);
1271     __ bind(next_test);
1272   }
1273 
1274   // Didn't find receiver; find next empty slot and fill it in
1275   for (uint i = 0; i < ReceiverTypeData::row_limit(); i++) {
1276     Label next_test;
1277     __ lea(rscratch2,
1278            Address(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_offset(i))));
1279     Address recv_addr(rscratch2);
1280     __ ldr(rscratch1, recv_addr);
1281     __ cbnz(rscratch1, next_test);
1282     __ str(recv, recv_addr);
1283     __ mov(rscratch1, DataLayout::counter_increment);
1284     __ lea(rscratch2, Address(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_count_offset(i))));
1285     __ str(rscratch1, Address(rscratch2));
1286     __ b(*update_done);
1287     __ bind(next_test);
1288   }
1289 }
1290 
1291 void LIR_Assembler::emit_typecheck_helper(LIR_OpTypeCheck *op, Label* success, Label* failure, Label* obj_is_null) {
1292   // we always need a stub for the failure case.
1293   CodeStub* stub = op->stub();
1294   Register obj = op->object()->as_register();
1295   Register k_RInfo = op->tmp1()->as_register();
1296   Register klass_RInfo = op->tmp2()->as_register();
1297   Register dst = op->result_opr()->as_register();
1298   ciKlass* k = op->klass();
1299   Register Rtmp1 = noreg;
1300 
1301   // check if it needs to be profiled
1302   ciMethodData* md;
1303   ciProfileData* data;
1304 
1305   if (op->should_profile()) {
1306     ciMethod* method = op->profiled_method();
1307     assert(method != NULL, "Should have method");
1308     int bci = op->profiled_bci();
1309     md = method->method_data_or_null();
1310     assert(md != NULL, "Sanity");
1311     data = md->bci_to_data(bci);
1312     assert(data != NULL,                "need data for type check");
1313     assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
1314   }
1315   Label profile_cast_success, profile_cast_failure;
1316   Label *success_target = op->should_profile() ? &profile_cast_success : success;
1317   Label *failure_target = op->should_profile() ? &profile_cast_failure : failure;
1318 
1319   if (obj == k_RInfo) {
1320     k_RInfo = dst;
1321   } else if (obj == klass_RInfo) {
1322     klass_RInfo = dst;
1323   }
1324   if (k->is_loaded()) {
1325     select_different_registers(obj, dst, k_RInfo, klass_RInfo);
1326   } else {
1327     Rtmp1 = op->tmp3()->as_register();
1328     select_different_registers(obj, dst, k_RInfo, klass_RInfo, Rtmp1);
1329   }
1330 
1331   assert_different_registers(obj, k_RInfo, klass_RInfo);
1332 
1333     if (op->should_profile()) {
1334       Label not_null;
1335       __ cbnz(obj, not_null);
1336       // Object is null; update MDO and exit
1337       Register mdo  = klass_RInfo;
1338       __ mov_metadata(mdo, md->constant_encoding());
1339       Address data_addr
1340         = __ form_address(rscratch2, mdo,
1341                           md->byte_offset_of_slot(data, DataLayout::DataLayout::header_offset()),
1342                           LogBytesPerWord);
1343       int header_bits = DataLayout::flag_mask_to_header_mask(BitData::null_seen_byte_constant());
1344       __ ldr(rscratch1, data_addr);
1345       __ orr(rscratch1, rscratch1, header_bits);
1346       __ str(rscratch1, data_addr);
1347       __ b(*obj_is_null);
1348       __ bind(not_null);
1349     } else {
1350       __ cbz(obj, *obj_is_null);
1351     }
1352 
1353   if (!k->is_loaded()) {
1354     klass2reg_with_patching(k_RInfo, op->info_for_patch());
1355   } else {
1356     __ mov_metadata(k_RInfo, k->constant_encoding());
1357   }
1358   __ verify_oop(obj);
1359 
1360   if (op->fast_check()) {
1361     // get object class
1362     // not a safepoint as obj null check happens earlier
1363     __ load_klass(rscratch1, obj);
1364     __ cmp( rscratch1, k_RInfo);
1365 
1366     __ b(*failure_target, Assembler::NE);
1367     // successful cast, fall through to profile or jump
1368   } else {
1369     // get object class
1370     // not a safepoint as obj null check happens earlier
1371     __ load_klass(klass_RInfo, obj);
1372     if (k->is_loaded()) {
1373       // See if we get an immediate positive hit
1374       __ ldr(rscratch1, Address(klass_RInfo, long(k->super_check_offset())));
1375       __ cmp(k_RInfo, rscratch1);
1376       if ((juint)in_bytes(Klass::secondary_super_cache_offset()) != k->super_check_offset()) {
1377         __ b(*failure_target, Assembler::NE);
1378         // successful cast, fall through to profile or jump
1379       } else {
1380         // See if we get an immediate positive hit
1381         __ b(*success_target, Assembler::EQ);
1382         // check for self
1383         __ cmp(klass_RInfo, k_RInfo);
1384         __ b(*success_target, Assembler::EQ);
1385 
1386         __ push(klass_RInfo);
1387         __ push(k_RInfo);
1388         __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
1389         __ ldr(klass_RInfo, Address(__ post(sp, 2 * wordSize)));
1390 
1391         // result is a boolean
1392         __ cbz(klass_RInfo, *failure_target);
1393         // successful cast, fall through to profile or jump
1394       }
1395     } else {
1396       // perform the fast part of the checking logic
1397       __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, NULL);
1398       // call out-of-line instance of __ check_klass_subtype_slow_path(...):
1399       __ push(klass_RInfo);
1400       __ push(k_RInfo);
1401       __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
1402       __ ldr(k_RInfo, Address(__ post(sp, 2 * wordSize)));
1403 
1404       // result is a boolean
1405       __ cbz(k_RInfo, *failure_target);
1406       // successful cast, fall through to profile or jump
1407     }
1408   }
1409   if (op->should_profile()) {
1410     Register mdo  = klass_RInfo, recv = k_RInfo;
1411     __ bind(profile_cast_success);
1412     __ mov_metadata(mdo, md->constant_encoding());
1413     __ load_klass(recv, obj);
1414     Label update_done;
1415     type_profile_helper(mdo, md, data, recv, success);
1416     __ b(*success);
1417 
1418     __ bind(profile_cast_failure);
1419     __ mov_metadata(mdo, md->constant_encoding());
1420     Address counter_addr
1421       = __ form_address(rscratch2, mdo,
1422                         md->byte_offset_of_slot(data, CounterData::count_offset()),
1423                         LogBytesPerWord);
1424     __ ldr(rscratch1, counter_addr);
1425     __ sub(rscratch1, rscratch1, DataLayout::counter_increment);
1426     __ str(rscratch1, counter_addr);
1427     __ b(*failure);
1428   }
1429   __ b(*success);
1430 }
1431 
1432 
1433 void LIR_Assembler::emit_opTypeCheck(LIR_OpTypeCheck* op) {
1434   LIR_Code code = op->code();
1435   if (code == lir_store_check) {
1436     Register value = op->object()->as_register();
1437     Register array = op->array()->as_register();
1438     Register k_RInfo = op->tmp1()->as_register();
1439     Register klass_RInfo = op->tmp2()->as_register();
1440     Register Rtmp1 = op->tmp3()->as_register();
1441 
1442     CodeStub* stub = op->stub();
1443 
1444     // check if it needs to be profiled
1445     ciMethodData* md;
1446     ciProfileData* data;
1447 
1448     if (op->should_profile()) {
1449       ciMethod* method = op->profiled_method();
1450       assert(method != NULL, "Should have method");
1451       int bci = op->profiled_bci();
1452       md = method->method_data_or_null();
1453       assert(md != NULL, "Sanity");
1454       data = md->bci_to_data(bci);
1455       assert(data != NULL,                "need data for type check");
1456       assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
1457     }
1458     Label profile_cast_success, profile_cast_failure, done;
1459     Label *success_target = op->should_profile() ? &profile_cast_success : &done;
1460     Label *failure_target = op->should_profile() ? &profile_cast_failure : stub->entry();
1461 
1462     if (op->should_profile()) {
1463       Label not_null;
1464       __ cbnz(value, not_null);
1465       // Object is null; update MDO and exit
1466       Register mdo  = klass_RInfo;
1467       __ mov_metadata(mdo, md->constant_encoding());
1468       Address data_addr
1469         = __ form_address(rscratch2, mdo,
1470                           md->byte_offset_of_slot(data, DataLayout::header_offset()),
1471                           LogBytesPerInt);
1472       int header_bits = DataLayout::flag_mask_to_header_mask(BitData::null_seen_byte_constant());
1473       __ ldr(rscratch1, data_addr);
1474       __ orr(rscratch1, rscratch1, header_bits);
1475       __ str(rscratch1, data_addr);
1476       __ b(done);
1477       __ bind(not_null);
1478     } else {
1479       __ cbz(value, done);
1480     }
1481 
1482     add_debug_info_for_null_check_here(op->info_for_exception());
1483     __ load_klass(k_RInfo, array);
1484     __ load_klass(klass_RInfo, value);
1485 
1486     // get instance klass (it's already uncompressed)
1487     __ ldr(k_RInfo, Address(k_RInfo, ObjArrayKlass::element_klass_offset()));
1488     // perform the fast part of the checking logic
1489     __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, NULL);
1490     // call out-of-line instance of __ check_klass_subtype_slow_path(...):
1491     __ push(klass_RInfo);
1492     __ push(k_RInfo);
1493     __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
1494     __ ldr(k_RInfo, Address(__ post(sp, 2 * wordSize)));
1495     // result is a boolean
1496     __ cbz(k_RInfo, *failure_target);
1497     // fall through to the success case
1498 
1499     if (op->should_profile()) {
1500       Register mdo  = klass_RInfo, recv = k_RInfo;
1501       __ bind(profile_cast_success);
1502       __ mov_metadata(mdo, md->constant_encoding());
1503       __ load_klass(recv, value);
1504       type_profile_helper(mdo, md, data, recv, &done);
1505       __ b(done);
1506 
1507       __ bind(profile_cast_failure);
1508       __ mov_metadata(mdo, md->constant_encoding());
1509       Address counter_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
1510       __ lea(rscratch2, counter_addr);
1511       __ ldr(rscratch1, Address(rscratch2));
1512       __ sub(rscratch1, rscratch1, DataLayout::counter_increment);
1513       __ str(rscratch1, Address(rscratch2));
1514       __ b(*stub->entry());
1515     }
1516 
1517     __ bind(done);
1518   } else if (code == lir_checkcast) {
1519     Register obj = op->object()->as_register();
1520     Register dst = op->result_opr()->as_register();
1521     Label success;
1522     emit_typecheck_helper(op, &success, op->stub()->entry(), &success);
1523     __ bind(success);
1524     if (dst != obj) {
1525       __ mov(dst, obj);
1526     }
1527   } else if (code == lir_instanceof) {
1528     Register obj = op->object()->as_register();
1529     Register dst = op->result_opr()->as_register();
1530     Label success, failure, done;
1531     emit_typecheck_helper(op, &success, &failure, &failure);
1532     __ bind(failure);
1533     __ mov(dst, 0);
1534     __ b(done);
1535     __ bind(success);
1536     __ mov(dst, 1);
1537     __ bind(done);
1538   } else {
1539     ShouldNotReachHere();
1540   }
1541 }
1542 
1543 // TODO: reuse masm cmpxchgw
1544 void LIR_Assembler::casw(Register addr, Register newval, Register cmpval, Register result) {
1545   assert(newval != cmpval, "must be different");
1546   Label retry_load, nope;
1547   // flush and load exclusive from the memory location
1548   // and fail if it is not what we expect
1549   __ bind(retry_load);
1550   __ ldrex(result, addr);
1551   __ cmp(result, cmpval);
1552   __ mov(result, 1, Assembler::NE);
1553   __ b(nope, Assembler::NE);
1554   // if we store+flush with no intervening write rscratch1 wil be zero
1555   __ strex(result, newval, addr);
1556   // retry so we only ever return after a load fails to compare
1557   // ensures we don't return a stale value after a failed write.
1558   __ cbnz(result, retry_load);
1559   __ membar(__ AnyAny);
1560   __ bind(nope);
1561 }
1562 
1563 void LIR_Assembler::casl(Register addr, Register newval_lo, Register newval_hi, Register cmpval_lo,  Register cmpval_hi,  Register tmp_lo, Register tmp_hi, Register result) {
1564   assert(newval_lo->successor() == newval_hi, "must be contiguous");
1565   assert(tmp_lo->successor() == tmp_hi, "must be contiguous");
1566   assert(tmp_lo->encoding_nocheck() % 2 == 0,  "Must be an even register");
1567   assert_different_registers(newval_lo, newval_hi, cmpval_lo,  cmpval_hi,  tmp_lo, tmp_hi);
1568 
1569   Label retry_load, nope;
1570   // flush and load exclusive from the memory location
1571   // and fail if it is not what we expect
1572   __ bind(retry_load);
1573   __ mov(result, 1);
1574   __ ldrexd(tmp_lo, addr);
1575   __ cmp(tmp_lo, cmpval_lo);
1576   __ b(nope, Assembler::NE);
1577   __ cmp(tmp_hi, cmpval_hi);
1578   __ b(nope, Assembler::NE);
1579   // if we store+flush with no intervening write rscratch1 wil be zero
1580   __ strexd(result, newval_lo, addr);
1581   // retry so we only ever return after a load fails to compare
1582   // ensures we don't return a stale value after a failed write.
1583   __ cbnz(result, retry_load);
1584   __ membar(__ AnyAny);
1585   __ bind(nope);
1586 }
1587 
1588 
1589 void LIR_Assembler::emit_compare_and_swap(LIR_OpCompareAndSwap* op) {
1590   Register addr = as_reg(op->addr());
1591   Register result = as_reg(op->result_opr());
1592   if (op->code() == lir_cas_obj || op->code() == lir_cas_int) {
1593     Register newval = as_reg(op->new_value());
1594     Register cmpval = as_reg(op->cmp_value());
1595     casw(addr, newval, cmpval, result);
1596   } else if (op->code() == lir_cas_long){
1597     Register newval_lo = op->new_value()->as_register_lo();
1598     Register newval_hi = op->new_value()->as_register_hi();
1599     Register cmpval_lo = op->cmp_value()->as_register_lo();
1600     Register cmpval_hi = op->cmp_value()->as_register_hi();
1601     Register tmp_lo = op->tmp1()->as_register_lo();
1602     Register tmp_hi = op->tmp1()->as_register_hi();
1603     casl(addr, newval_lo, newval_hi, cmpval_lo, cmpval_hi, tmp_lo, tmp_hi, result);
1604   } else {
1605       ShouldNotReachHere();
1606   }
1607 }
1608 
1609 static void patch_condition(address start_insn, address end_insn, Assembler::Condition cond) {
1610   for (uint32_t* insn_p = (uint32_t*) start_insn; (address) insn_p < end_insn; ++insn_p) {
1611     uint32_t insn = *insn_p;
1612     assert((insn >> 28) == Assembler::AL, "instructions in patch"
1613      " should allow conditional form and be in ALWAYS condition");
1614     *insn_p = (insn & 0x0fffffff) | (cond << 28);
1615   }
1616 }
1617 
1618 void LIR_Assembler::cmove(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Opr result, BasicType type) {
1619 
1620   Assembler::Condition acond, ncond;
1621   switch (condition) {
1622   case lir_cond_equal:        acond = Assembler::EQ; ncond = Assembler::NE; break;
1623   case lir_cond_notEqual:     acond = Assembler::NE; ncond = Assembler::EQ; break;
1624   case lir_cond_less:         acond = Assembler::LT; ncond = Assembler::GE; break;
1625   case lir_cond_greaterEqual: acond = Assembler::GE; ncond = Assembler::LT; break;
1626   case lir_cond_lessEqual:    acond = Assembler::LE; ncond = Assembler::GT; break;
1627   case lir_cond_greater:      acond = Assembler::GT; ncond = Assembler::LE; break;
1628   case lir_cond_belowEqual:   Unimplemented(); break;
1629   case lir_cond_aboveEqual:   Unimplemented(); break;
1630   default:                    ShouldNotReachHere();
1631   }
1632   if (type == T_LONG) {
1633       // for the lessEqual and greater conditions the jlong operands are swapped
1634       // during comparison and hence should use mirror condition in conditional
1635       // instruction. see comp_op())
1636     switch (condition) {
1637     case lir_cond_lessEqual:    acond = Assembler::GE; ncond = Assembler::LT; break;
1638     case lir_cond_greater:      acond = Assembler::LT; ncond = Assembler::GE; break;
1639     }
1640   }
1641 
1642   address true_instrs = __ pc();
1643   if (opr1->is_cpu_register()) {
1644     reg2reg(opr1, result);
1645   } else if (opr1->is_stack()) {
1646     stack2reg(opr1, result, result->type());
1647   } else if (opr1->is_constant()) {
1648     const2reg(opr1, result, lir_patch_none, NULL);
1649   } else {
1650     ShouldNotReachHere();
1651   }
1652   patch_condition(true_instrs, __ pc(), acond);
1653 
1654   address false_instrs = __ pc();
1655   if (opr2->is_cpu_register()) {
1656     reg2reg(opr2, result);
1657   } else if (opr2->is_stack()) {
1658     stack2reg(opr2, result, result->type());
1659   } else if (opr2->is_constant()) {
1660     const2reg(opr2, result, lir_patch_none, NULL);
1661   } else {
1662     ShouldNotReachHere();
1663   }
1664   patch_condition(false_instrs, __ pc(), ncond);
1665 }
1666 
1667 void LIR_Assembler::arith_op(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dest, CodeEmitInfo* info, bool pop_fpu_stack) {
1668   assert(info == NULL, "should never be used, idiv/irem and ldiv/lrem not handled by this method");
1669 
1670   if (left->is_single_cpu()) {
1671     Register lreg = left->as_register();
1672     Register dreg = as_reg(dest);
1673 
1674     if (right->is_single_cpu()) {
1675       // cpu register - cpu register
1676 
1677       assert((left->type() == T_INT || left->type() == T_OBJECT)
1678              && right->type() == T_INT
1679              && dest->type() == T_INT,
1680              "should be");
1681       Register rreg = right->as_register();
1682       switch (code) {
1683       case lir_add: __ add (dest->as_register(), lreg, rreg); break;
1684       case lir_sub: __ sub (dest->as_register(), lreg, rreg); break;
1685       case lir_mul: __ mul (dest->as_register(), lreg, rreg); break;
1686       default:      ShouldNotReachHere();
1687       }
1688 
1689     } else if (right->is_double_cpu()) {
1690       ShouldNotReachHere(); // for obj+long op the generator casts long to int before invoking add
1691     } else if (right->is_constant()) {
1692       // cpu register - constant
1693       jint c = right->as_constant_ptr()->as_jint();
1694 
1695       assert(code == lir_add || code == lir_sub, "mismatched arithmetic op");
1696       if (c == 0 && dreg == lreg) {
1697         COMMENT("effective nop elided");
1698         return;
1699       }
1700 
1701       if (Assembler::operand_valid_for_add_sub_immediate(c)) {
1702         switch (code) {
1703         case lir_add: __ add(dreg, lreg, c); break;
1704         case lir_sub: __ sub(dreg, lreg, c); break;
1705         default: ShouldNotReachHere();
1706         }
1707       } else {
1708         __ mov(rscratch1, c);
1709         switch (code) {
1710         case lir_add: __ add(dreg, lreg, rscratch1); break;
1711         case lir_sub: __ sub(dreg, lreg, rscratch1); break;
1712         default: ShouldNotReachHere();
1713         }
1714       }
1715     } else {
1716       ShouldNotReachHere();
1717     }
1718 
1719   } else if (left->is_double_cpu()) {
1720     Register lreg_lo = left->as_register_lo();
1721     Register lreg_hi = left->as_register_hi();
1722 
1723     if (right->is_double_cpu()) {
1724       // cpu register - cpu register
1725       Register rreg_lo = right->as_register_lo();
1726       Register rreg_hi = right->as_register_hi();
1727       Register dreg_lo = dest->as_register_lo();
1728       Register dreg_hi = dest->as_register_hi();
1729       if (code == lir_add || code == lir_sub) {
1730         check_register_collision(dreg_lo, &lreg_hi, &rreg_hi);
1731       }
1732       switch (code) {
1733       case lir_add: __ adds (dreg_lo, lreg_lo, rreg_lo);
1734                     __ adc (dreg_hi, lreg_hi, rreg_hi); break;
1735       case lir_sub: __ subs (dreg_lo, lreg_lo, rreg_lo);
1736                     __ sbc (dreg_hi, lreg_hi, rreg_hi); break;
1737       case lir_mul: __ mult_long (dreg_lo, dreg_hi,
1738                         lreg_lo, lreg_hi, rreg_lo, rreg_hi); break;
1739       default:
1740         ShouldNotReachHere();
1741       }
1742 
1743     } else if (right->is_constant()) {
1744       const jint c_lo = right->as_constant_ptr()->as_jint_lo_bits();
1745       const jint c_hi = right->as_constant_ptr()->as_jint_hi_bits();
1746       const Register dreg_lo = dest->as_register_lo();
1747       const Register dreg_hi = dest->as_register_hi();
1748       assert(code == lir_add || code == lir_sub, "mismatched arithmetic op");
1749       if (c_lo == 0 && c_hi == 0 && dreg_lo == lreg_lo && dreg_hi == lreg_hi) {
1750         COMMENT("effective nop elided");
1751         return;
1752       }
1753       check_register_collision(dreg_lo, &lreg_hi, NULL, rscratch2);
1754       switch (code) {
1755         case lir_add:
1756           if (Assembler::operand_valid_for_add_sub_immediate(c_lo))
1757             __ adds(dreg_lo, lreg_lo, c_lo);
1758           else {
1759             __ mov(rscratch1, c_lo);
1760             __ adds(dreg_lo, lreg_lo, rscratch1);
1761           }
1762           if (Assembler::operand_valid_for_add_sub_immediate(c_hi))
1763             __ adc(dreg_hi, lreg_hi, c_hi);
1764           else {
1765             __ mov(rscratch1, c_hi);
1766             __ adc(dreg_lo, lreg_hi, rscratch1);
1767           }
1768           break;
1769         case lir_sub:
1770           if (Assembler::operand_valid_for_add_sub_immediate(c_lo))
1771             __ subs(dreg_lo, lreg_lo, c_lo);
1772           else {
1773             __ mov(rscratch1, c_lo);
1774             __ subs(dreg_lo, lreg_lo, rscratch1);
1775           }
1776           if (Assembler::operand_valid_for_add_sub_immediate(c_hi))
1777             __ sbc(dreg_hi, lreg_hi, c_hi);
1778           else {
1779             __ mov(rscratch1, c_hi);
1780             __ sbc(dreg_hi, lreg_hi, rscratch1);
1781           }
1782           break;
1783         default:
1784           ShouldNotReachHere();
1785       }
1786     } else {
1787       ShouldNotReachHere();
1788     }
1789   } else if (left->is_single_fpu()) {
1790     assert(right->is_single_fpu(), "right hand side of float arithmetics needs to be float register");
1791     switch (code) {
1792     case lir_add: __ vadd_f32 (dest->as_float_reg(), left->as_float_reg(), right->as_float_reg()); break;
1793     case lir_sub: __ vsub_f32 (dest->as_float_reg(), left->as_float_reg(), right->as_float_reg()); break;
1794     case lir_mul: __ vmul_f32 (dest->as_float_reg(), left->as_float_reg(), right->as_float_reg()); break;
1795     case lir_div: __ vdiv_f32 (dest->as_float_reg(), left->as_float_reg(), right->as_float_reg()); break;
1796     default:
1797       ShouldNotReachHere();
1798     }
1799   } else if (left->is_double_fpu()) {
1800     if (right->is_double_fpu()) {
1801       // cpu register - cpu register
1802       switch (code) {
1803       case lir_add: __ vadd_f64 (dest->as_double_reg(), left->as_double_reg(), right->as_double_reg()); break;
1804       case lir_sub: __ vsub_f64 (dest->as_double_reg(), left->as_double_reg(), right->as_double_reg()); break;
1805       case lir_mul: __ vmul_f64 (dest->as_double_reg(), left->as_double_reg(), right->as_double_reg()); break;
1806       case lir_div: __ vdiv_f64 (dest->as_double_reg(), left->as_double_reg(), right->as_double_reg()); break;
1807       default:
1808         ShouldNotReachHere();
1809       }
1810     } else {
1811       if (right->is_constant()) {
1812         ShouldNotReachHere();
1813       }
1814       ShouldNotReachHere();
1815     }
1816   } else if (left->is_single_stack() || left->is_address()) {
1817     assert(left == dest, "left and dest must be equal");
1818     ShouldNotReachHere();
1819   } else {
1820     ShouldNotReachHere();
1821   }
1822 }
1823 
1824 void LIR_Assembler::intrinsic_op(LIR_Code code, LIR_Opr value, LIR_Opr unused, LIR_Opr dest, LIR_Op* op) {
1825   switch(code) {
1826   case lir_abs : __ vabs_f64(dest->as_double_reg(), value->as_double_reg()); break;
1827   case lir_sqrt: __ vsqrt_f64(dest->as_double_reg(), value->as_double_reg()); break;
1828   default      : ShouldNotReachHere();
1829   }
1830 }
1831 
1832 void LIR_Assembler::logic_op(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dst) {
1833 
1834   assert(left->is_single_cpu() || left->is_double_cpu(), "expect single or double register");
1835   Register Rleft = left->is_single_cpu() ? left->as_register() :
1836                                            left->as_register_lo();
1837    if (dst->is_single_cpu()) {
1838      Register Rdst = dst->as_register();
1839      if (right->is_constant()) {
1840        switch (code) {
1841          case lir_logic_and: __ andr (Rdst, Rleft, right->as_jint()); break;
1842          case lir_logic_or:  __ orr (Rdst, Rleft, right->as_jint()); break;
1843          case lir_logic_xor: __ eor (Rdst, Rleft, right->as_jint()); break;
1844          default: ShouldNotReachHere(); break;
1845        }
1846      } else {
1847        Register Rright = right->is_single_cpu() ? right->as_register() :
1848                                                   right->as_register_lo();
1849        switch (code) {
1850          case lir_logic_and: __ andr (Rdst, Rleft, Rright); break;
1851          case lir_logic_or:  __ orr (Rdst, Rleft, Rright); break;
1852          case lir_logic_xor: __ eor (Rdst, Rleft, Rright); break;
1853          default: ShouldNotReachHere(); break;
1854        }
1855      }
1856    } else {
1857      assert(dst->is_double_cpu(), "mismatched logic op operand size");
1858      const Register Rdst_lo = dst->as_register_lo();
1859      const Register Rdst_hi = dst->as_register_hi();
1860      Register Rleft_hi = left->as_register_hi();
1861      if (right->is_constant()) {
1862        // LIR generator enforces jlong constants to be valid_immediate12
1863        // so we know they fit into 32-bit int
1864        switch (code) {
1865          case lir_logic_and: __ andr (Rdst_lo, Rleft, (int)right->as_jlong()); break;
1866          case lir_logic_or:  __ orr (Rdst_lo, Rleft, (int)right->as_jlong()); break;
1867          case lir_logic_xor: __ eor (Rdst_lo, Rleft, (int)right->as_jlong()); break;
1868          default: ShouldNotReachHere(); break;
1869        }
1870      } else {
1871        assert(right->is_double_cpu(), "mismatched logic op operand size");
1872        Register Rright_lo = right->as_register_lo();
1873        Register Rright_hi = right->as_register_hi();
1874        check_register_collision(Rdst_lo, &Rleft_hi, &Rright_hi);
1875        switch (code) {
1876          case lir_logic_and: __ andr (Rdst_lo, Rleft, Rright_lo);
1877                              __ andr (Rdst_hi, Rleft_hi, Rright_hi); break;
1878          case lir_logic_or:  __ orr (Rdst_lo, Rleft, Rright_lo);
1879                              __ orr (Rdst_hi, Rleft_hi, Rright_hi); break;
1880          case lir_logic_xor: __ eor (Rdst_lo, Rleft, Rright_lo);
1881                              __ eor (Rdst_hi, Rleft_hi, Rright_hi); break;
1882          default: ShouldNotReachHere(); break;
1883        }
1884      }
1885    }
1886 }
1887 
1888 
1889 
1890 void LIR_Assembler::arithmetic_idiv(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr temp, LIR_Opr result, CodeEmitInfo* info) { Unimplemented(); }
1891 
1892 void LIR_Assembler::comp_op(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Op2* op) {
1893   if (opr1->is_single_cpu()) {
1894     Register reg1 = as_reg(opr1);
1895     if (opr2->is_single_cpu()) {
1896       // cpu register - cpu register
1897       Register reg2 = opr2->as_register();
1898       __ cmp(reg1, reg2);
1899     } else if (opr2->is_constant()) {
1900       LIR_Const* c = opr2->as_constant_ptr();
1901       if (c->type() == T_INT) {
1902         __ cmp(reg1, c->as_jint(), rscratch1, Assembler::C_DFLT);
1903       } else if (c->type() == T_OBJECT || c->type() == T_ARRAY) {
1904         jobject o = c->as_jobject();
1905         if (o == NULL) {
1906           __ cmp(reg1, (int32_t)NULL_WORD);
1907         } else {
1908           __ movoop(rscratch1, o);
1909           __ cmpptr(reg1, rscratch1);
1910         }
1911       } else {
1912         fatal(err_msg("unexpected type: %s", basictype_to_str(c->type())));
1913       }
1914     } else if (opr2->is_address()) {
1915       __ ldr(rscratch2, as_Address(opr2->as_address_ptr(), rscratch1, Address::IDT_INT));
1916       __ cmp(reg1, rscratch2);
1917     } else {
1918       ShouldNotReachHere();
1919     }
1920 
1921   } else if (opr1->is_double_cpu()) {
1922     assert(opr1->type() == T_LONG, "expect jlong type");
1923     assert(opr2->type() == T_LONG, "expect jlong type");
1924     Register xlo = opr1->as_register_lo();
1925     Register xhi = opr1->as_register_hi();
1926     if (opr2->is_double_cpu()) {
1927       // cpu register - cpu register
1928       Register ylo = opr2->as_register_lo();
1929       Register yhi = opr2->as_register_hi();
1930       switch (condition) {
1931         case lir_cond_equal:
1932         case lir_cond_notEqual:
1933         case lir_cond_belowEqual:
1934         case lir_cond_aboveEqual:
1935           // these need APSR.ZC. the ops below set them correctly (but not APSR.V)
1936           __ cmp(xhi, yhi);
1937           __ cmp(xlo, ylo, Assembler::EQ);
1938           break;
1939         case lir_cond_less:
1940         case lir_cond_greaterEqual:
1941           __ cmp(xlo, ylo);
1942           __ sbcs(rscratch1, xhi, yhi);
1943           break;
1944         case lir_cond_lessEqual:
1945         case lir_cond_greater:
1946           // here goes a trick: the below operations do not produce the valid
1947           // value for the APSR.Z flag and there is no easy way to set it. so
1948           // we exchange the order of arguments in the comparison and use the
1949           // opposite condition in the conditional statement that follows.
1950           // GE should be used instead of LE and LT in place of GT.
1951           // the comp_op() could only be followed by: emit_opBranch(), cmove() and
1952           // emit_assert(). these are patched to be aware of this trick
1953           __ cmp(ylo, xlo);
1954           __ sbcs(rscratch1, yhi, xhi);
1955           break;
1956       }
1957     } else if (opr2->is_constant()) {
1958       jlong y = opr2->as_jlong();
1959       assert(Assembler::operand_valid_for_add_sub_immediate(y), "immediate overflow");
1960       switch (condition) {
1961         case lir_cond_equal:
1962         case lir_cond_notEqual:
1963         case lir_cond_belowEqual:
1964         case lir_cond_aboveEqual:
1965           __ cmp(xhi, (int)(y >> 32));
1966           __ cmp(xlo, (int)y, Assembler::EQ);
1967           break;
1968         case lir_cond_less:
1969         case lir_cond_greaterEqual:
1970           __ cmp(xlo, (int)y);
1971           __ sbcs(rscratch1, xhi, (int)(y >> 32));
1972           break;
1973         case lir_cond_lessEqual:
1974         case lir_cond_greater:
1975           __ rsbs(rscratch1, xlo, (int)y);
1976           __ rscs(rscratch1, xhi, (int)(y >> 32));
1977           break;
1978       }
1979     } else {
1980       ShouldNotReachHere();
1981     }
1982   } else if (opr1->is_single_fpu()) {
1983     FloatRegister reg1 = opr1->as_float_reg();
1984     assert(opr2->is_single_fpu(), "expect single float register");
1985     FloatRegister reg2 = opr2->as_float_reg();
1986     __ vcmp_f32(reg1, reg2);
1987     __ get_fpsr();
1988   } else if (opr1->is_double_fpu()) {
1989     FloatRegister reg1 = opr1->as_double_reg();
1990     assert(opr2->is_double_fpu(), "expect double float register");
1991     FloatRegister reg2 = opr2->as_double_reg();
1992     __ vcmp_f64(reg1, reg2);
1993     __ get_fpsr();
1994   } else {
1995     ShouldNotReachHere();
1996   }
1997 }
1998 
1999 void LIR_Assembler::comp_fl2i(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dst, LIR_Op2* op){
2000   if (code == lir_cmp_fd2i || code == lir_ucmp_fd2i) {
2001     bool is_unordered_less = (code == lir_ucmp_fd2i);
2002     if (left->is_single_fpu()) {
2003       __ float_cmp(true, is_unordered_less ? -1 : 1, left->as_float_reg(), right->as_float_reg(), dst->as_register());
2004     } else if (left->is_double_fpu()) {
2005       __ float_cmp(false, is_unordered_less ? -1 : 1, left->as_double_reg(), right->as_double_reg(), dst->as_register());
2006     } else {
2007       ShouldNotReachHere();
2008     }
2009   } else if (code == lir_cmp_l2i) {
2010     __ mov(dst->as_register(), 1);
2011     __ subs(rscratch1, left->as_register_lo(), right->as_register_lo());
2012     __ sbc(rscratch2, left->as_register_hi(), right->as_register_hi());
2013     __ orrs(rscratch1, rscratch1, rscratch2);
2014     __ mov(dst->as_register(), -1, Assembler::MI);
2015     __ mov(dst->as_register(), 0, Assembler::EQ);
2016   } else {
2017     ShouldNotReachHere();
2018   }
2019 }
2020 
2021 
2022 void LIR_Assembler::align_call(LIR_Code code) {  }
2023 
2024 
2025 void LIR_Assembler::call(LIR_OpJavaCall* op, relocInfo::relocType rtype) {
2026   __ trampoline_call(Address(op->addr(), rtype));
2027   add_call_info(code_offset(), op->info());
2028 }
2029 
2030 
2031 void LIR_Assembler::ic_call(LIR_OpJavaCall* op) {
2032   __ ic_call(op->addr());
2033   add_call_info(code_offset(), op->info());
2034 }
2035 
2036 
2037 /* Currently, vtable-dispatch is only enabled for sparc platforms */
2038 void LIR_Assembler::vtable_call(LIR_OpJavaCall* op) {
2039   ShouldNotReachHere();
2040 }
2041 
2042 
2043 void LIR_Assembler::emit_static_call_stub() {
2044   address call_pc = __ pc();
2045   address stub = __ start_a_stub(call_stub_size);
2046   if (stub == NULL) {
2047     bailout("static call stub overflow");
2048     return;
2049   }
2050 
2051   int start = __ offset();
2052 
2053   __ relocate(static_stub_Relocation::spec(call_pc));
2054   __ mov_metadata(rmethod, (Metadata*)NULL);
2055   __ movptr(rscratch1, 0);
2056   __ b(rscratch1);
2057 
2058   assert(__ offset() - start <= call_stub_size, "stub too big");
2059   __ end_a_stub();
2060 }
2061 
2062 
2063 void LIR_Assembler::throw_op(LIR_Opr exceptionPC, LIR_Opr exceptionOop, CodeEmitInfo* info) {
2064   assert(exceptionOop->as_register() == r0, "must match");
2065   assert(exceptionPC->as_register() == r3, "must match");
2066 
2067   // exception object is not added to oop map by LinearScan
2068   // (LinearScan assumes that no oops are in fixed registers)
2069   info->add_register_oop(exceptionOop);
2070   Runtime1::StubID unwind_id;
2071 
2072   // get current pc information
2073   // pc is only needed if the method has an exception handler, the unwind code does not need it.
2074   int pc_for_athrow_offset = __ offset();
2075   __ add(exceptionPC->as_register(), r15_pc, -8);
2076   add_call_info(pc_for_athrow_offset, info); // for exception handler
2077 
2078   __ verify_not_null_oop(r0);
2079   // search an exception handler (r0: exception oop, r3: throwing pc)
2080   if (compilation()->has_fpu_code()) {
2081     unwind_id = Runtime1::handle_exception_id;
2082   } else {
2083     unwind_id = Runtime1::handle_exception_nofpu_id;
2084   }
2085   __ far_call(RuntimeAddress(Runtime1::entry_for(unwind_id)));
2086 
2087   // FIXME: enough room for two byte trap   ????
2088   __ nop();
2089 }
2090 
2091 
2092 void LIR_Assembler::unwind_op(LIR_Opr exceptionOop) {
2093   assert(exceptionOop->as_register() == r0, "must match");
2094 
2095   __ b(_unwind_handler_entry);
2096 }
2097 
2098 
2099 void LIR_Assembler::shift_op(LIR_Code code, LIR_Opr left, LIR_Opr count, LIR_Opr dest, LIR_Opr tmp) {
2100   Register lreg = left->is_single_cpu() ? left->as_register() : left->as_register_lo();
2101   Register dreg = dest->is_single_cpu() ? dest->as_register() : dest->as_register_lo();
2102 
2103   switch (left->type()) {
2104     case T_INT:
2105     case T_ADDRESS:
2106     case T_OBJECT:
2107       __ andr(rscratch1, count->as_register(), 0x1f);
2108       switch (code) {
2109         case lir_shl: __ lsl(dreg, lreg, rscratch1); break;
2110         case lir_shr: __ asr(dreg, lreg, rscratch1); break;
2111         case lir_ushr: __ lsr(dreg, lreg, rscratch1); break;
2112         default:
2113           ShouldNotReachHere();
2114           break;
2115       }
2116       break;
2117     case T_LONG:
2118     {
2119       Register lreg_hi = left->as_register_hi();
2120       Register dreg_hi = dest->as_register_hi();
2121       const int word_bits = 8 * wordSize;
2122 
2123       if (code == lir_shl || code == lir_ushr) {
2124         check_register_collision(dreg, &lreg, &lreg_hi, rscratch1);
2125         check_register_collision(dreg_hi, &lreg, &lreg_hi, rscratch2);
2126       }
2127 
2128       switch (code) {
2129         case lir_shl:
2130           __ andr(dreg, count->as_register(), 0x3f);
2131           __ sub(dreg_hi, dreg, word_bits);
2132           __ lsl(lreg_hi, lreg_hi, dreg);
2133           __ orr(lreg_hi, lreg_hi, lreg, lsl(dreg_hi));
2134           __ rsb(dreg_hi, dreg, word_bits);
2135           __ orr(dreg_hi, lreg_hi, lreg, lsr(dreg_hi));
2136           __ lsl(dreg, lreg, dreg);
2137           break;
2138         case lir_shr: {
2139           __ mov(rscratch2, lreg_hi);
2140           __ andr(rscratch1, count->as_register(), 0x3f);
2141           __ lsr(dreg, lreg, rscratch1);
2142           __ rsb(dreg_hi, rscratch1, word_bits);
2143           __ orr(dreg, dreg, rscratch2, lsl(dreg_hi));
2144           __ asr(dreg_hi, rscratch2, rscratch1);
2145           __ subs(rscratch1, rscratch1, word_bits);
2146           __ mov(dreg, rscratch2, asr(rscratch1), Assembler::GT);
2147         }
2148           break;
2149         case lir_ushr:
2150           __ andr(dreg, count->as_register(), 0x3f);
2151           __ lsr(lreg, lreg, dreg);
2152           __ rsb(dreg_hi, dreg, word_bits);
2153           __ orr(lreg, lreg, lreg_hi, lsl(dreg_hi));
2154           __ lsr(dreg_hi, lreg_hi, dreg);
2155           __ sub(dreg, dreg, word_bits);
2156           __ orr(dreg, lreg, lreg_hi, lsr(dreg));
2157           break;
2158         default:
2159           ShouldNotReachHere();
2160           break;
2161       }
2162     }
2163       break;
2164     default:
2165       ShouldNotReachHere();
2166       break;
2167   }
2168 }
2169 
2170 
2171 void LIR_Assembler::shift_op(LIR_Code code, LIR_Opr left, jint count, LIR_Opr dest) {
2172   Register dreg = dest->is_single_cpu() ? dest->as_register() : dest->as_register_lo();
2173   Register lreg = left->is_single_cpu() ? left->as_register() : left->as_register_lo();
2174 
2175   if (!count) {
2176       reg2reg(left, dest);
2177       return;
2178    }
2179 
2180   switch (left->type()) {
2181     case T_INT:
2182     case T_ADDRESS:
2183     case T_OBJECT:
2184       switch (code) {
2185         case lir_shl: __ lsl(dreg, lreg, count); break;
2186         case lir_shr: __ asr(dreg, lreg, count); break;
2187         case lir_ushr: __ lsr(dreg, lreg, count); break;
2188         default:
2189           ShouldNotReachHere();
2190           break;
2191       }
2192       break;
2193     case T_LONG: {
2194       Register lreg_hi = left->as_register_hi();
2195       Register dreg_hi = dest->as_register_hi();
2196       const int word_bits = 8 * wordSize;
2197 
2198       switch (code) {
2199         case lir_shl:
2200           if (count >= word_bits) {
2201             __ lsl(dreg_hi, lreg, count - word_bits);
2202             __ mov(dreg, 0);
2203           } else {
2204             check_register_collision(dreg_hi, &lreg);
2205             __ lsl(dreg_hi, lreg_hi, count);
2206             __ orr(dreg_hi, dreg_hi, lreg, lsr(word_bits - count));
2207             __ lsl(dreg, lreg, count);
2208           }
2209           break;
2210         case lir_shr:
2211           if (count >= word_bits) {
2212             __ asr(dreg, lreg_hi, count - word_bits);
2213             __ asr(dreg_hi, lreg_hi, word_bits);
2214           } else {
2215             check_register_collision(dreg, &lreg_hi);
2216             __ lsr(dreg, lreg, count);
2217             __ orr(dreg, dreg, lreg_hi, lsl(word_bits - count));
2218             __ asr(dreg_hi, lreg_hi, count);
2219           }
2220           break;
2221         case lir_ushr:
2222           if (count >= word_bits) {
2223             __ lsr(dreg, lreg_hi, count - word_bits);
2224             __ mov(dreg_hi, 0);
2225           } else {
2226             check_register_collision(dreg, &lreg_hi);
2227             __ lsr(dreg, lreg, count);
2228             __ orr(dreg, dreg, lreg_hi, lsl(word_bits - count));
2229             __ lsr(dreg_hi, lreg_hi, count);
2230           }
2231           break;
2232         default:
2233           ShouldNotReachHere();
2234           break;
2235       }
2236     }
2237       break;
2238     default:
2239       ShouldNotReachHere();
2240       break;
2241   }
2242 }
2243 
2244 
2245 void LIR_Assembler::store_parameter(Register r, int offset_from_sp_in_words) {
2246   assert(offset_from_sp_in_words >= 0, "invalid offset from sp");
2247   int offset_from_sp_in_bytes = offset_from_sp_in_words * BytesPerWord;
2248   assert(offset_from_sp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2249   __ str (r, Address(sp, offset_from_sp_in_bytes));
2250 }
2251 
2252 
2253 void LIR_Assembler::store_parameter(jint c,     int offset_from_sp_in_words) {
2254   assert(offset_from_sp_in_words >= 0, "invalid offset from sp");
2255   int offset_from_sp_in_bytes = offset_from_sp_in_words * BytesPerWord;
2256   assert(offset_from_sp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2257   __ mov (rscratch1, c);
2258   __ str (rscratch1, Address(sp, offset_from_sp_in_bytes));
2259 }
2260 
2261 // This code replaces a call to arraycopy; no exception may
2262 // be thrown in this code, they must be thrown in the System.arraycopy
2263 // activation frame; we could save some checks if this would not be the case
2264 void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) {
2265   ciArrayKlass* default_type = op->expected_type();
2266   Register src = op->src()->as_register();
2267   Register dst = op->dst()->as_register();
2268   Register src_pos = op->src_pos()->as_register();
2269   Register dst_pos = op->dst_pos()->as_register();
2270   Register length  = op->length()->as_register();
2271   Register tmp = op->tmp()->as_register();
2272   // due to limited number of registers available and in order to simplify
2273   // the code we fix the registers used by the arguments to this intrinsic.
2274   // see the comment in LIRGenerator::do_ArrayCopy
2275   assert(src == j_rarg0, "assumed by implementation");
2276   assert(src_pos == j_rarg1, "assumed by implementation");
2277   assert(dst == j_rarg2, "assumed by implementation");
2278   assert(dst_pos == j_rarg3, "assumed by implementation");
2279   assert(length == r4, "assumed by implementation");
2280   assert(tmp == r5, "assumed by implementation");
2281 
2282   CodeStub* stub = op->stub();
2283   int flags = op->flags();
2284   BasicType basic_type = default_type != NULL ? default_type->element_type()->basic_type() : T_ILLEGAL;
2285   if (basic_type == T_ARRAY) basic_type = T_OBJECT;
2286 
2287   // if we don't know anything, just go through the generic arraycopy
2288   if (default_type == NULL // || basic_type == T_OBJECT
2289       ) {
2290     Label done;
2291     assert(src == r1 && src_pos == r2, "mismatch in calling convention");
2292 
2293     // Save the arguments in case the generic arraycopy fails and we
2294     // have to fall back to the JNI stub
2295     __ str(dst,     Address(sp, 0*BytesPerWord));
2296     __ str(dst_pos, Address(sp, 1*BytesPerWord));
2297     __ str(length,  Address(sp, 2*BytesPerWord));
2298     __ str(src_pos, Address(sp, 3*BytesPerWord));
2299     __ str(src,     Address(sp, 4*BytesPerWord));
2300 
2301     address C_entry = CAST_FROM_FN_PTR(address, Runtime1::arraycopy);
2302     address copyfunc_addr = StubRoutines::generic_arraycopy();
2303 
2304     // The arguments are in java calling convention so we shift them
2305     // to C convention
2306     assert(c_rarg0 == j_rarg3, "assumed in the code below");
2307     __ mov(rscratch1, c_rarg0);
2308     assert_different_registers(c_rarg0, j_rarg1, j_rarg2);
2309     __ mov(c_rarg0, j_rarg0);
2310     assert_different_registers(c_rarg1, j_rarg2, j_rarg3);
2311     __ mov(c_rarg1, j_rarg1);
2312     assert_different_registers(c_rarg2, j_rarg3);
2313     __ mov(c_rarg2, j_rarg2);
2314     __ mov(c_rarg3, rscratch1);
2315     __ str(length, Address(sp)); // the below C function follows C calling convention,
2316                                  // so should put 5th arg to stack
2317 
2318     if (copyfunc_addr == NULL) { // Use C version if stub was not generated
2319       __ mov(rscratch1, RuntimeAddress(C_entry));
2320       __ bl(rscratch1);
2321     } else {
2322 #ifndef PRODUCT
2323       if (PrintC1Statistics) {
2324         __ increment(ExternalAddress((address)&Runtime1::_generic_arraycopystub_cnt));
2325       }
2326 #endif
2327       __ far_call(RuntimeAddress(copyfunc_addr));
2328     }
2329 
2330     __ cbz(r0, *stub->continuation());
2331 
2332     // Reload values from the stack so they are where the stub
2333     // expects them.
2334     __ ldr(dst,     Address(sp, 0*BytesPerWord));
2335     __ ldr(dst_pos, Address(sp, 1*BytesPerWord));
2336     __ ldr(length,  Address(sp, 2*BytesPerWord));
2337     __ ldr(src_pos, Address(sp, 3*BytesPerWord));
2338     __ ldr(src,     Address(sp, 4*BytesPerWord));
2339 
2340     if (copyfunc_addr != NULL) {
2341       // r0 is -1^K where K == partial copied count
2342       __ inv(rscratch1, r0);
2343       // adjust length down and src/end pos up by partial copied count
2344       __ sub(length, length, rscratch1);
2345       __ add(src_pos, src_pos, rscratch1);
2346       __ add(dst_pos, dst_pos, rscratch1);
2347     }
2348     __ b(*stub->entry());
2349 
2350     __ bind(*stub->continuation());
2351     return;
2352   }
2353 
2354   assert(default_type != NULL && default_type->is_array_klass() && default_type->is_loaded(), "must be true at this point");
2355 
2356   int elem_size = type2aelembytes(basic_type);
2357   int scale = exact_log2(elem_size);
2358 
2359   Address src_length_addr = Address(src, arrayOopDesc::length_offset_in_bytes());
2360   Address dst_length_addr = Address(dst, arrayOopDesc::length_offset_in_bytes());
2361   Address src_klass_addr = Address(src, oopDesc::klass_offset_in_bytes());
2362   Address dst_klass_addr = Address(dst, oopDesc::klass_offset_in_bytes());
2363 
2364   // test for NULL
2365   if (flags & LIR_OpArrayCopy::src_null_check) {
2366     __ cbz(src, *stub->entry());
2367   }
2368   if (flags & LIR_OpArrayCopy::dst_null_check) {
2369     __ cbz(dst, *stub->entry());
2370   }
2371 
2372   // check if negative
2373   if (flags & LIR_OpArrayCopy::src_pos_positive_check) {
2374     __ cmp(src_pos, 0);
2375     __ b(*stub->entry(), Assembler::LT);
2376   }
2377   if (flags & LIR_OpArrayCopy::dst_pos_positive_check) {
2378     __ cmp(dst_pos, 0);
2379     __ b(*stub->entry(), Assembler::LT);
2380   }
2381 
2382   if (flags & LIR_OpArrayCopy::length_positive_check) {
2383     __ cmp(length, 0);
2384     __ b(*stub->entry(), Assembler::LT);
2385   }
2386 
2387   if (flags & LIR_OpArrayCopy::src_range_check) {
2388     __ add(tmp, src_pos, length);
2389     __ ldr(rscratch1, src_length_addr);
2390     __ cmp(tmp, rscratch1);
2391     __ b(*stub->entry(), Assembler::HI);
2392   }
2393   if (flags & LIR_OpArrayCopy::dst_range_check) {
2394     __ add(tmp, dst_pos, length);
2395     __ ldr(rscratch1, dst_length_addr);
2396     __ cmp(tmp, rscratch1);
2397     __ b(*stub->entry(), Assembler::HI);
2398   }
2399 
2400   // FIXME: The logic in LIRGenerator::arraycopy_helper clears
2401   // length_positive_check if the source of our length operand is an
2402   // arraylength.  However, that arraylength might be zero, and the
2403   // stub that we're about to call contains an assertion that count !=
2404   // 0 .  So we make this check purely in order not to trigger an
2405   // assertion failure.
2406   __ cbz(length, *stub->continuation());
2407 
2408   if (flags & LIR_OpArrayCopy::type_check) {
2409     // We don't know the array types are compatible
2410     if (basic_type != T_OBJECT) {
2411       // Simple test for basic type arrays
2412       __ ldr(tmp, src_klass_addr);
2413       __ ldr(rscratch1, dst_klass_addr);
2414       __ cmp(tmp, rscratch1);
2415       __ b(*stub->entry(), Assembler::NE);
2416     } else {
2417       // For object arrays, if src is a sub class of dst then we can
2418       // safely do the copy.
2419       Label cont, slow;
2420 
2421       __ push(RegSet::of(src, dst), sp);
2422 
2423       __ load_klass(src, src);
2424       __ load_klass(dst, dst);
2425 
2426       __ check_klass_subtype_fast_path(src, dst, tmp, &cont, &slow, NULL);
2427 
2428       __ push(src); // sub
2429       __ push(dst); // super
2430       __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
2431       // result on TOS
2432       __ pop(src); // result
2433       __ pop(dst);
2434 
2435       __ cbnz(src, cont);
2436 
2437       __ bind(slow);
2438       __ pop(RegSet::of(src, dst), sp);
2439 
2440       address copyfunc_addr = StubRoutines::checkcast_arraycopy();
2441       if (copyfunc_addr != NULL) { // use stub if available
2442         // src is not a sub class of dst so we have to do a
2443         // per-element check.
2444 
2445         int mask = LIR_OpArrayCopy::src_objarray|LIR_OpArrayCopy::dst_objarray;
2446         if ((flags & mask) != mask) {
2447           // Check that at least both of them object arrays.
2448           assert(flags & mask, "one of the two should be known to be an object array");
2449 
2450           if (!(flags & LIR_OpArrayCopy::src_objarray)) {
2451             __ load_klass(tmp, src);
2452           } else if (!(flags & LIR_OpArrayCopy::dst_objarray)) {
2453             __ load_klass(tmp, dst);
2454           }
2455           int lh_offset = in_bytes(Klass::layout_helper_offset());
2456           Address klass_lh_addr(tmp, lh_offset);
2457           jint objArray_lh = Klass::array_layout_helper(T_OBJECT);
2458           __ ldr(rscratch1, klass_lh_addr);
2459           __ mov(rscratch2, objArray_lh);
2460           __ eor(rscratch1, rscratch1, rscratch2);
2461           __ cbnz(rscratch1, *stub->entry());
2462         }
2463 
2464        // Spill because stubs can use any register they like and it's
2465        // easier to restore just those that we care about.
2466         __ str(dst,     Address(sp, 0*BytesPerWord));
2467         __ str(dst_pos, Address(sp, 1*BytesPerWord));
2468         __ str(length,  Address(sp, 2*BytesPerWord));
2469         __ str(src_pos, Address(sp, 3*BytesPerWord));
2470         __ str(src,     Address(sp, 4*BytesPerWord));
2471 
2472         assert(dst_pos == r0, "assumed in the code below");
2473         __ mov(rscratch1, dst_pos); // save dst_pos which is r0
2474         __ lea(c_rarg0, Address(src, src_pos, lsl(scale)));
2475         __ add(c_rarg0, c_rarg0, arrayOopDesc::base_offset_in_bytes(basic_type));
2476         assert_different_registers(c_rarg0, dst, length);
2477         __ lea(c_rarg1, Address(dst, rscratch1, lsl(scale)));
2478         __ add(c_rarg1, c_rarg1, arrayOopDesc::base_offset_in_bytes(basic_type));
2479         assert_different_registers(c_rarg1, dst, length);
2480 
2481         __ load_klass(c_rarg2, dst);
2482         __ ldr(c_rarg2, Address(c_rarg2, ObjArrayKlass::element_klass_offset()));
2483         __ ldr(c_rarg3, Address(c_rarg2, Klass::super_check_offset_offset()));
2484         __ far_call(RuntimeAddress(copyfunc_addr));
2485 
2486 #ifndef PRODUCT
2487         if (PrintC1Statistics) {
2488           Label failed;
2489           __ cbnz(r0, failed);
2490           __ increment(ExternalAddress((address)&Runtime1::_arraycopy_checkcast_cnt));
2491           __ bind(failed);
2492         }
2493 #endif
2494 
2495         __ cbz(r0, *stub->continuation());
2496 
2497 #ifndef PRODUCT
2498         if (PrintC1Statistics) {
2499           __ increment(ExternalAddress((address)&Runtime1::_arraycopy_checkcast_attempt_cnt));
2500         }
2501 #endif
2502         assert_different_registers(dst, dst_pos, length, src_pos, src, rscratch1);
2503         __ mov(rscratch1, r0);
2504 
2505         // Restore previously spilled arguments
2506         __ ldr(dst,     Address(sp, 0*BytesPerWord));
2507         __ ldr(dst_pos, Address(sp, 1*BytesPerWord));
2508         __ ldr(length,  Address(sp, 2*BytesPerWord));
2509         __ ldr(src_pos, Address(sp, 3*BytesPerWord));
2510         __ ldr(src,     Address(sp, 4*BytesPerWord));
2511 
2512         // return value is -1^K where K is partial copied count
2513         __ mvn(rscratch1, rscratch1);
2514         // adjust length down and src/end pos up by partial copied count
2515         __ sub(length, length, rscratch1);
2516         __ add(src_pos, src_pos, rscratch1);
2517         __ add(dst_pos, dst_pos, rscratch1);
2518       }
2519 
2520       __ b(*stub->entry());
2521 
2522       __ bind(cont);
2523       __ pop(RegSet::of(src, dst), sp);
2524     }
2525   }
2526 
2527 #ifdef ASSERT
2528   if (basic_type != T_OBJECT || !(flags & LIR_OpArrayCopy::type_check)) {
2529     // Sanity check the known type with the incoming class.  For the
2530     // primitive case the types must match exactly with src.klass and
2531     // dst.klass each exactly matching the default type.  For the
2532     // object array case, if no type check is needed then either the
2533     // dst type is exactly the expected type and the src type is a
2534     // subtype which we can't check or src is the same array as dst
2535     // but not necessarily exactly of type default_type.
2536     Label known_ok, halt;
2537     __ mov_metadata(tmp, default_type->constant_encoding());
2538 
2539     if (basic_type != T_OBJECT) {
2540 
2541       __ ldr(rscratch1, dst_klass_addr);
2542       __ cmp(tmp, rscratch1);
2543       __ b(halt, Assembler::NE);
2544       __ ldr(rscratch1, src_klass_addr);
2545       __ cmp(tmp, rscratch1);
2546       __ b(known_ok, Assembler::EQ);
2547     } else {
2548       __ ldr(rscratch1, dst_klass_addr);
2549       __ cmp(tmp, rscratch1);
2550       __ b(known_ok, Assembler::EQ);
2551       __ cmp(src, dst);
2552       __ b(known_ok, Assembler::EQ);
2553     }
2554     __ bind(halt);
2555     __ stop("incorrect type information in arraycopy");
2556     __ bind(known_ok);
2557   }
2558 #endif
2559 
2560   assert(dst_pos == r0, "assumed in the code below");
2561   __ mov(rscratch1, dst_pos); // save r0
2562   __ lea(c_rarg0, Address(src, src_pos, lsl(scale)));
2563   __ add(c_rarg0, c_rarg0, arrayOopDesc::base_offset_in_bytes(basic_type));
2564   assert_different_registers(c_rarg0, dst, rscratch1, length);
2565   __ lea(c_rarg1, Address(dst, rscratch1, lsl(scale)));
2566   __ add(c_rarg1, c_rarg1, arrayOopDesc::base_offset_in_bytes(basic_type));
2567   assert_different_registers(c_rarg1, dst, length);
2568   __ mov(c_rarg2, length);
2569 
2570   bool disjoint = (flags & LIR_OpArrayCopy::overlapping) == 0;
2571   bool aligned = (flags & LIR_OpArrayCopy::unaligned) == 0;
2572   const char *name;
2573   address entry = StubRoutines::select_arraycopy_function(basic_type, aligned, disjoint, name, false);
2574 
2575   CodeBlob *cb = CodeCache::find_blob(entry);
2576   if (cb) {
2577     __ far_call(RuntimeAddress(entry));
2578   } else {
2579     __ call_VM_leaf(entry, 3);
2580   }
2581 
2582   __ bind(*stub->continuation());
2583 }
2584 
2585 void LIR_Assembler::emit_lock(LIR_OpLock* op) {
2586   Register obj = op->obj_opr()->as_register();  // may not be an oop
2587   Register hdr = op->hdr_opr()->as_register();
2588   Register lock = op->lock_opr()->as_register();
2589   if (!UseFastLocking) {
2590     __ b(*op->stub()->entry());
2591   } else if (op->code() == lir_lock) {
2592     Register scratch = noreg;
2593     if (UseBiasedLocking) {
2594       scratch = op->scratch_opr()->as_register();
2595     }
2596     assert(BasicLock::displaced_header_offset_in_bytes() == 0, "lock_reg must point to the displaced header");
2597     // add debug info for NullPointerException only if one is possible
2598     int null_check_offset = __ lock_object(hdr, obj, lock, scratch, *op->stub()->entry());
2599     if (op->info() != NULL) {
2600       add_debug_info_for_null_check(null_check_offset, op->info());
2601     }
2602     // done
2603   } else if (op->code() == lir_unlock) {
2604     assert(BasicLock::displaced_header_offset_in_bytes() == 0, "lock_reg must point to the displaced header");
2605     __ unlock_object(hdr, obj, lock, *op->stub()->entry());
2606   } else {
2607     Unimplemented();
2608   }
2609   __ bind(*op->stub()->continuation());
2610 }
2611 
2612 
2613 void LIR_Assembler::emit_profile_call(LIR_OpProfileCall* op) {
2614   ciMethod* method = op->profiled_method();
2615   int bci          = op->profiled_bci();
2616   ciMethod* callee = op->profiled_callee();
2617 
2618   // Update counter for all call types
2619   ciMethodData* md = method->method_data_or_null();
2620   assert(md != NULL, "Sanity");
2621   ciProfileData* data = md->bci_to_data(bci);
2622   assert(data->is_CounterData(), "need CounterData for calls");
2623   assert(op->mdo()->is_single_cpu(),  "mdo must be allocated");
2624   Register mdo  = op->mdo()->as_register();
2625   __ mov_metadata(mdo, md->constant_encoding());
2626   Address counter_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
2627   Bytecodes::Code bc = method->java_code_at_bci(bci);
2628   const bool callee_is_static = callee->is_loaded() && callee->is_static();
2629   // Perform additional virtual call profiling for invokevirtual and
2630   // invokeinterface bytecodes
2631   if ((bc == Bytecodes::_invokevirtual || bc == Bytecodes::_invokeinterface) &&
2632       !callee_is_static &&  // required for optimized MH invokes
2633       C1ProfileVirtualCalls) {
2634     assert(op->recv()->is_single_cpu(), "recv must be allocated");
2635     Register recv = op->recv()->as_register();
2636     assert_different_registers(mdo, recv);
2637     assert(data->is_VirtualCallData(), "need VirtualCallData for virtual calls");
2638     ciKlass* known_klass = op->known_holder();
2639     if (C1OptimizeVirtualCallProfiling && known_klass != NULL) {
2640       // We know the type that will be seen at this call site; we can
2641       // statically update the MethodData* rather than needing to do
2642       // dynamic tests on the receiver type
2643 
2644       // NOTE: we should probably put a lock around this search to
2645       // avoid collisions by concurrent compilations
2646       ciVirtualCallData* vc_data = (ciVirtualCallData*) data;
2647       uint i;
2648       for (i = 0; i < VirtualCallData::row_limit(); i++) {
2649         ciKlass* receiver = vc_data->receiver(i);
2650         if (known_klass->equals(receiver)) {
2651           Address data_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_count_offset(i)));
2652           __ addptr(data_addr, DataLayout::counter_increment);
2653           return;
2654         }
2655       }
2656 
2657       // Receiver type not found in profile data; select an empty slot
2658 
2659       // Note that this is less efficient than it should be because it
2660       // always does a write to the receiver part of the
2661       // VirtualCallData rather than just the first time
2662       for (i = 0; i < VirtualCallData::row_limit(); i++) {
2663         ciKlass* receiver = vc_data->receiver(i);
2664         if (receiver == NULL) {
2665           Address recv_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_offset(i)));
2666           __ mov_metadata(rscratch1, known_klass->constant_encoding());
2667           __ lea(rscratch2, recv_addr);
2668           __ str(rscratch1, Address(rscratch2));
2669           Address data_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_count_offset(i)));
2670           __ addptr(data_addr, DataLayout::counter_increment);
2671           return;
2672         }
2673       }
2674     } else {
2675       __ load_klass(recv, recv);
2676       Label update_done;
2677       type_profile_helper(mdo, md, data, recv, &update_done);
2678       // Receiver did not match any saved receiver and there is no empty row for it.
2679       // Increment total counter to indicate polymorphic case.
2680       __ addptr(counter_addr, DataLayout::counter_increment);
2681 
2682       __ bind(update_done);
2683     }
2684   } else {
2685     // Static call
2686     __ addptr(counter_addr, DataLayout::counter_increment);
2687   }
2688 }
2689 
2690 
2691 void LIR_Assembler::emit_delay(LIR_OpDelay*) {
2692   Unimplemented();
2693 }
2694 
2695 
2696 void LIR_Assembler::monitor_address(int monitor_no, LIR_Opr dst) {
2697   __ lea(dst->as_register(), frame_map()->address_for_monitor_lock(monitor_no));
2698 }
2699 
2700 void LIR_Assembler::emit_updatecrc32(LIR_OpUpdateCRC32* op) {
2701   assert(op->crc()->is_single_cpu(),  "crc must be register");
2702   assert(op->val()->is_single_cpu(),  "byte value must be register");
2703   assert(op->result_opr()->is_single_cpu(), "result must be register");
2704   Register crc = op->crc()->as_register();
2705   Register val = op->val()->as_register();
2706   Register res = op->result_opr()->as_register();
2707 
2708   assert_different_registers(val, crc, res);
2709   __ lea(res, ExternalAddress(StubRoutines::crc_table_addr()));
2710 
2711   __ inv(crc, crc);
2712   __ update_byte_crc32(crc, val, res);
2713   __ inv(res, crc);
2714 }
2715 
2716 void LIR_Assembler::emit_profile_type(LIR_OpProfileType* op) {
2717   COMMENT("emit_profile_type {");
2718   Register obj = op->obj()->as_register();
2719   Register tmp = op->tmp()->as_pointer_register();
2720   Address mdo_addr = as_Address(op->mdp()->as_address_ptr(), noreg, Address::IDT_INT);
2721   ciKlass* exact_klass = op->exact_klass();
2722   intptr_t current_klass = op->current_klass();
2723   bool not_null = op->not_null();
2724   bool no_conflict = op->no_conflict();
2725 
2726   Label update, next, none;
2727 
2728   bool do_null = !not_null;
2729   bool exact_klass_set = exact_klass != NULL && ciTypeEntries::valid_ciklass(current_klass) == exact_klass;
2730   bool do_update = !TypeEntries::is_type_unknown(current_klass) && !exact_klass_set;
2731 
2732   assert(do_null || do_update, "why are we here?");
2733   assert(!TypeEntries::was_null_seen(current_klass) || do_update, "why are we here?");
2734   assert(mdo_addr.base() != rscratch1, "wrong register");
2735 
2736   __ verify_oop(obj);
2737 
2738   if (tmp != obj) {
2739     __ mov(tmp, obj);
2740   }
2741   if (do_null) {
2742     __ cbnz(tmp, update);
2743     if (!TypeEntries::was_null_seen(current_klass)) {
2744       __ ldr(rscratch2, mdo_addr);
2745       __ orr(rscratch2, rscratch2, TypeEntries::null_seen);
2746       __ str(rscratch2, mdo_addr);
2747     }
2748     if (do_update) {
2749 #ifndef ASSERT
2750       __ b(next);
2751     }
2752 #else
2753       __ b(next);
2754     }
2755   } else {
2756     __ cbnz(tmp, update);
2757     __ stop("unexpected null obj");
2758 #endif
2759   }
2760 
2761   __ bind(update);
2762 
2763   if (do_update) {
2764 #ifdef ASSERT
2765     if (exact_klass != NULL) {
2766       Label ok;
2767       __ load_klass(tmp, tmp);
2768       __ mov_metadata(rscratch1, exact_klass->constant_encoding());
2769       __ eor(rscratch1, tmp, rscratch1);
2770       __ cbz(rscratch1, ok);
2771       __ stop("exact klass and actual klass differ");
2772       __ bind(ok);
2773     }
2774 #endif
2775     if (!no_conflict) {
2776       if (exact_klass == NULL || TypeEntries::is_type_none(current_klass)) {
2777         if (exact_klass != NULL) {
2778           __ mov_metadata(tmp, exact_klass->constant_encoding());
2779         } else {
2780           __ load_klass(tmp, tmp);
2781         }
2782 
2783         __ ldr(rscratch2, mdo_addr);
2784         __ eor(tmp, tmp, rscratch2);
2785         __ andr(rscratch1, tmp, TypeEntries::type_klass_mask);
2786         // klass seen before, nothing to do. The unknown bit may have been
2787         // set already but no need to check.
2788         __ cbz(rscratch1, next);
2789 
2790         __ andr(rscratch1, tmp, TypeEntries::type_unknown);
2791         __ cbnz(rscratch1, next); // already unknown. Nothing to do anymore.
2792 
2793         if (TypeEntries::is_type_none(current_klass)) {
2794           __ cbz(rscratch2, none);
2795           __ cmp(rscratch2, TypeEntries::null_seen);
2796           __ b(none, Assembler::EQ);
2797           // There is a chance that the checks above (re-reading profiling
2798           // data from memory) fail if another thread has just set the
2799           // profiling to this obj's klass
2800           __ dmb(Assembler::ISH);
2801           __ ldr(rscratch2, mdo_addr);
2802           __ eor(tmp, tmp, rscratch2);
2803           __ andr(rscratch1, tmp, TypeEntries::type_klass_mask);
2804           __ cbz(rscratch1, next);
2805         }
2806       } else {
2807         assert(ciTypeEntries::valid_ciklass(current_klass) != NULL &&
2808                ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "conflict only");
2809 
2810         __ ldr(tmp, mdo_addr);
2811         __ andr(rscratch1, tmp, TypeEntries::type_unknown);
2812         __ cbnz(rscratch1, next); // already unknown. Nothing to do anymore.
2813       }
2814 
2815       // different than before. Cannot keep accurate profile.
2816       __ ldr(rscratch2, mdo_addr);
2817       __ orr(rscratch2, rscratch2, TypeEntries::type_unknown);
2818       __ str(rscratch2, mdo_addr);
2819 
2820       if (TypeEntries::is_type_none(current_klass)) {
2821         __ b(next);
2822 
2823         __ bind(none);
2824         // first time here. Set profile type.
2825         __ str(tmp, mdo_addr);
2826       }
2827     } else {
2828       // There's a single possible klass at this profile point
2829       assert(exact_klass != NULL, "should be");
2830       if (TypeEntries::is_type_none(current_klass)) {
2831         __ mov_metadata(tmp, exact_klass->constant_encoding());
2832         __ ldr(rscratch2, mdo_addr);
2833         __ eor(tmp, tmp, rscratch2);
2834         __ andr(rscratch1, tmp, TypeEntries::type_klass_mask);
2835         __ cbz(rscratch1, next);
2836 #ifdef ASSERT
2837         {
2838           Label ok;
2839           __ ldr(rscratch1, mdo_addr);
2840           __ cbz(rscratch1, ok);
2841           __ cmp(rscratch1, TypeEntries::null_seen);
2842           __ b(ok, Assembler::EQ);
2843           // may have been set by another thread
2844           __ dmb(Assembler::ISH);
2845           __ mov_metadata(rscratch1, exact_klass->constant_encoding());
2846           __ ldr(rscratch2, mdo_addr);
2847           __ eor(rscratch2, rscratch1, rscratch2);
2848           __ andr(rscratch2, rscratch2, TypeEntries::type_mask);
2849           __ cbz(rscratch2, ok);
2850 
2851           __ stop("unexpected profiling mismatch");
2852           __ bind(ok);
2853         }
2854 #endif
2855         // first time here. Set profile type.
2856         __ ldr(tmp, mdo_addr);
2857       } else {
2858         assert(ciTypeEntries::valid_ciklass(current_klass) != NULL &&
2859                ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "inconsistent");
2860 
2861         __ ldr(tmp, mdo_addr);
2862         __ andr(rscratch1, tmp, TypeEntries::type_unknown);
2863         __ cbnz(rscratch1, next); // already unknown. Nothing to do anymore.
2864 
2865         __ orr(tmp, tmp, TypeEntries::type_unknown);
2866         __ str(tmp, mdo_addr);
2867         // FIXME: Write barrier needed here?
2868       }
2869     }
2870 
2871     __ bind(next);
2872   }
2873   COMMENT("} emit_profile_type");
2874 }
2875 
2876 
2877 void LIR_Assembler::align_backward_branch_target() {
2878 }
2879 
2880 
2881 void LIR_Assembler::negate(LIR_Opr left, LIR_Opr dest) {
2882   if (left->is_single_cpu()) {
2883     assert(dest->is_single_cpu(), "expect single result reg");
2884     __ neg(dest->as_register(), left->as_register());
2885   } else if (left->is_double_cpu()) {
2886     assert(dest->is_double_cpu(), "expect double result reg");
2887     const Register l_lo = left->as_register_lo();
2888     Register l_hi = left->as_register_hi();
2889     check_register_collision(dest->as_register_lo(), &l_hi);
2890     __ rsbs(dest->as_register_lo(), l_lo, 0);
2891     __ rsc(dest->as_register_hi(), l_hi, 0);
2892   } else if (left->is_single_fpu()) {
2893     assert(dest->is_single_fpu(), "expect single float result reg");
2894     __ vneg_f32(dest->as_float_reg(), left->as_float_reg());
2895   } else {
2896     assert(left->is_double_fpu(), "expect double float operand reg");
2897     assert(dest->is_double_fpu(), "expect double float result reg");
2898     __ vneg_f64(dest->as_double_reg(), left->as_double_reg());
2899   }
2900 }
2901 
2902 
2903 void LIR_Assembler::leal(LIR_Opr addr, LIR_Opr dest) {
2904   __ lea(dest->as_register(), as_Address(addr->as_address_ptr(), noreg, Address::IDT_LEA));
2905 }
2906 
2907 
2908 void LIR_Assembler::rt_call(LIR_Opr result, address dest, const LIR_OprList* args, LIR_Opr tmp, CodeEmitInfo* info) {
2909   assert(!tmp->is_valid(), "don't need temporary");
2910   CodeBlob *cb = CodeCache::find_blob(dest);
2911   if (cb) {
2912     __ far_call(RuntimeAddress(dest));
2913   } else {
2914     __ lea(rscratch1, RuntimeAddress(dest));
2915     __ bl(rscratch1);
2916   }
2917   if (info != NULL) {
2918     add_call_info_here(info);
2919   }
2920   __ maybe_isb();
2921 }
2922 
2923 void LIR_Assembler::volatile_move_op(LIR_Opr src, LIR_Opr dest, BasicType type, CodeEmitInfo* info) {
2924   if (type == T_LONG || type == T_DOUBLE) {
2925     const LIR_Opr long_val = FrameMap::long0_opr;
2926 
2927     int null_check_offset = -1;
2928 
2929     if (src->is_register() && dest->is_address()) {
2930       // long1 reserved as temp by LinearScan::pd_add_temps
2931       const LIR_Opr long_tmp = FrameMap::long1_opr;
2932       __ lea(rscratch1, as_Address_lo(dest->as_address_ptr(), Address::IDT_LEA));
2933 
2934       if (type == T_DOUBLE) {
2935         // long0 reserved as temp by LinearScan::pd_add_temps
2936         __ vmov_f64(long_val->as_register_lo(), long_val->as_register_hi(), src->as_double_reg());
2937       } else {
2938         assert(type == T_LONG && src->is_same_register(long_val), "T_LONG src should be in long0 (by LIRGenerator)");
2939       }
2940 
2941       null_check_offset = __ offset();
2942       __ atomic_strd(long_val->as_register_lo(), long_val->as_register_hi(), rscratch1,
2943           long_tmp->as_register_lo(), long_tmp->as_register_hi());
2944 
2945     } else if (src->is_address() && dest->is_register()) {
2946       __ lea(rscratch1, as_Address_lo(src->as_address_ptr(), Address::IDT_LEA));
2947 
2948       null_check_offset = __ offset();
2949       __ atomic_ldrd(long_val->as_register_lo(), long_val->as_register_hi(), rscratch1);
2950 
2951       if (type == T_DOUBLE) {
2952         __ vmov_f64(dest->as_double_reg(), long_val->as_register_lo(), long_val->as_register_hi());
2953       } else {
2954         assert(type != T_LONG || dest->is_same_register(long_val), "T_LONG dest should be in long0 (by LIRGenerator)");
2955       }
2956     } else {
2957       Unimplemented();
2958     }
2959 
2960     if (info != NULL) {
2961       add_debug_info_for_null_check(null_check_offset, info);
2962     }
2963 
2964   } else {
2965     move_op(src, dest, type, lir_patch_none, info,
2966             /*pop_fpu_stack*/false, /*unaligned*/false, /*wide*/false);
2967   }
2968 }
2969 
2970 #ifdef ASSERT
2971 // emit run-time assertion
2972 void LIR_Assembler::emit_assert(LIR_OpAssert* op) {
2973   assert(op->code() == lir_assert, "must be");
2974 
2975   if (op->in_opr1()->is_valid()) {
2976     assert(op->in_opr2()->is_valid(), "both operands must be valid");
2977     comp_op(op->condition(), op->in_opr1(), op->in_opr2(), op);
2978   } else {
2979     assert(op->in_opr2()->is_illegal(), "both operands must be illegal");
2980     assert(op->condition() == lir_cond_always, "no other conditions allowed");
2981   }
2982 
2983   Label ok;
2984   if (op->condition() != lir_cond_always) {
2985     Assembler::Condition acond = Assembler::AL;
2986     switch (op->condition()) {
2987       case lir_cond_equal:        acond = Assembler::EQ;  break;
2988       case lir_cond_notEqual:     acond = Assembler::NE;  break;
2989       case lir_cond_less:         acond = Assembler::LT;  break;
2990       case lir_cond_greaterEqual: acond = Assembler::GE;  break;
2991       case lir_cond_lessEqual:    acond = Assembler::LE;  break;
2992       case lir_cond_greater:      acond = Assembler::GT;  break;
2993       case lir_cond_belowEqual:   acond = Assembler::LS;  break;
2994       case lir_cond_aboveEqual:   acond = Assembler::HS;  break;
2995       default:                    ShouldNotReachHere();
2996     }
2997     if (op->in_opr1()->type() == T_LONG) {
2998       // a special trick here to be able to effectively compare jlongs
2999       // for the lessEqual and greater conditions the jlong operands are swapped
3000       // during comparison and hence should use mirror condition in conditional
3001       // instruction
3002       // see LIR_Assembler::comp_op and LIR_Assembler::cmove
3003       switch (op->condition()) {
3004         case lir_cond_lessEqual:    acond = Assembler::GE;  break;
3005         case lir_cond_greater:      acond = Assembler::LT;  break;
3006       }
3007     }
3008     __ b(ok, acond);
3009   }
3010   if (op->halt()) {
3011     const char* str = __ code_string(op->msg());
3012     __ stop(str);
3013   } else {
3014     breakpoint();
3015   }
3016   __ bind(ok);
3017 }
3018 #endif
3019 
3020 #ifndef PRODUCT
3021 #define COMMENT(x)   do { __ block_comment(x); } while (0)
3022 #else
3023 #define COMMENT(x)
3024 #endif
3025 
3026 void LIR_Assembler::membar() {
3027   COMMENT("membar");
3028   __ membar(MacroAssembler::AnyAny);
3029 }
3030 
3031 void LIR_Assembler::membar_acquire() {
3032   __ membar(Assembler::LoadLoad|Assembler::LoadStore);
3033 }
3034 
3035 void LIR_Assembler::membar_release() {
3036   __ membar(Assembler::LoadStore|Assembler::StoreStore);
3037 }
3038 
3039 void LIR_Assembler::membar_loadload() {
3040   __ membar(Assembler::LoadLoad);
3041 }
3042 
3043 void LIR_Assembler::membar_storestore() {
3044   __ membar(MacroAssembler::StoreStore);
3045 }
3046 
3047 void LIR_Assembler::membar_loadstore() { __ membar(MacroAssembler::LoadStore); }
3048 
3049 void LIR_Assembler::membar_storeload() { __ membar(MacroAssembler::StoreLoad); }
3050 
3051 void LIR_Assembler::get_thread(LIR_Opr result_reg) {
3052   __ mov(result_reg->as_register(), rthread);
3053 }
3054 
3055 
3056 void LIR_Assembler::peephole(LIR_List *lir) {
3057 #if 0
3058   if (tableswitch_count >= max_tableswitches)
3059     return;
3060 
3061   /*
3062     This finite-state automaton recognizes sequences of compare-and-
3063     branch instructions.  We will turn them into a tableswitch.  You
3064     could argue that C1 really shouldn't be doing this sort of
3065     optimization, but without it the code is really horrible.
3066   */
3067 
3068   enum { start_s, cmp1_s, beq_s, cmp_s } state;
3069   int first_key, last_key = -2147483648;
3070   int next_key = 0;
3071   int start_insn = -1;
3072   int last_insn = -1;
3073   Register reg = noreg;
3074   LIR_Opr reg_opr;
3075   state = start_s;
3076 
3077   LIR_OpList* inst = lir->instructions_list();
3078   for (int i = 0; i < inst->length(); i++) {
3079     LIR_Op* op = inst->at(i);
3080     switch (state) {
3081     case start_s:
3082       first_key = -1;
3083       start_insn = i;
3084       switch (op->code()) {
3085       case lir_cmp:
3086         LIR_Opr opr1 = op->as_Op2()->in_opr1();
3087         LIR_Opr opr2 = op->as_Op2()->in_opr2();
3088         if (opr1->is_cpu_register() && opr1->is_single_cpu()
3089             && opr2->is_constant()
3090             && opr2->type() == T_INT) {
3091           reg_opr = opr1;
3092           reg = opr1->as_register();
3093           first_key = opr2->as_constant_ptr()->as_jint();
3094           next_key = first_key + 1;
3095           state = cmp_s;
3096           goto next_state;
3097         }
3098         break;
3099       }
3100       break;
3101     case cmp_s:
3102       switch (op->code()) {
3103       case lir_branch:
3104         if (op->as_OpBranch()->cond() == lir_cond_equal) {
3105           state = beq_s;
3106           last_insn = i;
3107           goto next_state;
3108         }
3109       }
3110       state = start_s;
3111       break;
3112     case beq_s:
3113       switch (op->code()) {
3114       case lir_cmp: {
3115         LIR_Opr opr1 = op->as_Op2()->in_opr1();
3116         LIR_Opr opr2 = op->as_Op2()->in_opr2();
3117         if (opr1->is_cpu_register() && opr1->is_single_cpu()
3118             && opr1->as_register() == reg
3119             && opr2->is_constant()
3120             && opr2->type() == T_INT
3121             && opr2->as_constant_ptr()->as_jint() == next_key) {
3122           last_key = next_key;
3123           next_key++;
3124           state = cmp_s;
3125           goto next_state;
3126         }
3127       }
3128       }
3129       last_key = next_key;
3130       state = start_s;
3131       break;
3132     default:
3133       assert(false, "impossible state");
3134     }
3135     if (state == start_s) {
3136       if (first_key < last_key - 5L && reg != noreg) {
3137         {
3138           // printf("found run register %d starting at insn %d low value %d high value %d\n",
3139           //        reg->encoding(),
3140           //        start_insn, first_key, last_key);
3141           //   for (int i = 0; i < inst->length(); i++) {
3142           //     inst->at(i)->print();
3143           //     tty->print("\n");
3144           //   }
3145           //   tty->print("\n");
3146         }
3147 
3148         struct tableswitch *sw = &switches[tableswitch_count];
3149         sw->_insn_index = start_insn, sw->_first_key = first_key,
3150           sw->_last_key = last_key, sw->_reg = reg;
3151         inst->insert_before(last_insn + 1, new LIR_OpLabel(&sw->_after));
3152         {
3153           // Insert the new table of branches
3154           int offset = last_insn;
3155           for (int n = first_key; n < last_key; n++) {
3156             inst->insert_before
3157               (last_insn + 1,
3158                new LIR_OpBranch(lir_cond_always, T_ILLEGAL,
3159                                 inst->at(offset)->as_OpBranch()->label()));
3160             offset -= 2, i++;
3161           }
3162         }
3163         // Delete all the old compare-and-branch instructions
3164         for (int n = first_key; n < last_key; n++) {
3165           inst->remove_at(start_insn);
3166           inst->remove_at(start_insn);
3167         }
3168         // Insert the tableswitch instruction
3169         inst->insert_before(start_insn,
3170                             new LIR_Op2(lir_cmp, lir_cond_always,
3171                                         LIR_OprFact::intConst(tableswitch_count),
3172                                         reg_opr));
3173         inst->insert_before(start_insn + 1, new LIR_OpLabel(&sw->_branches));
3174         tableswitch_count++;
3175       }
3176       reg = noreg;
3177       last_key = -2147483648;
3178     }
3179   next_state:
3180     ;
3181   }
3182 #endif
3183 }
3184 
3185 void LIR_Assembler::atomic_op(LIR_Code code, LIR_Opr src, LIR_Opr data, LIR_Opr dest, LIR_Opr tmp_op) {
3186   BasicType type = src->type();
3187   Address addr = as_Address(src->as_address_ptr(), Address::toInsnDataType(type));
3188 
3189   bool is_long = false;
3190 
3191   switch(type) {
3192   case T_INT:
3193   case T_OBJECT:
3194   case T_ARRAY:
3195     break;
3196   case T_LONG:
3197     is_long = true;
3198     break;
3199   default:
3200     ShouldNotReachHere();
3201   }
3202 
3203   switch (code) {
3204   case lir_xadd:
3205     {
3206       Register tmp = tmp_op->as_register();
3207       Register dst = as_reg(dest);
3208       Label again;
3209       __ lea(tmp, addr);
3210       __ bind(again);
3211       if(is_long) {
3212           assert(dest->as_register_lo()->successor() == dest->as_register_hi(), "must be contiguous");
3213           assert((dest->as_register_lo()->encoding() & 1) == 0, "must be even");
3214           _masm->ldrexd(dst, tmp);
3215       } else {
3216           _masm->ldrex(dst, tmp);
3217       }
3218       arith_op(lir_add, dest, data, dest, NULL, false);
3219       if (is_long) {
3220         _masm->strexd(rscratch1, dst, tmp);
3221       } else {
3222         _masm->strex(rscratch1, dst, tmp);
3223       }
3224       __ cbnz(rscratch1, again);
3225       arith_op(lir_sub, dest, data, dest, NULL, false);
3226       break;
3227     }
3228   case lir_xchg:
3229     {
3230       Register tmp = tmp_op->as_register();
3231       Register obj = as_reg(data);
3232       Register dst = as_reg(dest);
3233       assert_different_registers(obj, addr.base(), tmp, rscratch1, dst);
3234       Label again;
3235       __ lea(tmp, addr);
3236       __ bind(again);
3237       if(is_long) {
3238           assert(dest->as_register_lo()->successor() == dest->as_register_hi(), "must be contiguous");
3239           assert((dest->as_register_lo()->encoding() & 1) == 0, "must be even");
3240 
3241           assert(data->is_double_cpu(), "should be double register");
3242           assert(data->as_register_lo()->successor() == data->as_register_hi(), "must be contiguous");
3243           assert((data->as_register_lo()->encoding() & 1) == 0, "must be even");
3244 
3245           _masm->ldrexd(dst, tmp);
3246           _masm->strexd(rscratch1, obj, tmp);
3247       } else {
3248          _masm->ldrex(dst, tmp);
3249          _masm->strex(rscratch1, obj, tmp);
3250       }
3251       __ cbnz(rscratch1, again);
3252     }
3253     break;
3254   default:
3255     ShouldNotReachHere();
3256   }
3257   __ membar(__ AnyAny);
3258 }
3259 
3260 void LIR_Assembler::check_register_collision(Register d, Register *s1, Register *s2, Register tmp) {
3261   // use a temp if any of the registers used as a source of operation
3262   // collide with result register of the prerequisite operation
3263   if (d == *s1) {
3264     __ mov(tmp, d);
3265     *s1 = tmp;
3266   } else if (s2 && d == *s2) {
3267     __ mov(tmp, d);
3268     *s2 = tmp;
3269   }
3270 }
3271 
3272 #undef __