1 /*
   2  * Copyright (c) 2000, 2018, Oracle and/or its affiliates. All rights reserved.
   3  * Copyright (c) 2014, Red Hat Inc. All rights reserved.
   4  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   5  *
   6  * This code is free software; you can redistribute it and/or modify it
   7  * under the terms of the GNU General Public License version 2 only, as
   8  * published by the Free Software Foundation.
   9  *
  10  * This code is distributed in the hope that it will be useful, but WITHOUT
  11  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  12  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  13  * version 2 for more details (a copy is included in the LICENSE file that
  14  * accompanied this code).
  15  *
  16  * You should have received a copy of the GNU General Public License version
  17  * 2 along with this work; if not, write to the Free Software Foundation,
  18  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  19  *
  20  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  21  * or visit www.oracle.com if you need additional information or have any
  22  * questions.
  23  *
  24  */
  25 
  26 #include "precompiled.hpp"
  27 #include "asm/assembler.hpp"
  28 #include "c1/c1_CodeStubs.hpp"
  29 #include "c1/c1_Compilation.hpp"
  30 #include "c1/c1_LIRAssembler.hpp"
  31 #include "c1/c1_MacroAssembler.hpp"
  32 #include "c1/c1_Runtime1.hpp"
  33 #include "c1/c1_ValueStack.hpp"
  34 #include "ci/ciArrayKlass.hpp"
  35 #include "ci/ciInstance.hpp"
  36 #include "gc/shared/barrierSet.hpp"
  37 #include "gc/shared/cardTableModRefBS.hpp"
  38 #include "gc/shared/collectedHeap.hpp"
  39 #include "nativeInst_aarch64.hpp"
  40 #include "oops/objArrayKlass.hpp"
  41 #include "runtime/sharedRuntime.hpp"
  42 #include "vmreg_aarch64.inline.hpp"
  43 
  44 
  45 
  46 #ifndef PRODUCT
  47 #define COMMENT(x)   do { __ block_comment(x); } while (0)
  48 #else
  49 #define COMMENT(x)
  50 #endif
  51 
  52 NEEDS_CLEANUP // remove this definitions ?
  53 const Register IC_Klass    = rscratch2;   // where the IC klass is cached
  54 const Register SYNC_header = r0;   // synchronization header
  55 const Register SHIFT_count = r0;   // where count for shift operations must be
  56 
  57 #define __ _masm->
  58 
  59 
  60 static void select_different_registers(Register preserve,
  61                                        Register extra,
  62                                        Register &tmp1,
  63                                        Register &tmp2) {
  64   if (tmp1 == preserve) {
  65     assert_different_registers(tmp1, tmp2, extra);
  66     tmp1 = extra;
  67   } else if (tmp2 == preserve) {
  68     assert_different_registers(tmp1, tmp2, extra);
  69     tmp2 = extra;
  70   }
  71   assert_different_registers(preserve, tmp1, tmp2);
  72 }
  73 
  74 
  75 
  76 static void select_different_registers(Register preserve,
  77                                        Register extra,
  78                                        Register &tmp1,
  79                                        Register &tmp2,
  80                                        Register &tmp3) {
  81   if (tmp1 == preserve) {
  82     assert_different_registers(tmp1, tmp2, tmp3, extra);
  83     tmp1 = extra;
  84   } else if (tmp2 == preserve) {
  85     assert_different_registers(tmp1, tmp2, tmp3, extra);
  86     tmp2 = extra;
  87   } else if (tmp3 == preserve) {
  88     assert_different_registers(tmp1, tmp2, tmp3, extra);
  89     tmp3 = extra;
  90   }
  91   assert_different_registers(preserve, tmp1, tmp2, tmp3);
  92 }
  93 
  94 
  95 bool LIR_Assembler::is_small_constant(LIR_Opr opr) { Unimplemented(); return false; }
  96 
  97 
  98 LIR_Opr LIR_Assembler::receiverOpr() {
  99   return FrameMap::receiver_opr;
 100 }
 101 
 102 LIR_Opr LIR_Assembler::osrBufferPointer() {
 103   return FrameMap::as_pointer_opr(receiverOpr()->as_register());
 104 }
 105 
 106 //--------------fpu register translations-----------------------
 107 
 108 
 109 address LIR_Assembler::float_constant(float f) {
 110   address const_addr = __ float_constant(f);
 111   if (const_addr == NULL) {
 112     bailout("const section overflow");
 113     return __ code()->consts()->start();
 114   } else {
 115     return const_addr;
 116   }
 117 }
 118 
 119 
 120 address LIR_Assembler::double_constant(double d) {
 121   address const_addr = __ double_constant(d);
 122   if (const_addr == NULL) {
 123     bailout("const section overflow");
 124     return __ code()->consts()->start();
 125   } else {
 126     return const_addr;
 127   }
 128 }
 129 
 130 address LIR_Assembler::int_constant(jlong n) {
 131   address const_addr = __ long_constant(n);
 132   if (const_addr == NULL) {
 133     bailout("const section overflow");
 134     return __ code()->consts()->start();
 135   } else {
 136     return const_addr;
 137   }
 138 }
 139 
 140 void LIR_Assembler::set_24bit_FPU() { Unimplemented(); }
 141 
 142 void LIR_Assembler::reset_FPU() { Unimplemented(); }
 143 
 144 void LIR_Assembler::fpop() { Unimplemented(); }
 145 
 146 void LIR_Assembler::fxch(int i) { Unimplemented(); }
 147 
 148 void LIR_Assembler::fld(int i) { Unimplemented(); }
 149 
 150 void LIR_Assembler::ffree(int i) { Unimplemented(); }
 151 
 152 void LIR_Assembler::breakpoint() { Unimplemented(); }
 153 
 154 void LIR_Assembler::push(LIR_Opr opr) { Unimplemented(); }
 155 
 156 void LIR_Assembler::pop(LIR_Opr opr) { Unimplemented(); }
 157 
 158 bool LIR_Assembler::is_literal_address(LIR_Address* addr) { Unimplemented(); return false; }
 159 //-------------------------------------------
 160 
 161 static Register as_reg(LIR_Opr op) {
 162   return op->is_double_cpu() ? op->as_register_lo() : op->as_register();
 163 }
 164 
 165 static jlong as_long(LIR_Opr data) {
 166   jlong result;
 167   switch (data->type()) {
 168   case T_INT:
 169     result = (data->as_jint());
 170     break;
 171   case T_LONG:
 172     result = (data->as_jlong());
 173     break;
 174   default:
 175     ShouldNotReachHere();
 176     result = 0;  // unreachable
 177   }
 178   return result;
 179 }
 180 
 181 Address LIR_Assembler::as_Address(LIR_Address* addr, Register tmp) {
 182   Register base = addr->base()->as_pointer_register();
 183   LIR_Opr opr = addr->index();
 184   if (opr->is_cpu_register()) {
 185     Register index;
 186     if (opr->is_single_cpu())
 187       index = opr->as_register();
 188     else
 189       index = opr->as_register_lo();
 190     assert(addr->disp() == 0, "must be");
 191     switch(opr->type()) {
 192       case T_INT:
 193         return Address(base, index, Address::sxtw(addr->scale()));
 194       case T_LONG:
 195         return Address(base, index, Address::lsl(addr->scale()));
 196       default:
 197         ShouldNotReachHere();
 198       }
 199   } else  {
 200     intptr_t addr_offset = intptr_t(addr->disp());
 201     if (Address::offset_ok_for_immed(addr_offset, addr->scale()))
 202       return Address(base, addr_offset, Address::lsl(addr->scale()));
 203     else {
 204       __ mov(tmp, addr_offset);
 205       return Address(base, tmp, Address::lsl(addr->scale()));
 206     }
 207   }
 208   return Address();
 209 }
 210 
 211 Address LIR_Assembler::as_Address_hi(LIR_Address* addr) {
 212   ShouldNotReachHere();
 213   return Address();
 214 }
 215 
 216 Address LIR_Assembler::as_Address(LIR_Address* addr) {
 217   return as_Address(addr, rscratch1);
 218 }
 219 
 220 Address LIR_Assembler::as_Address_lo(LIR_Address* addr) {
 221   return as_Address(addr, rscratch1);  // Ouch
 222   // FIXME: This needs to be much more clever.  See x86.
 223 }
 224 
 225 
 226 void LIR_Assembler::osr_entry() {
 227   offsets()->set_value(CodeOffsets::OSR_Entry, code_offset());
 228   BlockBegin* osr_entry = compilation()->hir()->osr_entry();
 229   ValueStack* entry_state = osr_entry->state();
 230   int number_of_locks = entry_state->locks_size();
 231 
 232   // we jump here if osr happens with the interpreter
 233   // state set up to continue at the beginning of the
 234   // loop that triggered osr - in particular, we have
 235   // the following registers setup:
 236   //
 237   // r2: osr buffer
 238   //
 239 
 240   // build frame
 241   ciMethod* m = compilation()->method();
 242   __ build_frame(initial_frame_size_in_bytes(), bang_size_in_bytes());
 243 
 244   // OSR buffer is
 245   //
 246   // locals[nlocals-1..0]
 247   // monitors[0..number_of_locks]
 248   //
 249   // locals is a direct copy of the interpreter frame so in the osr buffer
 250   // so first slot in the local array is the last local from the interpreter
 251   // and last slot is local[0] (receiver) from the interpreter
 252   //
 253   // Similarly with locks. The first lock slot in the osr buffer is the nth lock
 254   // from the interpreter frame, the nth lock slot in the osr buffer is 0th lock
 255   // in the interpreter frame (the method lock if a sync method)
 256 
 257   // Initialize monitors in the compiled activation.
 258   //   r2: pointer to osr buffer
 259   //
 260   // All other registers are dead at this point and the locals will be
 261   // copied into place by code emitted in the IR.
 262 
 263   Register OSR_buf = osrBufferPointer()->as_pointer_register();
 264   { assert(frame::interpreter_frame_monitor_size() == BasicObjectLock::size(), "adjust code below");
 265     int monitor_offset = BytesPerWord * method()->max_locals() +
 266       (2 * BytesPerWord) * (number_of_locks - 1);
 267     // SharedRuntime::OSR_migration_begin() packs BasicObjectLocks in
 268     // the OSR buffer using 2 word entries: first the lock and then
 269     // the oop.
 270     for (int i = 0; i < number_of_locks; i++) {
 271       int slot_offset = monitor_offset - ((i * 2) * BytesPerWord);
 272 #ifdef ASSERT
 273       // verify the interpreter's monitor has a non-null object
 274       {
 275         Label L;
 276         __ ldr(rscratch1, Address(OSR_buf, slot_offset + 1*BytesPerWord));
 277         __ cbnz(rscratch1, L);
 278         __ stop("locked object is NULL");
 279         __ bind(L);
 280       }
 281 #endif
 282       __ ldr(r19, Address(OSR_buf, slot_offset + 0));
 283       __ str(r19, frame_map()->address_for_monitor_lock(i));
 284       __ ldr(r19, Address(OSR_buf, slot_offset + 1*BytesPerWord));
 285       __ str(r19, frame_map()->address_for_monitor_object(i));
 286     }
 287   }
 288 }
 289 
 290 
 291 // inline cache check; done before the frame is built.
 292 int LIR_Assembler::check_icache() {
 293   Register receiver = FrameMap::receiver_opr->as_register();
 294   Register ic_klass = IC_Klass;
 295   int start_offset = __ offset();
 296   __ inline_cache_check(receiver, ic_klass);
 297 
 298   // if icache check fails, then jump to runtime routine
 299   // Note: RECEIVER must still contain the receiver!
 300   Label dont;
 301   __ br(Assembler::EQ, dont);
 302   __ far_jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
 303 
 304   // We align the verified entry point unless the method body
 305   // (including its inline cache check) will fit in a single 64-byte
 306   // icache line.
 307   if (! method()->is_accessor() || __ offset() - start_offset > 4 * 4) {
 308     // force alignment after the cache check.
 309     __ align(CodeEntryAlignment);
 310   }
 311 
 312   __ bind(dont);
 313   return start_offset;
 314 }
 315 
 316 
 317 void LIR_Assembler::jobject2reg(jobject o, Register reg) {
 318   if (o == NULL) {
 319     __ mov(reg, zr);
 320   } else {
 321     __ movoop(reg, o, /*immediate*/true);
 322   }
 323 }
 324 
 325 void LIR_Assembler::deoptimize_trap(CodeEmitInfo *info) {
 326   address target = NULL;
 327   relocInfo::relocType reloc_type = relocInfo::none;
 328 
 329   switch (patching_id(info)) {
 330   case PatchingStub::access_field_id:
 331     target = Runtime1::entry_for(Runtime1::access_field_patching_id);
 332     reloc_type = relocInfo::section_word_type;
 333     break;
 334   case PatchingStub::load_klass_id:
 335     target = Runtime1::entry_for(Runtime1::load_klass_patching_id);
 336     reloc_type = relocInfo::metadata_type;
 337     break;
 338   case PatchingStub::load_mirror_id:
 339     target = Runtime1::entry_for(Runtime1::load_mirror_patching_id);
 340     reloc_type = relocInfo::oop_type;
 341     break;
 342   case PatchingStub::load_appendix_id:
 343     target = Runtime1::entry_for(Runtime1::load_appendix_patching_id);
 344     reloc_type = relocInfo::oop_type;
 345     break;
 346   default: ShouldNotReachHere();
 347   }
 348 
 349   __ far_call(RuntimeAddress(target));
 350   add_call_info_here(info);
 351 }
 352 
 353 void LIR_Assembler::jobject2reg_with_patching(Register reg, CodeEmitInfo *info) {
 354   deoptimize_trap(info);
 355 }
 356 
 357 
 358 // This specifies the rsp decrement needed to build the frame
 359 int LIR_Assembler::initial_frame_size_in_bytes() const {
 360   // if rounding, must let FrameMap know!
 361 
 362   // The frame_map records size in slots (32bit word)
 363 
 364   // subtract two words to account for return address and link
 365   return (frame_map()->framesize() - (2*VMRegImpl::slots_per_word))  * VMRegImpl::stack_slot_size;
 366 }
 367 
 368 
 369 int LIR_Assembler::emit_exception_handler() {
 370   // if the last instruction is a call (typically to do a throw which
 371   // is coming at the end after block reordering) the return address
 372   // must still point into the code area in order to avoid assertion
 373   // failures when searching for the corresponding bci => add a nop
 374   // (was bug 5/14/1999 - gri)
 375   __ nop();
 376 
 377   // generate code for exception handler
 378   address handler_base = __ start_a_stub(exception_handler_size());
 379   if (handler_base == NULL) {
 380     // not enough space left for the handler
 381     bailout("exception handler overflow");
 382     return -1;
 383   }
 384 
 385   int offset = code_offset();
 386 
 387   // the exception oop and pc are in r0, and r3
 388   // no other registers need to be preserved, so invalidate them
 389   __ invalidate_registers(false, true, true, false, true, true);
 390 
 391   // check that there is really an exception
 392   __ verify_not_null_oop(r0);
 393 
 394   // search an exception handler (r0: exception oop, r3: throwing pc)
 395   __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::handle_exception_from_callee_id)));  __ should_not_reach_here();
 396   guarantee(code_offset() - offset <= exception_handler_size(), "overflow");
 397   __ end_a_stub();
 398 
 399   return offset;
 400 }
 401 
 402 
 403 // Emit the code to remove the frame from the stack in the exception
 404 // unwind path.
 405 int LIR_Assembler::emit_unwind_handler() {
 406 #ifndef PRODUCT
 407   if (CommentedAssembly) {
 408     _masm->block_comment("Unwind handler");
 409   }
 410 #endif
 411 
 412   int offset = code_offset();
 413 
 414   // Fetch the exception from TLS and clear out exception related thread state
 415   __ ldr(r0, Address(rthread, JavaThread::exception_oop_offset()));
 416   __ str(zr, Address(rthread, JavaThread::exception_oop_offset()));
 417   __ str(zr, Address(rthread, JavaThread::exception_pc_offset()));
 418 
 419   __ bind(_unwind_handler_entry);
 420   __ verify_not_null_oop(r0);
 421   if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
 422     __ mov(r19, r0);  // Preserve the exception
 423   }
 424 
 425   // Preform needed unlocking
 426   MonitorExitStub* stub = NULL;
 427   if (method()->is_synchronized()) {
 428     monitor_address(0, FrameMap::r0_opr);
 429     stub = new MonitorExitStub(FrameMap::r0_opr, true, 0);
 430     __ unlock_object(r5, r4, r0, *stub->entry());
 431     __ bind(*stub->continuation());
 432   }
 433 
 434   if (compilation()->env()->dtrace_method_probes()) {
 435     __ call_Unimplemented();
 436 #if 0
 437     __ movptr(Address(rsp, 0), rax);
 438     __ mov_metadata(Address(rsp, sizeof(void*)), method()->constant_encoding());
 439     __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_exit)));
 440 #endif
 441   }
 442 
 443   if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
 444     __ mov(r0, r19);  // Restore the exception
 445   }
 446 
 447   // remove the activation and dispatch to the unwind handler
 448   __ block_comment("remove_frame and dispatch to the unwind handler");
 449   __ remove_frame(initial_frame_size_in_bytes());
 450   __ far_jump(RuntimeAddress(Runtime1::entry_for(Runtime1::unwind_exception_id)));
 451 
 452   // Emit the slow path assembly
 453   if (stub != NULL) {
 454     stub->emit_code(this);
 455   }
 456 
 457   return offset;
 458 }
 459 
 460 
 461 int LIR_Assembler::emit_deopt_handler() {
 462   // if the last instruction is a call (typically to do a throw which
 463   // is coming at the end after block reordering) the return address
 464   // must still point into the code area in order to avoid assertion
 465   // failures when searching for the corresponding bci => add a nop
 466   // (was bug 5/14/1999 - gri)
 467   __ nop();
 468 
 469   // generate code for exception handler
 470   address handler_base = __ start_a_stub(deopt_handler_size());
 471   if (handler_base == NULL) {
 472     // not enough space left for the handler
 473     bailout("deopt handler overflow");
 474     return -1;
 475   }
 476 
 477   int offset = code_offset();
 478 
 479   __ adr(lr, pc());
 480   __ far_jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
 481   guarantee(code_offset() - offset <= deopt_handler_size(), "overflow");
 482   __ end_a_stub();
 483 
 484   return offset;
 485 }
 486 
 487 void LIR_Assembler::add_debug_info_for_branch(address adr, CodeEmitInfo* info) {
 488   _masm->code_section()->relocate(adr, relocInfo::poll_type);
 489   int pc_offset = code_offset();
 490   flush_debug_info(pc_offset);
 491   info->record_debug_info(compilation()->debug_info_recorder(), pc_offset);
 492   if (info->exception_handlers() != NULL) {
 493     compilation()->add_exception_handlers_for_pco(pc_offset, info->exception_handlers());
 494   }
 495 }
 496 
 497 void LIR_Assembler::return_op(LIR_Opr result) {
 498   assert(result->is_illegal() || !result->is_single_cpu() || result->as_register() == r0, "word returns are in r0,");
 499 
 500   // Pop the stack before the safepoint code
 501   __ remove_frame(initial_frame_size_in_bytes());
 502 
 503   if (StackReservedPages > 0 && compilation()->has_reserved_stack_access()) {
 504     __ reserved_stack_check();
 505   }
 506 
 507   address polling_page(os::get_polling_page());
 508   __ read_polling_page(rscratch1, polling_page, relocInfo::poll_return_type);
 509   __ ret(lr);
 510 }
 511 
 512 int LIR_Assembler::safepoint_poll(LIR_Opr tmp, CodeEmitInfo* info) {
 513   address polling_page(os::get_polling_page());
 514   guarantee(info != NULL, "Shouldn't be NULL");
 515   assert(os::is_poll_address(polling_page), "should be");
 516   __ get_polling_page(rscratch1, polling_page, relocInfo::poll_type);
 517   add_debug_info_for_branch(info);  // This isn't just debug info:
 518                                     // it's the oop map
 519   __ read_polling_page(rscratch1, relocInfo::poll_type);
 520   return __ offset();
 521 }
 522 
 523 
 524 void LIR_Assembler::move_regs(Register from_reg, Register to_reg) {
 525   if (from_reg == r31_sp)
 526     from_reg = sp;
 527   if (to_reg == r31_sp)
 528     to_reg = sp;
 529   __ mov(to_reg, from_reg);
 530 }
 531 
 532 void LIR_Assembler::swap_reg(Register a, Register b) { Unimplemented(); }
 533 
 534 
 535 void LIR_Assembler::const2reg(LIR_Opr src, LIR_Opr dest, LIR_PatchCode patch_code, CodeEmitInfo* info) {
 536   assert(src->is_constant(), "should not call otherwise");
 537   assert(dest->is_register(), "should not call otherwise");
 538   LIR_Const* c = src->as_constant_ptr();
 539 
 540   switch (c->type()) {
 541     case T_INT: {
 542       assert(patch_code == lir_patch_none, "no patching handled here");
 543       __ movw(dest->as_register(), c->as_jint());
 544       break;
 545     }
 546 
 547     case T_ADDRESS: {
 548       assert(patch_code == lir_patch_none, "no patching handled here");
 549       __ mov(dest->as_register(), c->as_jint());
 550       break;
 551     }
 552 
 553     case T_LONG: {
 554       assert(patch_code == lir_patch_none, "no patching handled here");
 555       __ mov(dest->as_register_lo(), (intptr_t)c->as_jlong());
 556       break;
 557     }
 558 
 559     case T_OBJECT: {
 560         if (patch_code == lir_patch_none) {
 561           jobject2reg(c->as_jobject(), dest->as_register());
 562         } else {
 563           jobject2reg_with_patching(dest->as_register(), info);
 564         }
 565       break;
 566     }
 567 
 568     case T_METADATA: {
 569       if (patch_code != lir_patch_none) {
 570         klass2reg_with_patching(dest->as_register(), info);
 571       } else {
 572         __ mov_metadata(dest->as_register(), c->as_metadata());
 573       }
 574       break;
 575     }
 576 
 577     case T_FLOAT: {
 578       if (__ operand_valid_for_float_immediate(c->as_jfloat())) {
 579         __ fmovs(dest->as_float_reg(), (c->as_jfloat()));
 580       } else {
 581         __ adr(rscratch1, InternalAddress(float_constant(c->as_jfloat())));
 582         __ ldrs(dest->as_float_reg(), Address(rscratch1));
 583       }
 584       break;
 585     }
 586 
 587     case T_DOUBLE: {
 588       if (__ operand_valid_for_float_immediate(c->as_jdouble())) {
 589         __ fmovd(dest->as_double_reg(), (c->as_jdouble()));
 590       } else {
 591         __ adr(rscratch1, InternalAddress(double_constant(c->as_jdouble())));
 592         __ ldrd(dest->as_double_reg(), Address(rscratch1));
 593       }
 594       break;
 595     }
 596 
 597     default:
 598       ShouldNotReachHere();
 599   }
 600 }
 601 
 602 void LIR_Assembler::const2stack(LIR_Opr src, LIR_Opr dest) {
 603   LIR_Const* c = src->as_constant_ptr();
 604   switch (c->type()) {
 605   case T_OBJECT:
 606     {
 607       if (! c->as_jobject())
 608         __ str(zr, frame_map()->address_for_slot(dest->single_stack_ix()));
 609       else {
 610         const2reg(src, FrameMap::rscratch1_opr, lir_patch_none, NULL);
 611         reg2stack(FrameMap::rscratch1_opr, dest, c->type(), false);
 612       }
 613     }
 614     break;
 615   case T_ADDRESS:
 616     {
 617       const2reg(src, FrameMap::rscratch1_opr, lir_patch_none, NULL);
 618       reg2stack(FrameMap::rscratch1_opr, dest, c->type(), false);
 619     }
 620   case T_INT:
 621   case T_FLOAT:
 622     {
 623       Register reg = zr;
 624       if (c->as_jint_bits() == 0)
 625         __ strw(zr, frame_map()->address_for_slot(dest->single_stack_ix()));
 626       else {
 627         __ movw(rscratch1, c->as_jint_bits());
 628         __ strw(rscratch1, frame_map()->address_for_slot(dest->single_stack_ix()));
 629       }
 630     }
 631     break;
 632   case T_LONG:
 633   case T_DOUBLE:
 634     {
 635       Register reg = zr;
 636       if (c->as_jlong_bits() == 0)
 637         __ str(zr, frame_map()->address_for_slot(dest->double_stack_ix(),
 638                                                  lo_word_offset_in_bytes));
 639       else {
 640         __ mov(rscratch1, (intptr_t)c->as_jlong_bits());
 641         __ str(rscratch1, frame_map()->address_for_slot(dest->double_stack_ix(),
 642                                                         lo_word_offset_in_bytes));
 643       }
 644     }
 645     break;
 646   default:
 647     ShouldNotReachHere();
 648   }
 649 }
 650 
 651 void LIR_Assembler::const2mem(LIR_Opr src, LIR_Opr dest, BasicType type, CodeEmitInfo* info, bool wide) {
 652   assert(src->is_constant(), "should not call otherwise");
 653   LIR_Const* c = src->as_constant_ptr();
 654   LIR_Address* to_addr = dest->as_address_ptr();
 655 
 656   void (Assembler::* insn)(Register Rt, const Address &adr);
 657 
 658   switch (type) {
 659   case T_ADDRESS:
 660     assert(c->as_jint() == 0, "should be");
 661     insn = &Assembler::str;
 662     break;
 663   case T_LONG:
 664     assert(c->as_jlong() == 0, "should be");
 665     insn = &Assembler::str;
 666     break;
 667   case T_INT:
 668     assert(c->as_jint() == 0, "should be");
 669     insn = &Assembler::strw;
 670     break;
 671   case T_OBJECT:
 672   case T_ARRAY:
 673     assert(c->as_jobject() == 0, "should be");
 674     if (UseCompressedOops && !wide) {
 675       insn = &Assembler::strw;
 676     } else {
 677       insn = &Assembler::str;
 678     }
 679     break;
 680   case T_CHAR:
 681   case T_SHORT:
 682     assert(c->as_jint() == 0, "should be");
 683     insn = &Assembler::strh;
 684     break;
 685   case T_BOOLEAN:
 686   case T_BYTE:
 687     assert(c->as_jint() == 0, "should be");
 688     insn = &Assembler::strb;
 689     break;
 690   default:
 691     ShouldNotReachHere();
 692     insn = &Assembler::str;  // unreachable
 693   }
 694 
 695   if (info) add_debug_info_for_null_check_here(info);
 696   (_masm->*insn)(zr, as_Address(to_addr, rscratch1));
 697 }
 698 
 699 void LIR_Assembler::reg2reg(LIR_Opr src, LIR_Opr dest) {
 700   assert(src->is_register(), "should not call otherwise");
 701   assert(dest->is_register(), "should not call otherwise");
 702 
 703   // move between cpu-registers
 704   if (dest->is_single_cpu()) {
 705     if (src->type() == T_LONG) {
 706       // Can do LONG -> OBJECT
 707       move_regs(src->as_register_lo(), dest->as_register());
 708       return;
 709     }
 710     assert(src->is_single_cpu(), "must match");
 711     if (src->type() == T_OBJECT) {
 712       __ verify_oop(src->as_register());
 713     }
 714     move_regs(src->as_register(), dest->as_register());
 715 
 716   } else if (dest->is_double_cpu()) {
 717     if (src->type() == T_OBJECT || src->type() == T_ARRAY) {
 718       // Surprising to me but we can see move of a long to t_object
 719       __ verify_oop(src->as_register());
 720       move_regs(src->as_register(), dest->as_register_lo());
 721       return;
 722     }
 723     assert(src->is_double_cpu(), "must match");
 724     Register f_lo = src->as_register_lo();
 725     Register f_hi = src->as_register_hi();
 726     Register t_lo = dest->as_register_lo();
 727     Register t_hi = dest->as_register_hi();
 728     assert(f_hi == f_lo, "must be same");
 729     assert(t_hi == t_lo, "must be same");
 730     move_regs(f_lo, t_lo);
 731 
 732   } else if (dest->is_single_fpu()) {
 733     __ fmovs(dest->as_float_reg(), src->as_float_reg());
 734 
 735   } else if (dest->is_double_fpu()) {
 736     __ fmovd(dest->as_double_reg(), src->as_double_reg());
 737 
 738   } else {
 739     ShouldNotReachHere();
 740   }
 741 }
 742 
 743 void LIR_Assembler::reg2stack(LIR_Opr src, LIR_Opr dest, BasicType type, bool pop_fpu_stack) {
 744   if (src->is_single_cpu()) {
 745     if (type == T_ARRAY || type == T_OBJECT) {
 746       __ str(src->as_register(), frame_map()->address_for_slot(dest->single_stack_ix()));
 747       __ verify_oop(src->as_register());
 748     } else if (type == T_METADATA || type == T_DOUBLE) {
 749       __ str(src->as_register(), frame_map()->address_for_slot(dest->single_stack_ix()));
 750     } else {
 751       __ strw(src->as_register(), frame_map()->address_for_slot(dest->single_stack_ix()));
 752     }
 753 
 754   } else if (src->is_double_cpu()) {
 755     Address dest_addr_LO = frame_map()->address_for_slot(dest->double_stack_ix(), lo_word_offset_in_bytes);
 756     __ str(src->as_register_lo(), dest_addr_LO);
 757 
 758   } else if (src->is_single_fpu()) {
 759     Address dest_addr = frame_map()->address_for_slot(dest->single_stack_ix());
 760     __ strs(src->as_float_reg(), dest_addr);
 761 
 762   } else if (src->is_double_fpu()) {
 763     Address dest_addr = frame_map()->address_for_slot(dest->double_stack_ix());
 764     __ strd(src->as_double_reg(), dest_addr);
 765 
 766   } else {
 767     ShouldNotReachHere();
 768   }
 769 
 770 }
 771 
 772 
 773 void LIR_Assembler::reg2mem(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_PatchCode patch_code, CodeEmitInfo* info, bool pop_fpu_stack, bool wide, bool /* unaligned */) {
 774   LIR_Address* to_addr = dest->as_address_ptr();
 775   PatchingStub* patch = NULL;
 776   Register compressed_src = rscratch1;
 777 
 778   if (patch_code != lir_patch_none) {
 779     deoptimize_trap(info);
 780     return;
 781   }
 782 
 783   if (type == T_ARRAY || type == T_OBJECT) {
 784     __ verify_oop(src->as_register());
 785 
 786     if (UseCompressedOops && !wide) {
 787       __ encode_heap_oop(compressed_src, src->as_register());
 788     } else {
 789       compressed_src = src->as_register();
 790     }
 791   }
 792 
 793   int null_check_here = code_offset();
 794   switch (type) {
 795     case T_FLOAT: {
 796       __ strs(src->as_float_reg(), as_Address(to_addr));
 797       break;
 798     }
 799 
 800     case T_DOUBLE: {
 801       __ strd(src->as_double_reg(), as_Address(to_addr));
 802       break;
 803     }
 804 
 805     case T_ARRAY:   // fall through
 806     case T_OBJECT:  // fall through
 807       if (UseCompressedOops && !wide) {
 808         __ strw(compressed_src, as_Address(to_addr, rscratch2));
 809       } else {
 810          __ str(compressed_src, as_Address(to_addr));
 811       }
 812       break;
 813     case T_METADATA:
 814       // We get here to store a method pointer to the stack to pass to
 815       // a dtrace runtime call. This can't work on 64 bit with
 816       // compressed klass ptrs: T_METADATA can be a compressed klass
 817       // ptr or a 64 bit method pointer.
 818       ShouldNotReachHere();
 819       __ str(src->as_register(), as_Address(to_addr));
 820       break;
 821     case T_ADDRESS:
 822       __ str(src->as_register(), as_Address(to_addr));
 823       break;
 824     case T_INT:
 825       __ strw(src->as_register(), as_Address(to_addr));
 826       break;
 827 
 828     case T_LONG: {
 829       __ str(src->as_register_lo(), as_Address_lo(to_addr));
 830       break;
 831     }
 832 
 833     case T_BYTE:    // fall through
 834     case T_BOOLEAN: {
 835       __ strb(src->as_register(), as_Address(to_addr));
 836       break;
 837     }
 838 
 839     case T_CHAR:    // fall through
 840     case T_SHORT:
 841       __ strh(src->as_register(), as_Address(to_addr));
 842       break;
 843 
 844     default:
 845       ShouldNotReachHere();
 846   }
 847   if (info != NULL) {
 848     add_debug_info_for_null_check(null_check_here, info);
 849   }
 850 }
 851 
 852 
 853 void LIR_Assembler::stack2reg(LIR_Opr src, LIR_Opr dest, BasicType type) {
 854   assert(src->is_stack(), "should not call otherwise");
 855   assert(dest->is_register(), "should not call otherwise");
 856 
 857   if (dest->is_single_cpu()) {
 858     if (type == T_ARRAY || type == T_OBJECT) {
 859       __ ldr(dest->as_register(), frame_map()->address_for_slot(src->single_stack_ix()));
 860       __ verify_oop(dest->as_register());
 861     } else if (type == T_METADATA) {
 862       __ ldr(dest->as_register(), frame_map()->address_for_slot(src->single_stack_ix()));
 863     } else {
 864       __ ldrw(dest->as_register(), frame_map()->address_for_slot(src->single_stack_ix()));
 865     }
 866 
 867   } else if (dest->is_double_cpu()) {
 868     Address src_addr_LO = frame_map()->address_for_slot(src->double_stack_ix(), lo_word_offset_in_bytes);
 869     __ ldr(dest->as_register_lo(), src_addr_LO);
 870 
 871   } else if (dest->is_single_fpu()) {
 872     Address src_addr = frame_map()->address_for_slot(src->single_stack_ix());
 873     __ ldrs(dest->as_float_reg(), src_addr);
 874 
 875   } else if (dest->is_double_fpu()) {
 876     Address src_addr = frame_map()->address_for_slot(src->double_stack_ix());
 877     __ ldrd(dest->as_double_reg(), src_addr);
 878 
 879   } else {
 880     ShouldNotReachHere();
 881   }
 882 }
 883 
 884 
 885 void LIR_Assembler::klass2reg_with_patching(Register reg, CodeEmitInfo* info) {
 886   address target = NULL;
 887   relocInfo::relocType reloc_type = relocInfo::none;
 888 
 889   switch (patching_id(info)) {
 890   case PatchingStub::access_field_id:
 891     target = Runtime1::entry_for(Runtime1::access_field_patching_id);
 892     reloc_type = relocInfo::section_word_type;
 893     break;
 894   case PatchingStub::load_klass_id:
 895     target = Runtime1::entry_for(Runtime1::load_klass_patching_id);
 896     reloc_type = relocInfo::metadata_type;
 897     break;
 898   case PatchingStub::load_mirror_id:
 899     target = Runtime1::entry_for(Runtime1::load_mirror_patching_id);
 900     reloc_type = relocInfo::oop_type;
 901     break;
 902   case PatchingStub::load_appendix_id:
 903     target = Runtime1::entry_for(Runtime1::load_appendix_patching_id);
 904     reloc_type = relocInfo::oop_type;
 905     break;
 906   default: ShouldNotReachHere();
 907   }
 908 
 909   __ far_call(RuntimeAddress(target));
 910   add_call_info_here(info);
 911 }
 912 
 913 void LIR_Assembler::stack2stack(LIR_Opr src, LIR_Opr dest, BasicType type) {
 914 
 915   LIR_Opr temp;
 916   if (type == T_LONG || type == T_DOUBLE)
 917     temp = FrameMap::rscratch1_long_opr;
 918   else
 919     temp = FrameMap::rscratch1_opr;
 920 
 921   stack2reg(src, temp, src->type());
 922   reg2stack(temp, dest, dest->type(), false);
 923 }
 924 
 925 
 926 void LIR_Assembler::mem2reg(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_PatchCode patch_code, CodeEmitInfo* info, bool wide, bool /* unaligned */) {
 927   LIR_Address* addr = src->as_address_ptr();
 928   LIR_Address* from_addr = src->as_address_ptr();
 929 
 930   if (addr->base()->type() == T_OBJECT) {
 931     __ verify_oop(addr->base()->as_pointer_register());
 932   }
 933 
 934   if (patch_code != lir_patch_none) {
 935     deoptimize_trap(info);
 936     return;
 937   }
 938 
 939   if (info != NULL) {
 940     add_debug_info_for_null_check_here(info);
 941   }
 942   int null_check_here = code_offset();
 943   switch (type) {
 944     case T_FLOAT: {
 945       __ ldrs(dest->as_float_reg(), as_Address(from_addr));
 946       break;
 947     }
 948 
 949     case T_DOUBLE: {
 950       __ ldrd(dest->as_double_reg(), as_Address(from_addr));
 951       break;
 952     }
 953 
 954     case T_ARRAY:   // fall through
 955     case T_OBJECT:  // fall through
 956       if (UseCompressedOops && !wide) {
 957         __ ldrw(dest->as_register(), as_Address(from_addr));
 958       } else {
 959          __ ldr(dest->as_register(), as_Address(from_addr));
 960       }
 961       break;
 962     case T_METADATA:
 963       // We get here to store a method pointer to the stack to pass to
 964       // a dtrace runtime call. This can't work on 64 bit with
 965       // compressed klass ptrs: T_METADATA can be a compressed klass
 966       // ptr or a 64 bit method pointer.
 967       ShouldNotReachHere();
 968       __ ldr(dest->as_register(), as_Address(from_addr));
 969       break;
 970     case T_ADDRESS:
 971       // FIXME: OMG this is a horrible kludge.  Any offset from an
 972       // address that matches klass_offset_in_bytes() will be loaded
 973       // as a word, not a long.
 974       if (UseCompressedClassPointers && addr->disp() == oopDesc::klass_offset_in_bytes()) {
 975         __ ldrw(dest->as_register(), as_Address(from_addr));
 976       } else {
 977         __ ldr(dest->as_register(), as_Address(from_addr));
 978       }
 979       break;
 980     case T_INT:
 981       __ ldrw(dest->as_register(), as_Address(from_addr));
 982       break;
 983 
 984     case T_LONG: {
 985       __ ldr(dest->as_register_lo(), as_Address_lo(from_addr));
 986       break;
 987     }
 988 
 989     case T_BYTE:
 990       __ ldrsb(dest->as_register(), as_Address(from_addr));
 991       break;
 992     case T_BOOLEAN: {
 993       __ ldrb(dest->as_register(), as_Address(from_addr));
 994       break;
 995     }
 996 
 997     case T_CHAR:
 998       __ ldrh(dest->as_register(), as_Address(from_addr));
 999       break;
1000     case T_SHORT:
1001       __ ldrsh(dest->as_register(), as_Address(from_addr));
1002       break;
1003 
1004     default:
1005       ShouldNotReachHere();
1006   }
1007 
1008   if (type == T_ARRAY || type == T_OBJECT) {
1009     if (UseCompressedOops && !wide) {
1010       __ decode_heap_oop(dest->as_register());
1011     }
1012     __ verify_oop(dest->as_register());
1013   } else if (type == T_ADDRESS && addr->disp() == oopDesc::klass_offset_in_bytes()) {
1014     if (UseCompressedClassPointers) {
1015       __ decode_klass_not_null(dest->as_register());
1016     }
1017   }
1018 }
1019 
1020 
1021 int LIR_Assembler::array_element_size(BasicType type) const {
1022   int elem_size = type2aelembytes(type);
1023   return exact_log2(elem_size);
1024 }
1025 
1026 void LIR_Assembler::arithmetic_idiv(LIR_Op3* op, bool is_irem) {
1027   Register Rdividend = op->in_opr1()->as_register();
1028   Register Rdivisor  = op->in_opr2()->as_register();
1029   Register Rscratch  = op->in_opr3()->as_register();
1030   Register Rresult   = op->result_opr()->as_register();
1031   int divisor = -1;
1032 
1033   /*
1034   TODO: For some reason, using the Rscratch that gets passed in is
1035   not possible because the register allocator does not see the tmp reg
1036   as used, and assignes it the same register as Rdividend. We use rscratch1
1037    instead.
1038 
1039   assert(Rdividend != Rscratch, "");
1040   assert(Rdivisor  != Rscratch, "");
1041   */
1042 
1043   if (Rdivisor == noreg && is_power_of_2(divisor)) {
1044     // convert division by a power of two into some shifts and logical operations
1045   }
1046 
1047   __ corrected_idivl(Rresult, Rdividend, Rdivisor, is_irem, rscratch1);
1048 }
1049 
1050 void LIR_Assembler::emit_op3(LIR_Op3* op) {
1051   switch (op->code()) {
1052   case lir_idiv:
1053     arithmetic_idiv(op, false);
1054     break;
1055   case lir_irem:
1056     arithmetic_idiv(op, true);
1057     break;
1058   case lir_fmad:
1059     __ fmaddd(op->result_opr()->as_double_reg(),
1060               op->in_opr1()->as_double_reg(),
1061               op->in_opr2()->as_double_reg(),
1062               op->in_opr3()->as_double_reg());
1063     break;
1064   case lir_fmaf:
1065     __ fmadds(op->result_opr()->as_float_reg(),
1066               op->in_opr1()->as_float_reg(),
1067               op->in_opr2()->as_float_reg(),
1068               op->in_opr3()->as_float_reg());
1069     break;
1070   default:      ShouldNotReachHere(); break;
1071   }
1072 }
1073 
1074 void LIR_Assembler::emit_opBranch(LIR_OpBranch* op) {
1075 #ifdef ASSERT
1076   assert(op->block() == NULL || op->block()->label() == op->label(), "wrong label");
1077   if (op->block() != NULL)  _branch_target_blocks.append(op->block());
1078   if (op->ublock() != NULL) _branch_target_blocks.append(op->ublock());
1079 #endif
1080 
1081   if (op->cond() == lir_cond_always) {
1082     if (op->info() != NULL) add_debug_info_for_branch(op->info());
1083     __ b(*(op->label()));
1084   } else {
1085     Assembler::Condition acond;
1086     if (op->code() == lir_cond_float_branch) {
1087       bool is_unordered = (op->ublock() == op->block());
1088       // Assembler::EQ does not permit unordered branches, so we add
1089       // another branch here.  Likewise, Assembler::NE does not permit
1090       // ordered branches.
1091       if (is_unordered && op->cond() == lir_cond_equal
1092           || !is_unordered && op->cond() == lir_cond_notEqual)
1093         __ br(Assembler::VS, *(op->ublock()->label()));
1094       switch(op->cond()) {
1095       case lir_cond_equal:        acond = Assembler::EQ; break;
1096       case lir_cond_notEqual:     acond = Assembler::NE; break;
1097       case lir_cond_less:         acond = (is_unordered ? Assembler::LT : Assembler::LO); break;
1098       case lir_cond_lessEqual:    acond = (is_unordered ? Assembler::LE : Assembler::LS); break;
1099       case lir_cond_greaterEqual: acond = (is_unordered ? Assembler::HS : Assembler::GE); break;
1100       case lir_cond_greater:      acond = (is_unordered ? Assembler::HI : Assembler::GT); break;
1101       default:                    ShouldNotReachHere();
1102         acond = Assembler::EQ;  // unreachable
1103       }
1104     } else {
1105       switch (op->cond()) {
1106         case lir_cond_equal:        acond = Assembler::EQ; break;
1107         case lir_cond_notEqual:     acond = Assembler::NE; break;
1108         case lir_cond_less:         acond = Assembler::LT; break;
1109         case lir_cond_lessEqual:    acond = Assembler::LE; break;
1110         case lir_cond_greaterEqual: acond = Assembler::GE; break;
1111         case lir_cond_greater:      acond = Assembler::GT; break;
1112         case lir_cond_belowEqual:   acond = Assembler::LS; break;
1113         case lir_cond_aboveEqual:   acond = Assembler::HS; break;
1114         default:                    ShouldNotReachHere();
1115           acond = Assembler::EQ;  // unreachable
1116       }
1117     }
1118     __ br(acond,*(op->label()));
1119   }
1120 }
1121 
1122 
1123 
1124 void LIR_Assembler::emit_opConvert(LIR_OpConvert* op) {
1125   LIR_Opr src  = op->in_opr();
1126   LIR_Opr dest = op->result_opr();
1127 
1128   switch (op->bytecode()) {
1129     case Bytecodes::_i2f:
1130       {
1131         __ scvtfws(dest->as_float_reg(), src->as_register());
1132         break;
1133       }
1134     case Bytecodes::_i2d:
1135       {
1136         __ scvtfwd(dest->as_double_reg(), src->as_register());
1137         break;
1138       }
1139     case Bytecodes::_l2d:
1140       {
1141         __ scvtfd(dest->as_double_reg(), src->as_register_lo());
1142         break;
1143       }
1144     case Bytecodes::_l2f:
1145       {
1146         __ scvtfs(dest->as_float_reg(), src->as_register_lo());
1147         break;
1148       }
1149     case Bytecodes::_f2d:
1150       {
1151         __ fcvts(dest->as_double_reg(), src->as_float_reg());
1152         break;
1153       }
1154     case Bytecodes::_d2f:
1155       {
1156         __ fcvtd(dest->as_float_reg(), src->as_double_reg());
1157         break;
1158       }
1159     case Bytecodes::_i2c:
1160       {
1161         __ ubfx(dest->as_register(), src->as_register(), 0, 16);
1162         break;
1163       }
1164     case Bytecodes::_i2l:
1165       {
1166         __ sxtw(dest->as_register_lo(), src->as_register());
1167         break;
1168       }
1169     case Bytecodes::_i2s:
1170       {
1171         __ sxth(dest->as_register(), src->as_register());
1172         break;
1173       }
1174     case Bytecodes::_i2b:
1175       {
1176         __ sxtb(dest->as_register(), src->as_register());
1177         break;
1178       }
1179     case Bytecodes::_l2i:
1180       {
1181         _masm->block_comment("FIXME: This could be a no-op");
1182         __ uxtw(dest->as_register(), src->as_register_lo());
1183         break;
1184       }
1185     case Bytecodes::_d2l:
1186       {
1187         __ fcvtzd(dest->as_register_lo(), src->as_double_reg());
1188         break;
1189       }
1190     case Bytecodes::_f2i:
1191       {
1192         __ fcvtzsw(dest->as_register(), src->as_float_reg());
1193         break;
1194       }
1195     case Bytecodes::_f2l:
1196       {
1197         __ fcvtzs(dest->as_register_lo(), src->as_float_reg());
1198         break;
1199       }
1200     case Bytecodes::_d2i:
1201       {
1202         __ fcvtzdw(dest->as_register(), src->as_double_reg());
1203         break;
1204       }
1205     default: ShouldNotReachHere();
1206   }
1207 }
1208 
1209 void LIR_Assembler::emit_alloc_obj(LIR_OpAllocObj* op) {
1210   if (op->init_check()) {
1211     __ ldrb(rscratch1, Address(op->klass()->as_register(),
1212                                InstanceKlass::init_state_offset()));
1213     __ cmpw(rscratch1, InstanceKlass::fully_initialized);
1214     add_debug_info_for_null_check_here(op->stub()->info());
1215     __ br(Assembler::NE, *op->stub()->entry());
1216   }
1217   __ allocate_object(op->obj()->as_register(),
1218                      op->tmp1()->as_register(),
1219                      op->tmp2()->as_register(),
1220                      op->header_size(),
1221                      op->object_size(),
1222                      op->klass()->as_register(),
1223                      *op->stub()->entry());
1224   __ bind(*op->stub()->continuation());
1225 }
1226 
1227 void LIR_Assembler::emit_alloc_array(LIR_OpAllocArray* op) {
1228   Register len =  op->len()->as_register();
1229   __ uxtw(len, len);
1230 
1231   if (UseSlowPath ||
1232       (!UseFastNewObjectArray && (op->type() == T_OBJECT || op->type() == T_ARRAY)) ||
1233       (!UseFastNewTypeArray   && (op->type() != T_OBJECT && op->type() != T_ARRAY))) {
1234     __ b(*op->stub()->entry());
1235   } else {
1236     Register tmp1 = op->tmp1()->as_register();
1237     Register tmp2 = op->tmp2()->as_register();
1238     Register tmp3 = op->tmp3()->as_register();
1239     if (len == tmp1) {
1240       tmp1 = tmp3;
1241     } else if (len == tmp2) {
1242       tmp2 = tmp3;
1243     } else if (len == tmp3) {
1244       // everything is ok
1245     } else {
1246       __ mov(tmp3, len);
1247     }
1248     __ allocate_array(op->obj()->as_register(),
1249                       len,
1250                       tmp1,
1251                       tmp2,
1252                       arrayOopDesc::header_size(op->type()),
1253                       array_element_size(op->type()),
1254                       op->klass()->as_register(),
1255                       *op->stub()->entry());
1256   }
1257   __ bind(*op->stub()->continuation());
1258 }
1259 
1260 void LIR_Assembler::type_profile_helper(Register mdo,
1261                                         ciMethodData *md, ciProfileData *data,
1262                                         Register recv, Label* update_done) {
1263   for (uint i = 0; i < ReceiverTypeData::row_limit(); i++) {
1264     Label next_test;
1265     // See if the receiver is receiver[n].
1266     __ lea(rscratch2, Address(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_offset(i))));
1267     __ ldr(rscratch1, Address(rscratch2));
1268     __ cmp(recv, rscratch1);
1269     __ br(Assembler::NE, next_test);
1270     Address data_addr(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_count_offset(i)));
1271     __ addptr(data_addr, DataLayout::counter_increment);
1272     __ b(*update_done);
1273     __ bind(next_test);
1274   }
1275 
1276   // Didn't find receiver; find next empty slot and fill it in
1277   for (uint i = 0; i < ReceiverTypeData::row_limit(); i++) {
1278     Label next_test;
1279     __ lea(rscratch2,
1280            Address(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_offset(i))));
1281     Address recv_addr(rscratch2);
1282     __ ldr(rscratch1, recv_addr);
1283     __ cbnz(rscratch1, next_test);
1284     __ str(recv, recv_addr);
1285     __ mov(rscratch1, DataLayout::counter_increment);
1286     __ lea(rscratch2, Address(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_count_offset(i))));
1287     __ str(rscratch1, Address(rscratch2));
1288     __ b(*update_done);
1289     __ bind(next_test);
1290   }
1291 }
1292 
1293 void LIR_Assembler::emit_typecheck_helper(LIR_OpTypeCheck *op, Label* success, Label* failure, Label* obj_is_null) {
1294   // we always need a stub for the failure case.
1295   CodeStub* stub = op->stub();
1296   Register obj = op->object()->as_register();
1297   Register k_RInfo = op->tmp1()->as_register();
1298   Register klass_RInfo = op->tmp2()->as_register();
1299   Register dst = op->result_opr()->as_register();
1300   ciKlass* k = op->klass();
1301   Register Rtmp1 = noreg;
1302 
1303   // check if it needs to be profiled
1304   ciMethodData* md;
1305   ciProfileData* data;
1306 
1307   const bool should_profile = op->should_profile();
1308 
1309   if (should_profile) {
1310     ciMethod* method = op->profiled_method();
1311     assert(method != NULL, "Should have method");
1312     int bci = op->profiled_bci();
1313     md = method->method_data_or_null();
1314     assert(md != NULL, "Sanity");
1315     data = md->bci_to_data(bci);
1316     assert(data != NULL,                "need data for type check");
1317     assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
1318   }
1319   Label profile_cast_success, profile_cast_failure;
1320   Label *success_target = should_profile ? &profile_cast_success : success;
1321   Label *failure_target = should_profile ? &profile_cast_failure : failure;
1322 
1323   if (obj == k_RInfo) {
1324     k_RInfo = dst;
1325   } else if (obj == klass_RInfo) {
1326     klass_RInfo = dst;
1327   }
1328   if (k->is_loaded() && !UseCompressedClassPointers) {
1329     select_different_registers(obj, dst, k_RInfo, klass_RInfo);
1330   } else {
1331     Rtmp1 = op->tmp3()->as_register();
1332     select_different_registers(obj, dst, k_RInfo, klass_RInfo, Rtmp1);
1333   }
1334 
1335   assert_different_registers(obj, k_RInfo, klass_RInfo);
1336 
1337     if (should_profile) {
1338       Label not_null;
1339       __ cbnz(obj, not_null);
1340       // Object is null; update MDO and exit
1341       Register mdo  = klass_RInfo;
1342       __ mov_metadata(mdo, md->constant_encoding());
1343       Address data_addr
1344         = __ form_address(rscratch2, mdo,
1345                           md->byte_offset_of_slot(data, DataLayout::DataLayout::header_offset()),
1346                           LogBytesPerWord);
1347       int header_bits = DataLayout::flag_mask_to_header_mask(BitData::null_seen_byte_constant());
1348       __ ldr(rscratch1, data_addr);
1349       __ orr(rscratch1, rscratch1, header_bits);
1350       __ str(rscratch1, data_addr);
1351       __ b(*obj_is_null);
1352       __ bind(not_null);
1353     } else {
1354       __ cbz(obj, *obj_is_null);
1355     }
1356 
1357   if (!k->is_loaded()) {
1358     klass2reg_with_patching(k_RInfo, op->info_for_patch());
1359   } else {
1360     __ mov_metadata(k_RInfo, k->constant_encoding());
1361   }
1362   __ verify_oop(obj);
1363 
1364   if (op->fast_check()) {
1365     // get object class
1366     // not a safepoint as obj null check happens earlier
1367     __ load_klass(rscratch1, obj);
1368     __ cmp( rscratch1, k_RInfo);
1369 
1370     __ br(Assembler::NE, *failure_target);
1371     // successful cast, fall through to profile or jump
1372   } else {
1373     // get object class
1374     // not a safepoint as obj null check happens earlier
1375     __ load_klass(klass_RInfo, obj);
1376     if (k->is_loaded()) {
1377       // See if we get an immediate positive hit
1378       __ ldr(rscratch1, Address(klass_RInfo, long(k->super_check_offset())));
1379       __ cmp(k_RInfo, rscratch1);
1380       if ((juint)in_bytes(Klass::secondary_super_cache_offset()) != k->super_check_offset()) {
1381         __ br(Assembler::NE, *failure_target);
1382         // successful cast, fall through to profile or jump
1383       } else {
1384         // See if we get an immediate positive hit
1385         __ br(Assembler::EQ, *success_target);
1386         // check for self
1387         __ cmp(klass_RInfo, k_RInfo);
1388         __ br(Assembler::EQ, *success_target);
1389 
1390         __ stp(klass_RInfo, k_RInfo, Address(__ pre(sp, -2 * wordSize)));
1391         __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
1392         __ ldr(klass_RInfo, Address(__ post(sp, 2 * wordSize)));
1393         // result is a boolean
1394         __ cbzw(klass_RInfo, *failure_target);
1395         // successful cast, fall through to profile or jump
1396       }
1397     } else {
1398       // perform the fast part of the checking logic
1399       __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, NULL);
1400       // call out-of-line instance of __ check_klass_subtype_slow_path(...):
1401       __ stp(klass_RInfo, k_RInfo, Address(__ pre(sp, -2 * wordSize)));
1402       __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
1403       __ ldp(k_RInfo, klass_RInfo, Address(__ post(sp, 2 * wordSize)));
1404       // result is a boolean
1405       __ cbz(k_RInfo, *failure_target);
1406       // successful cast, fall through to profile or jump
1407     }
1408   }
1409   if (should_profile) {
1410     Register mdo  = klass_RInfo, recv = k_RInfo;
1411     __ bind(profile_cast_success);
1412     __ mov_metadata(mdo, md->constant_encoding());
1413     __ load_klass(recv, obj);
1414     Label update_done;
1415     type_profile_helper(mdo, md, data, recv, success);
1416     __ b(*success);
1417 
1418     __ bind(profile_cast_failure);
1419     __ mov_metadata(mdo, md->constant_encoding());
1420     Address counter_addr
1421       = __ form_address(rscratch2, mdo,
1422                         md->byte_offset_of_slot(data, CounterData::count_offset()),
1423                         LogBytesPerWord);
1424     __ ldr(rscratch1, counter_addr);
1425     __ sub(rscratch1, rscratch1, DataLayout::counter_increment);
1426     __ str(rscratch1, counter_addr);
1427     __ b(*failure);
1428   }
1429   __ b(*success);
1430 }
1431 
1432 
1433 void LIR_Assembler::emit_opTypeCheck(LIR_OpTypeCheck* op) {
1434   const bool should_profile = op->should_profile();
1435 
1436   LIR_Code code = op->code();
1437   if (code == lir_store_check) {
1438     Register value = op->object()->as_register();
1439     Register array = op->array()->as_register();
1440     Register k_RInfo = op->tmp1()->as_register();
1441     Register klass_RInfo = op->tmp2()->as_register();
1442     Register Rtmp1 = op->tmp3()->as_register();
1443 
1444     CodeStub* stub = op->stub();
1445 
1446     // check if it needs to be profiled
1447     ciMethodData* md;
1448     ciProfileData* data;
1449 
1450     if (should_profile) {
1451       ciMethod* method = op->profiled_method();
1452       assert(method != NULL, "Should have method");
1453       int bci = op->profiled_bci();
1454       md = method->method_data_or_null();
1455       assert(md != NULL, "Sanity");
1456       data = md->bci_to_data(bci);
1457       assert(data != NULL,                "need data for type check");
1458       assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
1459     }
1460     Label profile_cast_success, profile_cast_failure, done;
1461     Label *success_target = should_profile ? &profile_cast_success : &done;
1462     Label *failure_target = should_profile ? &profile_cast_failure : stub->entry();
1463 
1464     if (should_profile) {
1465       Label not_null;
1466       __ cbnz(value, not_null);
1467       // Object is null; update MDO and exit
1468       Register mdo  = klass_RInfo;
1469       __ mov_metadata(mdo, md->constant_encoding());
1470       Address data_addr
1471         = __ form_address(rscratch2, mdo,
1472                           md->byte_offset_of_slot(data, DataLayout::header_offset()),
1473                           LogBytesPerInt);
1474       int header_bits = DataLayout::flag_mask_to_header_mask(BitData::null_seen_byte_constant());
1475       __ ldrw(rscratch1, data_addr);
1476       __ orrw(rscratch1, rscratch1, header_bits);
1477       __ strw(rscratch1, data_addr);
1478       __ b(done);
1479       __ bind(not_null);
1480     } else {
1481       __ cbz(value, done);
1482     }
1483 
1484     add_debug_info_for_null_check_here(op->info_for_exception());
1485     __ load_klass(k_RInfo, array);
1486     __ load_klass(klass_RInfo, value);
1487 
1488     // get instance klass (it's already uncompressed)
1489     __ ldr(k_RInfo, Address(k_RInfo, ObjArrayKlass::element_klass_offset()));
1490     // perform the fast part of the checking logic
1491     __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, NULL);
1492     // call out-of-line instance of __ check_klass_subtype_slow_path(...):
1493     __ stp(klass_RInfo, k_RInfo, Address(__ pre(sp, -2 * wordSize)));
1494     __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
1495     __ ldp(k_RInfo, klass_RInfo, Address(__ post(sp, 2 * wordSize)));
1496     // result is a boolean
1497     __ cbzw(k_RInfo, *failure_target);
1498     // fall through to the success case
1499 
1500     if (should_profile) {
1501       Register mdo  = klass_RInfo, recv = k_RInfo;
1502       __ bind(profile_cast_success);
1503       __ mov_metadata(mdo, md->constant_encoding());
1504       __ load_klass(recv, value);
1505       Label update_done;
1506       type_profile_helper(mdo, md, data, recv, &done);
1507       __ b(done);
1508 
1509       __ bind(profile_cast_failure);
1510       __ mov_metadata(mdo, md->constant_encoding());
1511       Address counter_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
1512       __ lea(rscratch2, counter_addr);
1513       __ ldr(rscratch1, Address(rscratch2));
1514       __ sub(rscratch1, rscratch1, DataLayout::counter_increment);
1515       __ str(rscratch1, Address(rscratch2));
1516       __ b(*stub->entry());
1517     }
1518 
1519     __ bind(done);
1520   } else if (code == lir_checkcast) {
1521     Register obj = op->object()->as_register();
1522     Register dst = op->result_opr()->as_register();
1523     Label success;
1524     emit_typecheck_helper(op, &success, op->stub()->entry(), &success);
1525     __ bind(success);
1526     if (dst != obj) {
1527       __ mov(dst, obj);
1528     }
1529   } else if (code == lir_instanceof) {
1530     Register obj = op->object()->as_register();
1531     Register dst = op->result_opr()->as_register();
1532     Label success, failure, done;
1533     emit_typecheck_helper(op, &success, &failure, &failure);
1534     __ bind(failure);
1535     __ mov(dst, zr);
1536     __ b(done);
1537     __ bind(success);
1538     __ mov(dst, 1);
1539     __ bind(done);
1540   } else {
1541     ShouldNotReachHere();
1542   }
1543 }
1544 
1545 void LIR_Assembler::casw(Register addr, Register newval, Register cmpval) {
1546   __ cmpxchg(addr, cmpval, newval, Assembler::word, /* acquire*/ true, /* release*/ true, /* weak*/ false, rscratch1);
1547   __ cset(rscratch1, Assembler::NE);
1548   __ membar(__ AnyAny);
1549 }
1550 
1551 void LIR_Assembler::casl(Register addr, Register newval, Register cmpval) {
1552   __ cmpxchg(addr, cmpval, newval, Assembler::xword, /* acquire*/ true, /* release*/ true, /* weak*/ false, rscratch1);
1553   __ cset(rscratch1, Assembler::NE);
1554   __ membar(__ AnyAny);
1555 }
1556 
1557 
1558 void LIR_Assembler::emit_compare_and_swap(LIR_OpCompareAndSwap* op) {
1559   assert(VM_Version::supports_cx8(), "wrong machine");
1560   Register addr = as_reg(op->addr());
1561   Register newval = as_reg(op->new_value());
1562   Register cmpval = as_reg(op->cmp_value());
1563   Label succeed, fail, around;
1564 
1565   if (op->code() == lir_cas_obj) {
1566     if (UseCompressedOops) {
1567       Register t1 = op->tmp1()->as_register();
1568       assert(op->tmp1()->is_valid(), "must be");
1569       __ encode_heap_oop(t1, cmpval);
1570       cmpval = t1;
1571       __ encode_heap_oop(rscratch2, newval);
1572       newval = rscratch2;
1573       casw(addr, newval, cmpval);
1574     } else {
1575       casl(addr, newval, cmpval);
1576     }
1577   } else if (op->code() == lir_cas_int) {
1578     casw(addr, newval, cmpval);
1579   } else {
1580     casl(addr, newval, cmpval);
1581   }
1582 }
1583 
1584 
1585 void LIR_Assembler::cmove(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Opr result, BasicType type) {
1586 
1587   Assembler::Condition acond, ncond;
1588   switch (condition) {
1589   case lir_cond_equal:        acond = Assembler::EQ; ncond = Assembler::NE; break;
1590   case lir_cond_notEqual:     acond = Assembler::NE; ncond = Assembler::EQ; break;
1591   case lir_cond_less:         acond = Assembler::LT; ncond = Assembler::GE; break;
1592   case lir_cond_lessEqual:    acond = Assembler::LE; ncond = Assembler::GT; break;
1593   case lir_cond_greaterEqual: acond = Assembler::GE; ncond = Assembler::LT; break;
1594   case lir_cond_greater:      acond = Assembler::GT; ncond = Assembler::LE; break;
1595   case lir_cond_belowEqual:
1596   case lir_cond_aboveEqual:
1597   default:                    ShouldNotReachHere();
1598     acond = Assembler::EQ; ncond = Assembler::NE;  // unreachable
1599   }
1600 
1601   assert(result->is_single_cpu() || result->is_double_cpu(),
1602          "expect single register for result");
1603   if (opr1->is_constant() && opr2->is_constant()
1604       && opr1->type() == T_INT && opr2->type() == T_INT) {
1605     jint val1 = opr1->as_jint();
1606     jint val2 = opr2->as_jint();
1607     if (val1 == 0 && val2 == 1) {
1608       __ cset(result->as_register(), ncond);
1609       return;
1610     } else if (val1 == 1 && val2 == 0) {
1611       __ cset(result->as_register(), acond);
1612       return;
1613     }
1614   }
1615 
1616   if (opr1->is_constant() && opr2->is_constant()
1617       && opr1->type() == T_LONG && opr2->type() == T_LONG) {
1618     jlong val1 = opr1->as_jlong();
1619     jlong val2 = opr2->as_jlong();
1620     if (val1 == 0 && val2 == 1) {
1621       __ cset(result->as_register_lo(), ncond);
1622       return;
1623     } else if (val1 == 1 && val2 == 0) {
1624       __ cset(result->as_register_lo(), acond);
1625       return;
1626     }
1627   }
1628 
1629   if (opr1->is_stack()) {
1630     stack2reg(opr1, FrameMap::rscratch1_opr, result->type());
1631     opr1 = FrameMap::rscratch1_opr;
1632   } else if (opr1->is_constant()) {
1633     LIR_Opr tmp
1634       = opr1->type() == T_LONG ? FrameMap::rscratch1_long_opr : FrameMap::rscratch1_opr;
1635     const2reg(opr1, tmp, lir_patch_none, NULL);
1636     opr1 = tmp;
1637   }
1638 
1639   if (opr2->is_stack()) {
1640     stack2reg(opr2, FrameMap::rscratch2_opr, result->type());
1641     opr2 = FrameMap::rscratch2_opr;
1642   } else if (opr2->is_constant()) {
1643     LIR_Opr tmp
1644       = opr2->type() == T_LONG ? FrameMap::rscratch2_long_opr : FrameMap::rscratch2_opr;
1645     const2reg(opr2, tmp, lir_patch_none, NULL);
1646     opr2 = tmp;
1647   }
1648 
1649   if (result->type() == T_LONG)
1650     __ csel(result->as_register_lo(), opr1->as_register_lo(), opr2->as_register_lo(), acond);
1651   else
1652     __ csel(result->as_register(), opr1->as_register(), opr2->as_register(), acond);
1653 }
1654 
1655 void LIR_Assembler::arith_op(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dest, CodeEmitInfo* info, bool pop_fpu_stack) {
1656   assert(info == NULL, "should never be used, idiv/irem and ldiv/lrem not handled by this method");
1657 
1658   if (left->is_single_cpu()) {
1659     Register lreg = left->as_register();
1660     Register dreg = as_reg(dest);
1661 
1662     if (right->is_single_cpu()) {
1663       // cpu register - cpu register
1664 
1665       assert(left->type() == T_INT && right->type() == T_INT && dest->type() == T_INT,
1666              "should be");
1667       Register rreg = right->as_register();
1668       switch (code) {
1669       case lir_add: __ addw (dest->as_register(), lreg, rreg); break;
1670       case lir_sub: __ subw (dest->as_register(), lreg, rreg); break;
1671       case lir_mul: __ mulw (dest->as_register(), lreg, rreg); break;
1672       default:      ShouldNotReachHere();
1673       }
1674 
1675     } else if (right->is_double_cpu()) {
1676       Register rreg = right->as_register_lo();
1677       // single_cpu + double_cpu: can happen with obj+long
1678       assert(code == lir_add || code == lir_sub, "mismatched arithmetic op");
1679       switch (code) {
1680       case lir_add: __ add(dreg, lreg, rreg); break;
1681       case lir_sub: __ sub(dreg, lreg, rreg); break;
1682       default: ShouldNotReachHere();
1683       }
1684     } else if (right->is_constant()) {
1685       // cpu register - constant
1686       jlong c;
1687 
1688       // FIXME.  This is fugly: we really need to factor all this logic.
1689       switch(right->type()) {
1690       case T_LONG:
1691         c = right->as_constant_ptr()->as_jlong();
1692         break;
1693       case T_INT:
1694       case T_ADDRESS:
1695         c = right->as_constant_ptr()->as_jint();
1696         break;
1697       default:
1698         ShouldNotReachHere();
1699         c = 0;  // unreachable
1700         break;
1701       }
1702 
1703       assert(code == lir_add || code == lir_sub, "mismatched arithmetic op");
1704       if (c == 0 && dreg == lreg) {
1705         COMMENT("effective nop elided");
1706         return;
1707       }
1708       switch(left->type()) {
1709       case T_INT:
1710         switch (code) {
1711         case lir_add: __ addw(dreg, lreg, c); break;
1712         case lir_sub: __ subw(dreg, lreg, c); break;
1713         default: ShouldNotReachHere();
1714         }
1715         break;
1716       case T_OBJECT:
1717       case T_ADDRESS:
1718         switch (code) {
1719         case lir_add: __ add(dreg, lreg, c); break;
1720         case lir_sub: __ sub(dreg, lreg, c); break;
1721         default: ShouldNotReachHere();
1722         }
1723         break;
1724         ShouldNotReachHere();
1725       }
1726     } else {
1727       ShouldNotReachHere();
1728     }
1729 
1730   } else if (left->is_double_cpu()) {
1731     Register lreg_lo = left->as_register_lo();
1732 
1733     if (right->is_double_cpu()) {
1734       // cpu register - cpu register
1735       Register rreg_lo = right->as_register_lo();
1736       switch (code) {
1737       case lir_add: __ add (dest->as_register_lo(), lreg_lo, rreg_lo); break;
1738       case lir_sub: __ sub (dest->as_register_lo(), lreg_lo, rreg_lo); break;
1739       case lir_mul: __ mul (dest->as_register_lo(), lreg_lo, rreg_lo); break;
1740       case lir_div: __ corrected_idivq(dest->as_register_lo(), lreg_lo, rreg_lo, false, rscratch1); break;
1741       case lir_rem: __ corrected_idivq(dest->as_register_lo(), lreg_lo, rreg_lo, true, rscratch1); break;
1742       default:
1743         ShouldNotReachHere();
1744       }
1745 
1746     } else if (right->is_constant()) {
1747       jlong c = right->as_constant_ptr()->as_jlong_bits();
1748       Register dreg = as_reg(dest);
1749       assert(code == lir_add || code == lir_sub, "mismatched arithmetic op");
1750       if (c == 0 && dreg == lreg_lo) {
1751         COMMENT("effective nop elided");
1752         return;
1753       }
1754       switch (code) {
1755         case lir_add: __ add(dreg, lreg_lo, c); break;
1756         case lir_sub: __ sub(dreg, lreg_lo, c); break;
1757         default:
1758           ShouldNotReachHere();
1759       }
1760     } else {
1761       ShouldNotReachHere();
1762     }
1763   } else if (left->is_single_fpu()) {
1764     assert(right->is_single_fpu(), "right hand side of float arithmetics needs to be float register");
1765     switch (code) {
1766     case lir_add: __ fadds (dest->as_float_reg(), left->as_float_reg(), right->as_float_reg()); break;
1767     case lir_sub: __ fsubs (dest->as_float_reg(), left->as_float_reg(), right->as_float_reg()); break;
1768     case lir_mul: __ fmuls (dest->as_float_reg(), left->as_float_reg(), right->as_float_reg()); break;
1769     case lir_div: __ fdivs (dest->as_float_reg(), left->as_float_reg(), right->as_float_reg()); break;
1770     default:
1771       ShouldNotReachHere();
1772     }
1773   } else if (left->is_double_fpu()) {
1774     if (right->is_double_fpu()) {
1775       // cpu register - cpu register
1776       switch (code) {
1777       case lir_add: __ faddd (dest->as_double_reg(), left->as_double_reg(), right->as_double_reg()); break;
1778       case lir_sub: __ fsubd (dest->as_double_reg(), left->as_double_reg(), right->as_double_reg()); break;
1779       case lir_mul: __ fmuld (dest->as_double_reg(), left->as_double_reg(), right->as_double_reg()); break;
1780       case lir_div: __ fdivd (dest->as_double_reg(), left->as_double_reg(), right->as_double_reg()); break;
1781       default:
1782         ShouldNotReachHere();
1783       }
1784     } else {
1785       if (right->is_constant()) {
1786         ShouldNotReachHere();
1787       }
1788       ShouldNotReachHere();
1789     }
1790   } else if (left->is_single_stack() || left->is_address()) {
1791     assert(left == dest, "left and dest must be equal");
1792     ShouldNotReachHere();
1793   } else {
1794     ShouldNotReachHere();
1795   }
1796 }
1797 
1798 void LIR_Assembler::arith_fpu_implementation(LIR_Code code, int left_index, int right_index, int dest_index, bool pop_fpu_stack) { Unimplemented(); }
1799 
1800 
1801 void LIR_Assembler::intrinsic_op(LIR_Code code, LIR_Opr value, LIR_Opr unused, LIR_Opr dest, LIR_Op* op) {
1802   switch(code) {
1803   case lir_abs : __ fabsd(dest->as_double_reg(), value->as_double_reg()); break;
1804   case lir_sqrt: __ fsqrtd(dest->as_double_reg(), value->as_double_reg()); break;
1805   default      : ShouldNotReachHere();
1806   }
1807 }
1808 
1809 void LIR_Assembler::logic_op(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dst) {
1810 
1811   assert(left->is_single_cpu() || left->is_double_cpu(), "expect single or double register");
1812   Register Rleft = left->is_single_cpu() ? left->as_register() :
1813                                            left->as_register_lo();
1814    if (dst->is_single_cpu()) {
1815      Register Rdst = dst->as_register();
1816      if (right->is_constant()) {
1817        switch (code) {
1818          case lir_logic_and: __ andw (Rdst, Rleft, right->as_jint()); break;
1819          case lir_logic_or:  __ orrw (Rdst, Rleft, right->as_jint()); break;
1820          case lir_logic_xor: __ eorw (Rdst, Rleft, right->as_jint()); break;
1821          default: ShouldNotReachHere(); break;
1822        }
1823      } else {
1824        Register Rright = right->is_single_cpu() ? right->as_register() :
1825                                                   right->as_register_lo();
1826        switch (code) {
1827          case lir_logic_and: __ andw (Rdst, Rleft, Rright); break;
1828          case lir_logic_or:  __ orrw (Rdst, Rleft, Rright); break;
1829          case lir_logic_xor: __ eorw (Rdst, Rleft, Rright); break;
1830          default: ShouldNotReachHere(); break;
1831        }
1832      }
1833    } else {
1834      Register Rdst = dst->as_register_lo();
1835      if (right->is_constant()) {
1836        switch (code) {
1837          case lir_logic_and: __ andr (Rdst, Rleft, right->as_jlong()); break;
1838          case lir_logic_or:  __ orr (Rdst, Rleft, right->as_jlong()); break;
1839          case lir_logic_xor: __ eor (Rdst, Rleft, right->as_jlong()); break;
1840          default: ShouldNotReachHere(); break;
1841        }
1842      } else {
1843        Register Rright = right->is_single_cpu() ? right->as_register() :
1844                                                   right->as_register_lo();
1845        switch (code) {
1846          case lir_logic_and: __ andr (Rdst, Rleft, Rright); break;
1847          case lir_logic_or:  __ orr (Rdst, Rleft, Rright); break;
1848          case lir_logic_xor: __ eor (Rdst, Rleft, Rright); break;
1849          default: ShouldNotReachHere(); break;
1850        }
1851      }
1852    }
1853 }
1854 
1855 
1856 
1857 void LIR_Assembler::arithmetic_idiv(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr temp, LIR_Opr result, CodeEmitInfo* info) { Unimplemented(); }
1858 
1859 
1860 void LIR_Assembler::comp_op(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Op2* op) {
1861   if (opr1->is_constant() && opr2->is_single_cpu()) {
1862     // tableswitch
1863     Register reg = as_reg(opr2);
1864     struct tableswitch &table = switches[opr1->as_constant_ptr()->as_jint()];
1865     __ tableswitch(reg, table._first_key, table._last_key, table._branches, table._after);
1866   } else if (opr1->is_single_cpu() || opr1->is_double_cpu()) {
1867     Register reg1 = as_reg(opr1);
1868     if (opr2->is_single_cpu()) {
1869       // cpu register - cpu register
1870       Register reg2 = opr2->as_register();
1871       if (opr1->type() == T_OBJECT || opr1->type() == T_ARRAY) {
1872         __ cmp(reg1, reg2);
1873       } else {
1874         assert(opr2->type() != T_OBJECT && opr2->type() != T_ARRAY, "cmp int, oop?");
1875         __ cmpw(reg1, reg2);
1876       }
1877       return;
1878     }
1879     if (opr2->is_double_cpu()) {
1880       // cpu register - cpu register
1881       Register reg2 = opr2->as_register_lo();
1882       __ cmp(reg1, reg2);
1883       return;
1884     }
1885 
1886     if (opr2->is_constant()) {
1887       bool is_32bit = false; // width of register operand
1888       jlong imm;
1889 
1890       switch(opr2->type()) {
1891       case T_INT:
1892         imm = opr2->as_constant_ptr()->as_jint();
1893         is_32bit = true;
1894         break;
1895       case T_LONG:
1896         imm = opr2->as_constant_ptr()->as_jlong();
1897         break;
1898       case T_ADDRESS:
1899         imm = opr2->as_constant_ptr()->as_jint();
1900         break;
1901       case T_OBJECT:
1902       case T_ARRAY:
1903         imm = jlong(opr2->as_constant_ptr()->as_jobject());
1904         break;
1905       default:
1906         ShouldNotReachHere();
1907         imm = 0;  // unreachable
1908         break;
1909       }
1910 
1911       if (Assembler::operand_valid_for_add_sub_immediate(imm)) {
1912         if (is_32bit)
1913           __ cmpw(reg1, imm);
1914         else
1915           __ cmp(reg1, imm);
1916         return;
1917       } else {
1918         __ mov(rscratch1, imm);
1919         if (is_32bit)
1920           __ cmpw(reg1, rscratch1);
1921         else
1922           __ cmp(reg1, rscratch1);
1923         return;
1924       }
1925     } else
1926       ShouldNotReachHere();
1927   } else if (opr1->is_single_fpu()) {
1928     FloatRegister reg1 = opr1->as_float_reg();
1929     assert(opr2->is_single_fpu(), "expect single float register");
1930     FloatRegister reg2 = opr2->as_float_reg();
1931     __ fcmps(reg1, reg2);
1932   } else if (opr1->is_double_fpu()) {
1933     FloatRegister reg1 = opr1->as_double_reg();
1934     assert(opr2->is_double_fpu(), "expect double float register");
1935     FloatRegister reg2 = opr2->as_double_reg();
1936     __ fcmpd(reg1, reg2);
1937   } else {
1938     ShouldNotReachHere();
1939   }
1940 }
1941 
1942 void LIR_Assembler::comp_fl2i(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dst, LIR_Op2* op){
1943   if (code == lir_cmp_fd2i || code == lir_ucmp_fd2i) {
1944     bool is_unordered_less = (code == lir_ucmp_fd2i);
1945     if (left->is_single_fpu()) {
1946       __ float_cmp(true, is_unordered_less ? -1 : 1, left->as_float_reg(), right->as_float_reg(), dst->as_register());
1947     } else if (left->is_double_fpu()) {
1948       __ float_cmp(false, is_unordered_less ? -1 : 1, left->as_double_reg(), right->as_double_reg(), dst->as_register());
1949     } else {
1950       ShouldNotReachHere();
1951     }
1952   } else if (code == lir_cmp_l2i) {
1953     Label done;
1954     __ cmp(left->as_register_lo(), right->as_register_lo());
1955     __ mov(dst->as_register(), (u_int64_t)-1L);
1956     __ br(Assembler::LT, done);
1957     __ csinc(dst->as_register(), zr, zr, Assembler::EQ);
1958     __ bind(done);
1959   } else {
1960     ShouldNotReachHere();
1961   }
1962 }
1963 
1964 
1965 void LIR_Assembler::align_call(LIR_Code code) {  }
1966 
1967 
1968 void LIR_Assembler::call(LIR_OpJavaCall* op, relocInfo::relocType rtype) {
1969   address call = __ trampoline_call(Address(op->addr(), rtype));
1970   if (call == NULL) {
1971     bailout("trampoline stub overflow");
1972     return;
1973   }
1974   add_call_info(code_offset(), op->info());
1975 }
1976 
1977 
1978 void LIR_Assembler::ic_call(LIR_OpJavaCall* op) {
1979   address call = __ ic_call(op->addr());
1980   if (call == NULL) {
1981     bailout("trampoline stub overflow");
1982     return;
1983   }
1984   add_call_info(code_offset(), op->info());
1985 }
1986 
1987 
1988 /* Currently, vtable-dispatch is only enabled for sparc platforms */
1989 void LIR_Assembler::vtable_call(LIR_OpJavaCall* op) {
1990   ShouldNotReachHere();
1991 }
1992 
1993 
1994 void LIR_Assembler::emit_static_call_stub() {
1995   address call_pc = __ pc();
1996   address stub = __ start_a_stub(call_stub_size());
1997   if (stub == NULL) {
1998     bailout("static call stub overflow");
1999     return;
2000   }
2001 
2002   int start = __ offset();
2003 
2004   __ relocate(static_stub_Relocation::spec(call_pc));
2005   __ mov_metadata(rmethod, (Metadata*)NULL);
2006   __ movptr(rscratch1, 0);
2007   __ br(rscratch1);
2008 
2009   assert(__ offset() - start <= call_stub_size(), "stub too big");
2010   __ end_a_stub();
2011 }
2012 
2013 
2014 void LIR_Assembler::throw_op(LIR_Opr exceptionPC, LIR_Opr exceptionOop, CodeEmitInfo* info) {
2015   assert(exceptionOop->as_register() == r0, "must match");
2016   assert(exceptionPC->as_register() == r3, "must match");
2017 
2018   // exception object is not added to oop map by LinearScan
2019   // (LinearScan assumes that no oops are in fixed registers)
2020   info->add_register_oop(exceptionOop);
2021   Runtime1::StubID unwind_id;
2022 
2023   // get current pc information
2024   // pc is only needed if the method has an exception handler, the unwind code does not need it.
2025   int pc_for_athrow_offset = __ offset();
2026   InternalAddress pc_for_athrow(__ pc());
2027   __ adr(exceptionPC->as_register(), pc_for_athrow);
2028   add_call_info(pc_for_athrow_offset, info); // for exception handler
2029 
2030   __ verify_not_null_oop(r0);
2031   // search an exception handler (r0: exception oop, r3: throwing pc)
2032   if (compilation()->has_fpu_code()) {
2033     unwind_id = Runtime1::handle_exception_id;
2034   } else {
2035     unwind_id = Runtime1::handle_exception_nofpu_id;
2036   }
2037   __ far_call(RuntimeAddress(Runtime1::entry_for(unwind_id)));
2038 
2039   // FIXME: enough room for two byte trap   ????
2040   __ nop();
2041 }
2042 
2043 
2044 void LIR_Assembler::unwind_op(LIR_Opr exceptionOop) {
2045   assert(exceptionOop->as_register() == r0, "must match");
2046 
2047   __ b(_unwind_handler_entry);
2048 }
2049 
2050 
2051 void LIR_Assembler::shift_op(LIR_Code code, LIR_Opr left, LIR_Opr count, LIR_Opr dest, LIR_Opr tmp) {
2052   Register lreg = left->is_single_cpu() ? left->as_register() : left->as_register_lo();
2053   Register dreg = dest->is_single_cpu() ? dest->as_register() : dest->as_register_lo();
2054 
2055   switch (left->type()) {
2056     case T_INT: {
2057       switch (code) {
2058       case lir_shl:  __ lslvw (dreg, lreg, count->as_register()); break;
2059       case lir_shr:  __ asrvw (dreg, lreg, count->as_register()); break;
2060       case lir_ushr: __ lsrvw (dreg, lreg, count->as_register()); break;
2061       default:
2062         ShouldNotReachHere();
2063         break;
2064       }
2065       break;
2066     case T_LONG:
2067     case T_ADDRESS:
2068     case T_OBJECT:
2069       switch (code) {
2070       case lir_shl:  __ lslv (dreg, lreg, count->as_register()); break;
2071       case lir_shr:  __ asrv (dreg, lreg, count->as_register()); break;
2072       case lir_ushr: __ lsrv (dreg, lreg, count->as_register()); break;
2073       default:
2074         ShouldNotReachHere();
2075         break;
2076       }
2077       break;
2078     default:
2079       ShouldNotReachHere();
2080       break;
2081     }
2082   }
2083 }
2084 
2085 
2086 void LIR_Assembler::shift_op(LIR_Code code, LIR_Opr left, jint count, LIR_Opr dest) {
2087   Register dreg = dest->is_single_cpu() ? dest->as_register() : dest->as_register_lo();
2088   Register lreg = left->is_single_cpu() ? left->as_register() : left->as_register_lo();
2089 
2090   switch (left->type()) {
2091     case T_INT: {
2092       switch (code) {
2093       case lir_shl:  __ lslw (dreg, lreg, count); break;
2094       case lir_shr:  __ asrw (dreg, lreg, count); break;
2095       case lir_ushr: __ lsrw (dreg, lreg, count); break;
2096       default:
2097         ShouldNotReachHere();
2098         break;
2099       }
2100       break;
2101     case T_LONG:
2102     case T_ADDRESS:
2103     case T_OBJECT:
2104       switch (code) {
2105       case lir_shl:  __ lsl (dreg, lreg, count); break;
2106       case lir_shr:  __ asr (dreg, lreg, count); break;
2107       case lir_ushr: __ lsr (dreg, lreg, count); break;
2108       default:
2109         ShouldNotReachHere();
2110         break;
2111       }
2112       break;
2113     default:
2114       ShouldNotReachHere();
2115       break;
2116     }
2117   }
2118 }
2119 
2120 
2121 void LIR_Assembler::store_parameter(Register r, int offset_from_rsp_in_words) {
2122   assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2123   int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2124   assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2125   __ str (r, Address(sp, offset_from_rsp_in_bytes));
2126 }
2127 
2128 
2129 void LIR_Assembler::store_parameter(jint c,     int offset_from_rsp_in_words) {
2130   assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2131   int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2132   assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2133   __ mov (rscratch1, c);
2134   __ str (rscratch1, Address(sp, offset_from_rsp_in_bytes));
2135 }
2136 
2137 
2138 void LIR_Assembler::store_parameter(jobject o,  int offset_from_rsp_in_words) {
2139   ShouldNotReachHere();
2140   assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2141   int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2142   assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2143   __ lea(rscratch1, __ constant_oop_address(o));
2144   __ str(rscratch1, Address(sp, offset_from_rsp_in_bytes));
2145 }
2146 
2147 
2148 // This code replaces a call to arraycopy; no exception may
2149 // be thrown in this code, they must be thrown in the System.arraycopy
2150 // activation frame; we could save some checks if this would not be the case
2151 void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) {
2152   ciArrayKlass* default_type = op->expected_type();
2153   Register src = op->src()->as_register();
2154   Register dst = op->dst()->as_register();
2155   Register src_pos = op->src_pos()->as_register();
2156   Register dst_pos = op->dst_pos()->as_register();
2157   Register length  = op->length()->as_register();
2158   Register tmp = op->tmp()->as_register();
2159 
2160   CodeStub* stub = op->stub();
2161   int flags = op->flags();
2162   BasicType basic_type = default_type != NULL ? default_type->element_type()->basic_type() : T_ILLEGAL;
2163   if (basic_type == T_ARRAY) basic_type = T_OBJECT;
2164 
2165   // if we don't know anything, just go through the generic arraycopy
2166   if (default_type == NULL // || basic_type == T_OBJECT
2167       ) {
2168     Label done;
2169     assert(src == r1 && src_pos == r2, "mismatch in calling convention");
2170 
2171     // Save the arguments in case the generic arraycopy fails and we
2172     // have to fall back to the JNI stub
2173     __ stp(dst,     dst_pos, Address(sp, 0*BytesPerWord));
2174     __ stp(length,  src_pos, Address(sp, 2*BytesPerWord));
2175     __ str(src,              Address(sp, 4*BytesPerWord));
2176 
2177     address C_entry = CAST_FROM_FN_PTR(address, Runtime1::arraycopy);
2178     address copyfunc_addr = StubRoutines::generic_arraycopy();
2179 
2180     // The arguments are in java calling convention so we shift them
2181     // to C convention
2182     assert_different_registers(c_rarg0, j_rarg1, j_rarg2, j_rarg3, j_rarg4);
2183     __ mov(c_rarg0, j_rarg0);
2184     assert_different_registers(c_rarg1, j_rarg2, j_rarg3, j_rarg4);
2185     __ mov(c_rarg1, j_rarg1);
2186     assert_different_registers(c_rarg2, j_rarg3, j_rarg4);
2187     __ mov(c_rarg2, j_rarg2);
2188     assert_different_registers(c_rarg3, j_rarg4);
2189     __ mov(c_rarg3, j_rarg3);
2190     __ mov(c_rarg4, j_rarg4);
2191     if (copyfunc_addr == NULL) { // Use C version if stub was not generated
2192       __ mov(rscratch1, RuntimeAddress(C_entry));
2193       __ blrt(rscratch1, 5, 0, 1);
2194     } else {
2195 #ifndef PRODUCT
2196       if (PrintC1Statistics) {
2197         __ incrementw(ExternalAddress((address)&Runtime1::_generic_arraycopystub_cnt));
2198       }
2199 #endif
2200       __ far_call(RuntimeAddress(copyfunc_addr));
2201     }
2202 
2203     __ cbz(r0, *stub->continuation());
2204 
2205     // Reload values from the stack so they are where the stub
2206     // expects them.
2207     __ ldp(dst,     dst_pos, Address(sp, 0*BytesPerWord));
2208     __ ldp(length,  src_pos, Address(sp, 2*BytesPerWord));
2209     __ ldr(src,              Address(sp, 4*BytesPerWord));
2210 
2211     if (copyfunc_addr != NULL) {
2212       // r0 is -1^K where K == partial copied count
2213       __ eonw(rscratch1, r0, 0);
2214       // adjust length down and src/end pos up by partial copied count
2215       __ subw(length, length, rscratch1);
2216       __ addw(src_pos, src_pos, rscratch1);
2217       __ addw(dst_pos, dst_pos, rscratch1);
2218     }
2219     __ b(*stub->entry());
2220 
2221     __ bind(*stub->continuation());
2222     return;
2223   }
2224 
2225   assert(default_type != NULL && default_type->is_array_klass() && default_type->is_loaded(), "must be true at this point");
2226 
2227   int elem_size = type2aelembytes(basic_type);
2228   int shift_amount;
2229   int scale = exact_log2(elem_size);
2230 
2231   Address src_length_addr = Address(src, arrayOopDesc::length_offset_in_bytes());
2232   Address dst_length_addr = Address(dst, arrayOopDesc::length_offset_in_bytes());
2233   Address src_klass_addr = Address(src, oopDesc::klass_offset_in_bytes());
2234   Address dst_klass_addr = Address(dst, oopDesc::klass_offset_in_bytes());
2235 
2236   // test for NULL
2237   if (flags & LIR_OpArrayCopy::src_null_check) {
2238     __ cbz(src, *stub->entry());
2239   }
2240   if (flags & LIR_OpArrayCopy::dst_null_check) {
2241     __ cbz(dst, *stub->entry());
2242   }
2243 
2244   // If the compiler was not able to prove that exact type of the source or the destination
2245   // of the arraycopy is an array type, check at runtime if the source or the destination is
2246   // an instance type.
2247   if (flags & LIR_OpArrayCopy::type_check) {
2248     if (!(flags & LIR_OpArrayCopy::LIR_OpArrayCopy::dst_objarray)) {
2249       __ load_klass(tmp, dst);
2250       __ ldrw(rscratch1, Address(tmp, in_bytes(Klass::layout_helper_offset())));
2251       __ cmpw(rscratch1, Klass::_lh_neutral_value);
2252       __ br(Assembler::GE, *stub->entry());
2253     }
2254 
2255     if (!(flags & LIR_OpArrayCopy::LIR_OpArrayCopy::src_objarray)) {
2256       __ load_klass(tmp, src);
2257       __ ldrw(rscratch1, Address(tmp, in_bytes(Klass::layout_helper_offset())));
2258       __ cmpw(rscratch1, Klass::_lh_neutral_value);
2259       __ br(Assembler::GE, *stub->entry());
2260     }
2261   }
2262 
2263   // check if negative
2264   if (flags & LIR_OpArrayCopy::src_pos_positive_check) {
2265     __ cmpw(src_pos, 0);
2266     __ br(Assembler::LT, *stub->entry());
2267   }
2268   if (flags & LIR_OpArrayCopy::dst_pos_positive_check) {
2269     __ cmpw(dst_pos, 0);
2270     __ br(Assembler::LT, *stub->entry());
2271   }
2272 
2273   if (flags & LIR_OpArrayCopy::length_positive_check) {
2274     __ cmpw(length, 0);
2275     __ br(Assembler::LT, *stub->entry());
2276   }
2277 
2278   if (flags & LIR_OpArrayCopy::src_range_check) {
2279     __ addw(tmp, src_pos, length);
2280     __ ldrw(rscratch1, src_length_addr);
2281     __ cmpw(tmp, rscratch1);
2282     __ br(Assembler::HI, *stub->entry());
2283   }
2284   if (flags & LIR_OpArrayCopy::dst_range_check) {
2285     __ addw(tmp, dst_pos, length);
2286     __ ldrw(rscratch1, dst_length_addr);
2287     __ cmpw(tmp, rscratch1);
2288     __ br(Assembler::HI, *stub->entry());
2289   }
2290 
2291   if (flags & LIR_OpArrayCopy::type_check) {
2292     // We don't know the array types are compatible
2293     if (basic_type != T_OBJECT) {
2294       // Simple test for basic type arrays
2295       if (UseCompressedClassPointers) {
2296         __ ldrw(tmp, src_klass_addr);
2297         __ ldrw(rscratch1, dst_klass_addr);
2298         __ cmpw(tmp, rscratch1);
2299       } else {
2300         __ ldr(tmp, src_klass_addr);
2301         __ ldr(rscratch1, dst_klass_addr);
2302         __ cmp(tmp, rscratch1);
2303       }
2304       __ br(Assembler::NE, *stub->entry());
2305     } else {
2306       // For object arrays, if src is a sub class of dst then we can
2307       // safely do the copy.
2308       Label cont, slow;
2309 
2310 #define PUSH(r1, r2)                                    \
2311       stp(r1, r2, __ pre(sp, -2 * wordSize));
2312 
2313 #define POP(r1, r2)                                     \
2314       ldp(r1, r2, __ post(sp, 2 * wordSize));
2315 
2316       __ PUSH(src, dst);
2317 
2318       __ load_klass(src, src);
2319       __ load_klass(dst, dst);
2320 
2321       __ check_klass_subtype_fast_path(src, dst, tmp, &cont, &slow, NULL);
2322 
2323       __ PUSH(src, dst);
2324       __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
2325       __ POP(src, dst);
2326 
2327       __ cbnz(src, cont);
2328 
2329       __ bind(slow);
2330       __ POP(src, dst);
2331 
2332       address copyfunc_addr = StubRoutines::checkcast_arraycopy();
2333       if (copyfunc_addr != NULL) { // use stub if available
2334         // src is not a sub class of dst so we have to do a
2335         // per-element check.
2336 
2337         int mask = LIR_OpArrayCopy::src_objarray|LIR_OpArrayCopy::dst_objarray;
2338         if ((flags & mask) != mask) {
2339           // Check that at least both of them object arrays.
2340           assert(flags & mask, "one of the two should be known to be an object array");
2341 
2342           if (!(flags & LIR_OpArrayCopy::src_objarray)) {
2343             __ load_klass(tmp, src);
2344           } else if (!(flags & LIR_OpArrayCopy::dst_objarray)) {
2345             __ load_klass(tmp, dst);
2346           }
2347           int lh_offset = in_bytes(Klass::layout_helper_offset());
2348           Address klass_lh_addr(tmp, lh_offset);
2349           jint objArray_lh = Klass::array_layout_helper(T_OBJECT);
2350           __ ldrw(rscratch1, klass_lh_addr);
2351           __ mov(rscratch2, objArray_lh);
2352           __ eorw(rscratch1, rscratch1, rscratch2);
2353           __ cbnzw(rscratch1, *stub->entry());
2354         }
2355 
2356        // Spill because stubs can use any register they like and it's
2357        // easier to restore just those that we care about.
2358         __ stp(dst,     dst_pos, Address(sp, 0*BytesPerWord));
2359         __ stp(length,  src_pos, Address(sp, 2*BytesPerWord));
2360         __ str(src,              Address(sp, 4*BytesPerWord));
2361 
2362         __ lea(c_rarg0, Address(src, src_pos, Address::uxtw(scale)));
2363         __ add(c_rarg0, c_rarg0, arrayOopDesc::base_offset_in_bytes(basic_type));
2364         assert_different_registers(c_rarg0, dst, dst_pos, length);
2365         __ lea(c_rarg1, Address(dst, dst_pos, Address::uxtw(scale)));
2366         __ add(c_rarg1, c_rarg1, arrayOopDesc::base_offset_in_bytes(basic_type));
2367         assert_different_registers(c_rarg1, dst, length);
2368         __ uxtw(c_rarg2, length);
2369         assert_different_registers(c_rarg2, dst);
2370 
2371         __ load_klass(c_rarg4, dst);
2372         __ ldr(c_rarg4, Address(c_rarg4, ObjArrayKlass::element_klass_offset()));
2373         __ ldrw(c_rarg3, Address(c_rarg4, Klass::super_check_offset_offset()));
2374         __ far_call(RuntimeAddress(copyfunc_addr));
2375 
2376 #ifndef PRODUCT
2377         if (PrintC1Statistics) {
2378           Label failed;
2379           __ cbnz(r0, failed);
2380           __ incrementw(ExternalAddress((address)&Runtime1::_arraycopy_checkcast_cnt));
2381           __ bind(failed);
2382         }
2383 #endif
2384 
2385         __ cbz(r0, *stub->continuation());
2386 
2387 #ifndef PRODUCT
2388         if (PrintC1Statistics) {
2389           __ incrementw(ExternalAddress((address)&Runtime1::_arraycopy_checkcast_attempt_cnt));
2390         }
2391 #endif
2392         assert_different_registers(dst, dst_pos, length, src_pos, src, r0, rscratch1);
2393 
2394         // Restore previously spilled arguments
2395         __ ldp(dst,     dst_pos, Address(sp, 0*BytesPerWord));
2396         __ ldp(length,  src_pos, Address(sp, 2*BytesPerWord));
2397         __ ldr(src,              Address(sp, 4*BytesPerWord));
2398 
2399         // return value is -1^K where K is partial copied count
2400         __ eonw(rscratch1, r0, zr);
2401         // adjust length down and src/end pos up by partial copied count
2402         __ subw(length, length, rscratch1);
2403         __ addw(src_pos, src_pos, rscratch1);
2404         __ addw(dst_pos, dst_pos, rscratch1);
2405       }
2406 
2407       __ b(*stub->entry());
2408 
2409       __ bind(cont);
2410       __ POP(src, dst);
2411     }
2412   }
2413 
2414 #ifdef ASSERT
2415   if (basic_type != T_OBJECT || !(flags & LIR_OpArrayCopy::type_check)) {
2416     // Sanity check the known type with the incoming class.  For the
2417     // primitive case the types must match exactly with src.klass and
2418     // dst.klass each exactly matching the default type.  For the
2419     // object array case, if no type check is needed then either the
2420     // dst type is exactly the expected type and the src type is a
2421     // subtype which we can't check or src is the same array as dst
2422     // but not necessarily exactly of type default_type.
2423     Label known_ok, halt;
2424     __ mov_metadata(tmp, default_type->constant_encoding());
2425     if (UseCompressedClassPointers) {
2426       __ encode_klass_not_null(tmp);
2427     }
2428 
2429     if (basic_type != T_OBJECT) {
2430 
2431       if (UseCompressedClassPointers) {
2432         __ ldrw(rscratch1, dst_klass_addr);
2433         __ cmpw(tmp, rscratch1);
2434       } else {
2435         __ ldr(rscratch1, dst_klass_addr);
2436         __ cmp(tmp, rscratch1);
2437       }
2438       __ br(Assembler::NE, halt);
2439       if (UseCompressedClassPointers) {
2440         __ ldrw(rscratch1, src_klass_addr);
2441         __ cmpw(tmp, rscratch1);
2442       } else {
2443         __ ldr(rscratch1, src_klass_addr);
2444         __ cmp(tmp, rscratch1);
2445       }
2446       __ br(Assembler::EQ, known_ok);
2447     } else {
2448       if (UseCompressedClassPointers) {
2449         __ ldrw(rscratch1, dst_klass_addr);
2450         __ cmpw(tmp, rscratch1);
2451       } else {
2452         __ ldr(rscratch1, dst_klass_addr);
2453         __ cmp(tmp, rscratch1);
2454       }
2455       __ br(Assembler::EQ, known_ok);
2456       __ cmp(src, dst);
2457       __ br(Assembler::EQ, known_ok);
2458     }
2459     __ bind(halt);
2460     __ stop("incorrect type information in arraycopy");
2461     __ bind(known_ok);
2462   }
2463 #endif
2464 
2465 #ifndef PRODUCT
2466   if (PrintC1Statistics) {
2467     __ incrementw(ExternalAddress(Runtime1::arraycopy_count_address(basic_type)));
2468   }
2469 #endif
2470 
2471   __ lea(c_rarg0, Address(src, src_pos, Address::uxtw(scale)));
2472   __ add(c_rarg0, c_rarg0, arrayOopDesc::base_offset_in_bytes(basic_type));
2473   assert_different_registers(c_rarg0, dst, dst_pos, length);
2474   __ lea(c_rarg1, Address(dst, dst_pos, Address::uxtw(scale)));
2475   __ add(c_rarg1, c_rarg1, arrayOopDesc::base_offset_in_bytes(basic_type));
2476   assert_different_registers(c_rarg1, dst, length);
2477   __ uxtw(c_rarg2, length);
2478   assert_different_registers(c_rarg2, dst);
2479 
2480   bool disjoint = (flags & LIR_OpArrayCopy::overlapping) == 0;
2481   bool aligned = (flags & LIR_OpArrayCopy::unaligned) == 0;
2482   const char *name;
2483   address entry = StubRoutines::select_arraycopy_function(basic_type, aligned, disjoint, name, false);
2484 
2485  CodeBlob *cb = CodeCache::find_blob(entry);
2486  if (cb) {
2487    __ far_call(RuntimeAddress(entry));
2488  } else {
2489    __ call_VM_leaf(entry, 3);
2490  }
2491 
2492   __ bind(*stub->continuation());
2493 }
2494 
2495 
2496 
2497 
2498 void LIR_Assembler::emit_lock(LIR_OpLock* op) {
2499   Register obj = op->obj_opr()->as_register();  // may not be an oop
2500   Register hdr = op->hdr_opr()->as_register();
2501   Register lock = op->lock_opr()->as_register();
2502   if (!UseFastLocking) {
2503     __ b(*op->stub()->entry());
2504   } else if (op->code() == lir_lock) {
2505     Register scratch = noreg;
2506     if (UseBiasedLocking) {
2507       scratch = op->scratch_opr()->as_register();
2508     }
2509     assert(BasicLock::displaced_header_offset_in_bytes() == 0, "lock_reg must point to the displaced header");
2510     // add debug info for NullPointerException only if one is possible
2511     int null_check_offset = __ lock_object(hdr, obj, lock, scratch, *op->stub()->entry());
2512     if (op->info() != NULL) {
2513       add_debug_info_for_null_check(null_check_offset, op->info());
2514     }
2515     // done
2516   } else if (op->code() == lir_unlock) {
2517     assert(BasicLock::displaced_header_offset_in_bytes() == 0, "lock_reg must point to the displaced header");
2518     __ unlock_object(hdr, obj, lock, *op->stub()->entry());
2519   } else {
2520     Unimplemented();
2521   }
2522   __ bind(*op->stub()->continuation());
2523 }
2524 
2525 
2526 void LIR_Assembler::emit_profile_call(LIR_OpProfileCall* op) {
2527   ciMethod* method = op->profiled_method();
2528   int bci          = op->profiled_bci();
2529   ciMethod* callee = op->profiled_callee();
2530 
2531   // Update counter for all call types
2532   ciMethodData* md = method->method_data_or_null();
2533   assert(md != NULL, "Sanity");
2534   ciProfileData* data = md->bci_to_data(bci);
2535   assert(data != NULL && data->is_CounterData(), "need CounterData for calls");
2536   assert(op->mdo()->is_single_cpu(),  "mdo must be allocated");
2537   Register mdo  = op->mdo()->as_register();
2538   __ mov_metadata(mdo, md->constant_encoding());
2539   Address counter_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
2540   // Perform additional virtual call profiling for invokevirtual and
2541   // invokeinterface bytecodes
2542   if (op->should_profile_receiver_type()) {
2543     assert(op->recv()->is_single_cpu(), "recv must be allocated");
2544     Register recv = op->recv()->as_register();
2545     assert_different_registers(mdo, recv);
2546     assert(data->is_VirtualCallData(), "need VirtualCallData for virtual calls");
2547     ciKlass* known_klass = op->known_holder();
2548     if (C1OptimizeVirtualCallProfiling && known_klass != NULL) {
2549       // We know the type that will be seen at this call site; we can
2550       // statically update the MethodData* rather than needing to do
2551       // dynamic tests on the receiver type
2552 
2553       // NOTE: we should probably put a lock around this search to
2554       // avoid collisions by concurrent compilations
2555       ciVirtualCallData* vc_data = (ciVirtualCallData*) data;
2556       uint i;
2557       for (i = 0; i < VirtualCallData::row_limit(); i++) {
2558         ciKlass* receiver = vc_data->receiver(i);
2559         if (known_klass->equals(receiver)) {
2560           Address data_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_count_offset(i)));
2561           __ addptr(data_addr, DataLayout::counter_increment);
2562           return;
2563         }
2564       }
2565 
2566       // Receiver type not found in profile data; select an empty slot
2567 
2568       // Note that this is less efficient than it should be because it
2569       // always does a write to the receiver part of the
2570       // VirtualCallData rather than just the first time
2571       for (i = 0; i < VirtualCallData::row_limit(); i++) {
2572         ciKlass* receiver = vc_data->receiver(i);
2573         if (receiver == NULL) {
2574           Address recv_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_offset(i)));
2575           __ mov_metadata(rscratch1, known_klass->constant_encoding());
2576           __ lea(rscratch2, recv_addr);
2577           __ str(rscratch1, Address(rscratch2));
2578           Address data_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_count_offset(i)));
2579           __ addptr(data_addr, DataLayout::counter_increment);
2580           return;
2581         }
2582       }
2583     } else {
2584       __ load_klass(recv, recv);
2585       Label update_done;
2586       type_profile_helper(mdo, md, data, recv, &update_done);
2587       // Receiver did not match any saved receiver and there is no empty row for it.
2588       // Increment total counter to indicate polymorphic case.
2589       __ addptr(counter_addr, DataLayout::counter_increment);
2590 
2591       __ bind(update_done);
2592     }
2593   } else {
2594     // Static call
2595     __ addptr(counter_addr, DataLayout::counter_increment);
2596   }
2597 }
2598 
2599 
2600 void LIR_Assembler::emit_delay(LIR_OpDelay*) {
2601   Unimplemented();
2602 }
2603 
2604 
2605 void LIR_Assembler::monitor_address(int monitor_no, LIR_Opr dst) {
2606   __ lea(dst->as_register(), frame_map()->address_for_monitor_lock(monitor_no));
2607 }
2608 
2609 void LIR_Assembler::emit_updatecrc32(LIR_OpUpdateCRC32* op) {
2610   assert(op->crc()->is_single_cpu(),  "crc must be register");
2611   assert(op->val()->is_single_cpu(),  "byte value must be register");
2612   assert(op->result_opr()->is_single_cpu(), "result must be register");
2613   Register crc = op->crc()->as_register();
2614   Register val = op->val()->as_register();
2615   Register res = op->result_opr()->as_register();
2616 
2617   assert_different_registers(val, crc, res);
2618   unsigned long offset;
2619   __ adrp(res, ExternalAddress(StubRoutines::crc_table_addr()), offset);
2620   if (offset) __ add(res, res, offset);
2621 
2622   __ mvnw(crc, crc); // ~crc
2623   __ update_byte_crc32(crc, val, res);
2624   __ mvnw(res, crc); // ~crc
2625 }
2626 
2627 void LIR_Assembler::emit_profile_type(LIR_OpProfileType* op) {
2628   COMMENT("emit_profile_type {");
2629   Register obj = op->obj()->as_register();
2630   Register tmp = op->tmp()->as_pointer_register();
2631   Address mdo_addr = as_Address(op->mdp()->as_address_ptr());
2632   ciKlass* exact_klass = op->exact_klass();
2633   intptr_t current_klass = op->current_klass();
2634   bool not_null = op->not_null();
2635   bool no_conflict = op->no_conflict();
2636 
2637   Label update, next, none;
2638 
2639   bool do_null = !not_null;
2640   bool exact_klass_set = exact_klass != NULL && ciTypeEntries::valid_ciklass(current_klass) == exact_klass;
2641   bool do_update = !TypeEntries::is_type_unknown(current_klass) && !exact_klass_set;
2642 
2643   assert(do_null || do_update, "why are we here?");
2644   assert(!TypeEntries::was_null_seen(current_klass) || do_update, "why are we here?");
2645   assert(mdo_addr.base() != rscratch1, "wrong register");
2646 
2647   __ verify_oop(obj);
2648 
2649   if (tmp != obj) {
2650     __ mov(tmp, obj);
2651   }
2652   if (do_null) {
2653     __ cbnz(tmp, update);
2654     if (!TypeEntries::was_null_seen(current_klass)) {
2655       __ ldr(rscratch2, mdo_addr);
2656       __ orr(rscratch2, rscratch2, TypeEntries::null_seen);
2657       __ str(rscratch2, mdo_addr);
2658     }
2659     if (do_update) {
2660 #ifndef ASSERT
2661       __ b(next);
2662     }
2663 #else
2664       __ b(next);
2665     }
2666   } else {
2667     __ cbnz(tmp, update);
2668     __ stop("unexpected null obj");
2669 #endif
2670   }
2671 
2672   __ bind(update);
2673 
2674   if (do_update) {
2675 #ifdef ASSERT
2676     if (exact_klass != NULL) {
2677       Label ok;
2678       __ load_klass(tmp, tmp);
2679       __ mov_metadata(rscratch1, exact_klass->constant_encoding());
2680       __ eor(rscratch1, tmp, rscratch1);
2681       __ cbz(rscratch1, ok);
2682       __ stop("exact klass and actual klass differ");
2683       __ bind(ok);
2684     }
2685 #endif
2686     if (!no_conflict) {
2687       if (exact_klass == NULL || TypeEntries::is_type_none(current_klass)) {
2688         if (exact_klass != NULL) {
2689           __ mov_metadata(tmp, exact_klass->constant_encoding());
2690         } else {
2691           __ load_klass(tmp, tmp);
2692         }
2693 
2694         __ ldr(rscratch2, mdo_addr);
2695         __ eor(tmp, tmp, rscratch2);
2696         __ andr(rscratch1, tmp, TypeEntries::type_klass_mask);
2697         // klass seen before, nothing to do. The unknown bit may have been
2698         // set already but no need to check.
2699         __ cbz(rscratch1, next);
2700 
2701         __ tbnz(tmp, exact_log2(TypeEntries::type_unknown), next); // already unknown. Nothing to do anymore.
2702 
2703         if (TypeEntries::is_type_none(current_klass)) {
2704           __ cbz(rscratch2, none);
2705           __ cmp(rscratch2, TypeEntries::null_seen);
2706           __ br(Assembler::EQ, none);
2707           // There is a chance that the checks above (re-reading profiling
2708           // data from memory) fail if another thread has just set the
2709           // profiling to this obj's klass
2710           __ dmb(Assembler::ISHLD);
2711           __ ldr(rscratch2, mdo_addr);
2712           __ eor(tmp, tmp, rscratch2);
2713           __ andr(rscratch1, tmp, TypeEntries::type_klass_mask);
2714           __ cbz(rscratch1, next);
2715         }
2716       } else {
2717         assert(ciTypeEntries::valid_ciklass(current_klass) != NULL &&
2718                ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "conflict only");
2719 
2720         __ ldr(tmp, mdo_addr);
2721         __ tbnz(tmp, exact_log2(TypeEntries::type_unknown), next); // already unknown. Nothing to do anymore.
2722       }
2723 
2724       // different than before. Cannot keep accurate profile.
2725       __ ldr(rscratch2, mdo_addr);
2726       __ orr(rscratch2, rscratch2, TypeEntries::type_unknown);
2727       __ str(rscratch2, mdo_addr);
2728 
2729       if (TypeEntries::is_type_none(current_klass)) {
2730         __ b(next);
2731 
2732         __ bind(none);
2733         // first time here. Set profile type.
2734         __ str(tmp, mdo_addr);
2735       }
2736     } else {
2737       // There's a single possible klass at this profile point
2738       assert(exact_klass != NULL, "should be");
2739       if (TypeEntries::is_type_none(current_klass)) {
2740         __ mov_metadata(tmp, exact_klass->constant_encoding());
2741         __ ldr(rscratch2, mdo_addr);
2742         __ eor(tmp, tmp, rscratch2);
2743         __ andr(rscratch1, tmp, TypeEntries::type_klass_mask);
2744         __ cbz(rscratch1, next);
2745 #ifdef ASSERT
2746         {
2747           Label ok;
2748           __ ldr(rscratch1, mdo_addr);
2749           __ cbz(rscratch1, ok);
2750           __ cmp(rscratch1, TypeEntries::null_seen);
2751           __ br(Assembler::EQ, ok);
2752           // may have been set by another thread
2753           __ dmb(Assembler::ISHLD);
2754           __ mov_metadata(rscratch1, exact_klass->constant_encoding());
2755           __ ldr(rscratch2, mdo_addr);
2756           __ eor(rscratch2, rscratch1, rscratch2);
2757           __ andr(rscratch2, rscratch2, TypeEntries::type_mask);
2758           __ cbz(rscratch2, ok);
2759 
2760           __ stop("unexpected profiling mismatch");
2761           __ bind(ok);
2762         }
2763 #endif
2764         // first time here. Set profile type.
2765         __ ldr(tmp, mdo_addr);
2766       } else {
2767         assert(ciTypeEntries::valid_ciklass(current_klass) != NULL &&
2768                ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "inconsistent");
2769 
2770         __ ldr(tmp, mdo_addr);
2771         __ tbnz(tmp, exact_log2(TypeEntries::type_unknown), next); // already unknown. Nothing to do anymore.
2772 
2773         __ orr(tmp, tmp, TypeEntries::type_unknown);
2774         __ str(tmp, mdo_addr);
2775         // FIXME: Write barrier needed here?
2776       }
2777     }
2778 
2779     __ bind(next);
2780   }
2781   COMMENT("} emit_profile_type");
2782 }
2783 
2784 
2785 void LIR_Assembler::align_backward_branch_target() {
2786 }
2787 
2788 
2789 void LIR_Assembler::negate(LIR_Opr left, LIR_Opr dest) {
2790   if (left->is_single_cpu()) {
2791     assert(dest->is_single_cpu(), "expect single result reg");
2792     __ negw(dest->as_register(), left->as_register());
2793   } else if (left->is_double_cpu()) {
2794     assert(dest->is_double_cpu(), "expect double result reg");
2795     __ neg(dest->as_register_lo(), left->as_register_lo());
2796   } else if (left->is_single_fpu()) {
2797     assert(dest->is_single_fpu(), "expect single float result reg");
2798     __ fnegs(dest->as_float_reg(), left->as_float_reg());
2799   } else {
2800     assert(left->is_double_fpu(), "expect double float operand reg");
2801     assert(dest->is_double_fpu(), "expect double float result reg");
2802     __ fnegd(dest->as_double_reg(), left->as_double_reg());
2803   }
2804 }
2805 
2806 
2807 void LIR_Assembler::leal(LIR_Opr addr, LIR_Opr dest) {
2808   __ lea(dest->as_register_lo(), as_Address(addr->as_address_ptr()));
2809 }
2810 
2811 
2812 void LIR_Assembler::rt_call(LIR_Opr result, address dest, const LIR_OprList* args, LIR_Opr tmp, CodeEmitInfo* info) {
2813   assert(!tmp->is_valid(), "don't need temporary");
2814 
2815   CodeBlob *cb = CodeCache::find_blob(dest);
2816   if (cb) {
2817     __ far_call(RuntimeAddress(dest));
2818   } else {
2819     __ mov(rscratch1, RuntimeAddress(dest));
2820     int len = args->length();
2821     int type = 0;
2822     if (! result->is_illegal()) {
2823       switch (result->type()) {
2824       case T_VOID:
2825         type = 0;
2826         break;
2827       case T_INT:
2828       case T_LONG:
2829       case T_OBJECT:
2830         type = 1;
2831         break;
2832       case T_FLOAT:
2833         type = 2;
2834         break;
2835       case T_DOUBLE:
2836         type = 3;
2837         break;
2838       default:
2839         ShouldNotReachHere();
2840         break;
2841       }
2842     }
2843     int num_gpargs = 0;
2844     int num_fpargs = 0;
2845     for (int i = 0; i < args->length(); i++) {
2846       LIR_Opr arg = args->at(i);
2847       if (arg->type() == T_FLOAT || arg->type() == T_DOUBLE) {
2848         num_fpargs++;
2849       } else {
2850         num_gpargs++;
2851       }
2852     }
2853     __ blrt(rscratch1, num_gpargs, num_fpargs, type);
2854   }
2855 
2856   if (info != NULL) {
2857     add_call_info_here(info);
2858   }
2859   __ maybe_isb();
2860 }
2861 
2862 void LIR_Assembler::volatile_move_op(LIR_Opr src, LIR_Opr dest, BasicType type, CodeEmitInfo* info) {
2863   if (dest->is_address() || src->is_address()) {
2864     move_op(src, dest, type, lir_patch_none, info,
2865             /*pop_fpu_stack*/false, /*unaligned*/false, /*wide*/false);
2866   } else {
2867     ShouldNotReachHere();
2868   }
2869 }
2870 
2871 #ifdef ASSERT
2872 // emit run-time assertion
2873 void LIR_Assembler::emit_assert(LIR_OpAssert* op) {
2874   assert(op->code() == lir_assert, "must be");
2875 
2876   if (op->in_opr1()->is_valid()) {
2877     assert(op->in_opr2()->is_valid(), "both operands must be valid");
2878     comp_op(op->condition(), op->in_opr1(), op->in_opr2(), op);
2879   } else {
2880     assert(op->in_opr2()->is_illegal(), "both operands must be illegal");
2881     assert(op->condition() == lir_cond_always, "no other conditions allowed");
2882   }
2883 
2884   Label ok;
2885   if (op->condition() != lir_cond_always) {
2886     Assembler::Condition acond = Assembler::AL;
2887     switch (op->condition()) {
2888       case lir_cond_equal:        acond = Assembler::EQ;  break;
2889       case lir_cond_notEqual:     acond = Assembler::NE;  break;
2890       case lir_cond_less:         acond = Assembler::LT;  break;
2891       case lir_cond_lessEqual:    acond = Assembler::LE;  break;
2892       case lir_cond_greaterEqual: acond = Assembler::GE;  break;
2893       case lir_cond_greater:      acond = Assembler::GT;  break;
2894       case lir_cond_belowEqual:   acond = Assembler::LS;  break;
2895       case lir_cond_aboveEqual:   acond = Assembler::HS;  break;
2896       default:                    ShouldNotReachHere();
2897     }
2898     __ br(acond, ok);
2899   }
2900   if (op->halt()) {
2901     const char* str = __ code_string(op->msg());
2902     __ stop(str);
2903   } else {
2904     breakpoint();
2905   }
2906   __ bind(ok);
2907 }
2908 #endif
2909 
2910 #ifndef PRODUCT
2911 #define COMMENT(x)   do { __ block_comment(x); } while (0)
2912 #else
2913 #define COMMENT(x)
2914 #endif
2915 
2916 void LIR_Assembler::membar() {
2917   COMMENT("membar");
2918   __ membar(MacroAssembler::AnyAny);
2919 }
2920 
2921 void LIR_Assembler::membar_acquire() {
2922   __ membar(Assembler::LoadLoad|Assembler::LoadStore);
2923 }
2924 
2925 void LIR_Assembler::membar_release() {
2926   __ membar(Assembler::LoadStore|Assembler::StoreStore);
2927 }
2928 
2929 void LIR_Assembler::membar_loadload() {
2930   __ membar(Assembler::LoadLoad);
2931 }
2932 
2933 void LIR_Assembler::membar_storestore() {
2934   __ membar(MacroAssembler::StoreStore);
2935 }
2936 
2937 void LIR_Assembler::membar_loadstore() { __ membar(MacroAssembler::LoadStore); }
2938 
2939 void LIR_Assembler::membar_storeload() { __ membar(MacroAssembler::StoreLoad); }
2940 
2941 void LIR_Assembler::on_spin_wait() {
2942   Unimplemented();
2943 }
2944 
2945 void LIR_Assembler::get_thread(LIR_Opr result_reg) {
2946   __ mov(result_reg->as_register(), rthread);
2947 }
2948 
2949 
2950 void LIR_Assembler::peephole(LIR_List *lir) {
2951 #if 0
2952   if (tableswitch_count >= max_tableswitches)
2953     return;
2954 
2955   /*
2956     This finite-state automaton recognizes sequences of compare-and-
2957     branch instructions.  We will turn them into a tableswitch.  You
2958     could argue that C1 really shouldn't be doing this sort of
2959     optimization, but without it the code is really horrible.
2960   */
2961 
2962   enum { start_s, cmp1_s, beq_s, cmp_s } state;
2963   int first_key, last_key = -2147483648;
2964   int next_key = 0;
2965   int start_insn = -1;
2966   int last_insn = -1;
2967   Register reg = noreg;
2968   LIR_Opr reg_opr;
2969   state = start_s;
2970 
2971   LIR_OpList* inst = lir->instructions_list();
2972   for (int i = 0; i < inst->length(); i++) {
2973     LIR_Op* op = inst->at(i);
2974     switch (state) {
2975     case start_s:
2976       first_key = -1;
2977       start_insn = i;
2978       switch (op->code()) {
2979       case lir_cmp:
2980         LIR_Opr opr1 = op->as_Op2()->in_opr1();
2981         LIR_Opr opr2 = op->as_Op2()->in_opr2();
2982         if (opr1->is_cpu_register() && opr1->is_single_cpu()
2983             && opr2->is_constant()
2984             && opr2->type() == T_INT) {
2985           reg_opr = opr1;
2986           reg = opr1->as_register();
2987           first_key = opr2->as_constant_ptr()->as_jint();
2988           next_key = first_key + 1;
2989           state = cmp_s;
2990           goto next_state;
2991         }
2992         break;
2993       }
2994       break;
2995     case cmp_s:
2996       switch (op->code()) {
2997       case lir_branch:
2998         if (op->as_OpBranch()->cond() == lir_cond_equal) {
2999           state = beq_s;
3000           last_insn = i;
3001           goto next_state;
3002         }
3003       }
3004       state = start_s;
3005       break;
3006     case beq_s:
3007       switch (op->code()) {
3008       case lir_cmp: {
3009         LIR_Opr opr1 = op->as_Op2()->in_opr1();
3010         LIR_Opr opr2 = op->as_Op2()->in_opr2();
3011         if (opr1->is_cpu_register() && opr1->is_single_cpu()
3012             && opr1->as_register() == reg
3013             && opr2->is_constant()
3014             && opr2->type() == T_INT
3015             && opr2->as_constant_ptr()->as_jint() == next_key) {
3016           last_key = next_key;
3017           next_key++;
3018           state = cmp_s;
3019           goto next_state;
3020         }
3021       }
3022       }
3023       last_key = next_key;
3024       state = start_s;
3025       break;
3026     default:
3027       assert(false, "impossible state");
3028     }
3029     if (state == start_s) {
3030       if (first_key < last_key - 5L && reg != noreg) {
3031         {
3032           // printf("found run register %d starting at insn %d low value %d high value %d\n",
3033           //        reg->encoding(),
3034           //        start_insn, first_key, last_key);
3035           //   for (int i = 0; i < inst->length(); i++) {
3036           //     inst->at(i)->print();
3037           //     tty->print("\n");
3038           //   }
3039           //   tty->print("\n");
3040         }
3041 
3042         struct tableswitch *sw = &switches[tableswitch_count];
3043         sw->_insn_index = start_insn, sw->_first_key = first_key,
3044           sw->_last_key = last_key, sw->_reg = reg;
3045         inst->insert_before(last_insn + 1, new LIR_OpLabel(&sw->_after));
3046         {
3047           // Insert the new table of branches
3048           int offset = last_insn;
3049           for (int n = first_key; n < last_key; n++) {
3050             inst->insert_before
3051               (last_insn + 1,
3052                new LIR_OpBranch(lir_cond_always, T_ILLEGAL,
3053                                 inst->at(offset)->as_OpBranch()->label()));
3054             offset -= 2, i++;
3055           }
3056         }
3057         // Delete all the old compare-and-branch instructions
3058         for (int n = first_key; n < last_key; n++) {
3059           inst->remove_at(start_insn);
3060           inst->remove_at(start_insn);
3061         }
3062         // Insert the tableswitch instruction
3063         inst->insert_before(start_insn,
3064                             new LIR_Op2(lir_cmp, lir_cond_always,
3065                                         LIR_OprFact::intConst(tableswitch_count),
3066                                         reg_opr));
3067         inst->insert_before(start_insn + 1, new LIR_OpLabel(&sw->_branches));
3068         tableswitch_count++;
3069       }
3070       reg = noreg;
3071       last_key = -2147483648;
3072     }
3073   next_state:
3074     ;
3075   }
3076 #endif
3077 }
3078 
3079 void LIR_Assembler::atomic_op(LIR_Code code, LIR_Opr src, LIR_Opr data, LIR_Opr dest, LIR_Opr tmp_op) {
3080   Address addr = as_Address(src->as_address_ptr());
3081   BasicType type = src->type();
3082   bool is_oop = type == T_OBJECT || type == T_ARRAY;
3083 
3084   void (MacroAssembler::* add)(Register prev, RegisterOrConstant incr, Register addr);
3085   void (MacroAssembler::* xchg)(Register prev, Register newv, Register addr);
3086 
3087   switch(type) {
3088   case T_INT:
3089     xchg = &MacroAssembler::atomic_xchgalw;
3090     add = &MacroAssembler::atomic_addalw;
3091     break;
3092   case T_LONG:
3093     xchg = &MacroAssembler::atomic_xchgal;
3094     add = &MacroAssembler::atomic_addal;
3095     break;
3096   case T_OBJECT:
3097   case T_ARRAY:
3098     if (UseCompressedOops) {
3099       xchg = &MacroAssembler::atomic_xchgalw;
3100       add = &MacroAssembler::atomic_addalw;
3101     } else {
3102       xchg = &MacroAssembler::atomic_xchgal;
3103       add = &MacroAssembler::atomic_addal;
3104     }
3105     break;
3106   default:
3107     ShouldNotReachHere();
3108     xchg = &MacroAssembler::atomic_xchgal;
3109     add = &MacroAssembler::atomic_addal; // unreachable
3110   }
3111 
3112   switch (code) {
3113   case lir_xadd:
3114     {
3115       RegisterOrConstant inc;
3116       Register tmp = as_reg(tmp_op);
3117       Register dst = as_reg(dest);
3118       if (data->is_constant()) {
3119         inc = RegisterOrConstant(as_long(data));
3120         assert_different_registers(dst, addr.base(), tmp,
3121                                    rscratch1, rscratch2);
3122       } else {
3123         inc = RegisterOrConstant(as_reg(data));
3124         assert_different_registers(inc.as_register(), dst, addr.base(), tmp,
3125                                    rscratch1, rscratch2);
3126       }
3127       __ lea(tmp, addr);
3128       (_masm->*add)(dst, inc, tmp);
3129       break;
3130     }
3131   case lir_xchg:
3132     {
3133       Register tmp = tmp_op->as_register();
3134       Register obj = as_reg(data);
3135       Register dst = as_reg(dest);
3136       if (is_oop && UseCompressedOops) {
3137         __ encode_heap_oop(rscratch2, obj);
3138         obj = rscratch2;
3139       }
3140       assert_different_registers(obj, addr.base(), tmp, rscratch1, dst);
3141       __ lea(tmp, addr);
3142       (_masm->*xchg)(dst, obj, tmp);
3143       if (is_oop && UseCompressedOops) {
3144         __ decode_heap_oop(dst);
3145       }
3146     }
3147     break;
3148   default:
3149     ShouldNotReachHere();
3150   }
3151   __ membar(__ AnyAny);
3152 }
3153 
3154 #undef __