1 /*
   2  * Copyright (c) 2000, 2019, Oracle and/or its affiliates. All rights reserved.
   3  * Copyright (c) 2014, Red Hat Inc. All rights reserved.
   4  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   5  *
   6  * This code is free software; you can redistribute it and/or modify it
   7  * under the terms of the GNU General Public License version 2 only, as
   8  * published by the Free Software Foundation.
   9  *
  10  * This code is distributed in the hope that it will be useful, but WITHOUT
  11  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  12  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  13  * version 2 for more details (a copy is included in the LICENSE file that
  14  * accompanied this code).
  15  *
  16  * You should have received a copy of the GNU General Public License version
  17  * 2 along with this work; if not, write to the Free Software Foundation,
  18  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  19  *
  20  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  21  * or visit www.oracle.com if you need additional information or have any
  22  * questions.
  23  *
  24  */
  25 
  26 #include "precompiled.hpp"
  27 #include "asm/macroAssembler.inline.hpp"
  28 #include "asm/assembler.hpp"
  29 #include "c1/c1_CodeStubs.hpp"
  30 #include "c1/c1_Compilation.hpp"
  31 #include "c1/c1_LIRAssembler.hpp"
  32 #include "c1/c1_MacroAssembler.hpp"
  33 #include "c1/c1_Runtime1.hpp"
  34 #include "c1/c1_ValueStack.hpp"
  35 #include "ci/ciArrayKlass.hpp"
  36 #include "ci/ciInstance.hpp"
  37 #include "ci/ciValueKlass.hpp"
  38 #include "code/compiledIC.hpp"
  39 #include "gc/shared/barrierSet.hpp"
  40 #include "gc/shared/cardTableBarrierSet.hpp"
  41 #include "gc/shared/collectedHeap.hpp"
  42 #include "nativeInst_aarch64.hpp"
  43 #include "oops/objArrayKlass.hpp"
  44 #include "oops/oop.inline.hpp"
  45 #include "runtime/frame.inline.hpp"
  46 #include "runtime/sharedRuntime.hpp"
  47 #include "vmreg_aarch64.inline.hpp"
  48 
  49 
  50 
  51 #ifndef PRODUCT
  52 #define COMMENT(x)   do { __ block_comment(x); } while (0)
  53 #else
  54 #define COMMENT(x)
  55 #endif
  56 
  57 NEEDS_CLEANUP // remove this definitions ?
  58 const Register IC_Klass    = rscratch2;   // where the IC klass is cached
  59 const Register SYNC_header = r0;   // synchronization header
  60 const Register SHIFT_count = r0;   // where count for shift operations must be
  61 
  62 #define __ _masm->
  63 
  64 
  65 static void select_different_registers(Register preserve,
  66                                        Register extra,
  67                                        Register &tmp1,
  68                                        Register &tmp2) {
  69   if (tmp1 == preserve) {
  70     assert_different_registers(tmp1, tmp2, extra);
  71     tmp1 = extra;
  72   } else if (tmp2 == preserve) {
  73     assert_different_registers(tmp1, tmp2, extra);
  74     tmp2 = extra;
  75   }
  76   assert_different_registers(preserve, tmp1, tmp2);
  77 }
  78 
  79 
  80 
  81 static void select_different_registers(Register preserve,
  82                                        Register extra,
  83                                        Register &tmp1,
  84                                        Register &tmp2,
  85                                        Register &tmp3) {
  86   if (tmp1 == preserve) {
  87     assert_different_registers(tmp1, tmp2, tmp3, extra);
  88     tmp1 = extra;
  89   } else if (tmp2 == preserve) {
  90     assert_different_registers(tmp1, tmp2, tmp3, extra);
  91     tmp2 = extra;
  92   } else if (tmp3 == preserve) {
  93     assert_different_registers(tmp1, tmp2, tmp3, extra);
  94     tmp3 = extra;
  95   }
  96   assert_different_registers(preserve, tmp1, tmp2, tmp3);
  97 }
  98 
  99 
 100 bool LIR_Assembler::is_small_constant(LIR_Opr opr) { Unimplemented(); return false; }
 101 
 102 
 103 LIR_Opr LIR_Assembler::receiverOpr() {
 104   return FrameMap::receiver_opr;
 105 }
 106 
 107 LIR_Opr LIR_Assembler::osrBufferPointer() {
 108   return FrameMap::as_pointer_opr(receiverOpr()->as_register());
 109 }
 110 
 111 //--------------fpu register translations-----------------------
 112 
 113 
 114 address LIR_Assembler::float_constant(float f) {
 115   address const_addr = __ float_constant(f);
 116   if (const_addr == NULL) {
 117     bailout("const section overflow");
 118     return __ code()->consts()->start();
 119   } else {
 120     return const_addr;
 121   }
 122 }
 123 
 124 
 125 address LIR_Assembler::double_constant(double d) {
 126   address const_addr = __ double_constant(d);
 127   if (const_addr == NULL) {
 128     bailout("const section overflow");
 129     return __ code()->consts()->start();
 130   } else {
 131     return const_addr;
 132   }
 133 }
 134 
 135 address LIR_Assembler::int_constant(jlong n) {
 136   address const_addr = __ long_constant(n);
 137   if (const_addr == NULL) {
 138     bailout("const section overflow");
 139     return __ code()->consts()->start();
 140   } else {
 141     return const_addr;
 142   }
 143 }
 144 
 145 void LIR_Assembler::set_24bit_FPU() { Unimplemented(); }
 146 
 147 void LIR_Assembler::reset_FPU() { Unimplemented(); }
 148 
 149 void LIR_Assembler::fpop() { Unimplemented(); }
 150 
 151 void LIR_Assembler::fxch(int i) { Unimplemented(); }
 152 
 153 void LIR_Assembler::fld(int i) { Unimplemented(); }
 154 
 155 void LIR_Assembler::ffree(int i) { Unimplemented(); }
 156 
 157 void LIR_Assembler::breakpoint() { Unimplemented(); }
 158 
 159 void LIR_Assembler::push(LIR_Opr opr) { Unimplemented(); }
 160 
 161 void LIR_Assembler::pop(LIR_Opr opr) { Unimplemented(); }
 162 
 163 bool LIR_Assembler::is_literal_address(LIR_Address* addr) { Unimplemented(); return false; }
 164 //-------------------------------------------
 165 
 166 static Register as_reg(LIR_Opr op) {
 167   return op->is_double_cpu() ? op->as_register_lo() : op->as_register();
 168 }
 169 
 170 static jlong as_long(LIR_Opr data) {
 171   jlong result;
 172   switch (data->type()) {
 173   case T_INT:
 174     result = (data->as_jint());
 175     break;
 176   case T_LONG:
 177     result = (data->as_jlong());
 178     break;
 179   default:
 180     ShouldNotReachHere();
 181     result = 0;  // unreachable
 182   }
 183   return result;
 184 }
 185 
 186 Address LIR_Assembler::as_Address(LIR_Address* addr, Register tmp) {
 187   Register base = addr->base()->as_pointer_register();
 188   LIR_Opr opr = addr->index();
 189   if (opr->is_cpu_register()) {
 190     Register index;
 191     if (opr->is_single_cpu())
 192       index = opr->as_register();
 193     else
 194       index = opr->as_register_lo();
 195     assert(addr->disp() == 0, "must be");
 196     switch(opr->type()) {
 197       case T_INT:
 198         return Address(base, index, Address::sxtw(addr->scale()));
 199       case T_LONG:
 200         return Address(base, index, Address::lsl(addr->scale()));
 201       default:
 202         ShouldNotReachHere();
 203       }
 204   } else  {
 205     intptr_t addr_offset = intptr_t(addr->disp());
 206     if (Address::offset_ok_for_immed(addr_offset, addr->scale()))
 207       return Address(base, addr_offset, Address::lsl(addr->scale()));
 208     else {
 209       __ mov(tmp, addr_offset);
 210       return Address(base, tmp, Address::lsl(addr->scale()));
 211     }
 212   }
 213   return Address();
 214 }
 215 
 216 Address LIR_Assembler::as_Address_hi(LIR_Address* addr) {
 217   ShouldNotReachHere();
 218   return Address();
 219 }
 220 
 221 Address LIR_Assembler::as_Address(LIR_Address* addr) {
 222   return as_Address(addr, rscratch1);
 223 }
 224 
 225 Address LIR_Assembler::as_Address_lo(LIR_Address* addr) {
 226   return as_Address(addr, rscratch1);  // Ouch
 227   // FIXME: This needs to be much more clever.  See x86.
 228 }
 229 
 230 
 231 void LIR_Assembler::osr_entry() {
 232   offsets()->set_value(CodeOffsets::OSR_Entry, code_offset());
 233   BlockBegin* osr_entry = compilation()->hir()->osr_entry();
 234   ValueStack* entry_state = osr_entry->state();
 235   int number_of_locks = entry_state->locks_size();
 236 
 237   // we jump here if osr happens with the interpreter
 238   // state set up to continue at the beginning of the
 239   // loop that triggered osr - in particular, we have
 240   // the following registers setup:
 241   //
 242   // r2: osr buffer
 243   //
 244 
 245   // build frame
 246   ciMethod* m = compilation()->method();
 247   __ build_frame(initial_frame_size_in_bytes(), bang_size_in_bytes(), needs_stack_repair(), NULL); 
 248 
 249   // OSR buffer is
 250   //
 251   // locals[nlocals-1..0]
 252   // monitors[0..number_of_locks]
 253   //
 254   // locals is a direct copy of the interpreter frame so in the osr buffer
 255   // so first slot in the local array is the last local from the interpreter
 256   // and last slot is local[0] (receiver) from the interpreter
 257   //
 258   // Similarly with locks. The first lock slot in the osr buffer is the nth lock
 259   // from the interpreter frame, the nth lock slot in the osr buffer is 0th lock
 260   // in the interpreter frame (the method lock if a sync method)
 261 
 262   // Initialize monitors in the compiled activation.
 263   //   r2: pointer to osr buffer
 264   //
 265   // All other registers are dead at this point and the locals will be
 266   // copied into place by code emitted in the IR.
 267 
 268   Register OSR_buf = osrBufferPointer()->as_pointer_register();
 269   { assert(frame::interpreter_frame_monitor_size() == BasicObjectLock::size(), "adjust code below");
 270     int monitor_offset = BytesPerWord * method()->max_locals() +
 271       (2 * BytesPerWord) * (number_of_locks - 1);
 272     // SharedRuntime::OSR_migration_begin() packs BasicObjectLocks in
 273     // the OSR buffer using 2 word entries: first the lock and then
 274     // the oop.
 275     for (int i = 0; i < number_of_locks; i++) {
 276       int slot_offset = monitor_offset - ((i * 2) * BytesPerWord);
 277 #ifdef ASSERT
 278       // verify the interpreter's monitor has a non-null object
 279       {
 280         Label L;
 281         __ ldr(rscratch1, Address(OSR_buf, slot_offset + 1*BytesPerWord));
 282         __ cbnz(rscratch1, L);
 283         __ stop("locked object is NULL");
 284         __ bind(L);
 285       }
 286 #endif
 287       __ ldr(r19, Address(OSR_buf, slot_offset + 0));
 288       __ str(r19, frame_map()->address_for_monitor_lock(i));
 289       __ ldr(r19, Address(OSR_buf, slot_offset + 1*BytesPerWord));
 290       __ str(r19, frame_map()->address_for_monitor_object(i));
 291     }
 292   }
 293 }
 294 
 295 
 296 // inline cache check; done before the frame is built.
 297 int LIR_Assembler::check_icache() {
 298   Register receiver = FrameMap::receiver_opr->as_register();
 299   Register ic_klass = IC_Klass;
 300   int start_offset = __ offset();
 301   __ inline_cache_check(receiver, ic_klass);
 302 
 303   // if icache check fails, then jump to runtime routine
 304   // Note: RECEIVER must still contain the receiver!
 305   Label dont;
 306   __ br(Assembler::EQ, dont);
 307   __ far_jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
 308 
 309   // We align the verified entry point unless the method body
 310   // (including its inline cache check) will fit in a single 64-byte
 311   // icache line.
 312   if (! method()->is_accessor() || __ offset() - start_offset > 4 * 4) {
 313     // force alignment after the cache check.
 314     __ align(CodeEntryAlignment);
 315   }
 316 
 317   __ bind(dont);
 318   return start_offset;
 319 }
 320 
 321 void LIR_Assembler::clinit_barrier(ciMethod* method) {
 322   ShouldNotReachHere(); // not implemented
 323 }
 324 
 325 void LIR_Assembler::jobject2reg(jobject o, Register reg) {
 326   if (o == NULL) {
 327     __ mov(reg, zr);
 328   } else {
 329     __ movoop(reg, o, /*immediate*/true);
 330   }
 331 }
 332 
 333 void LIR_Assembler::deoptimize_trap(CodeEmitInfo *info) {
 334   address target = NULL;
 335   relocInfo::relocType reloc_type = relocInfo::none;
 336 
 337   switch (patching_id(info)) {
 338   case PatchingStub::access_field_id:
 339     target = Runtime1::entry_for(Runtime1::access_field_patching_id);
 340     reloc_type = relocInfo::section_word_type;
 341     break;
 342   case PatchingStub::load_klass_id:
 343     target = Runtime1::entry_for(Runtime1::load_klass_patching_id);
 344     reloc_type = relocInfo::metadata_type;
 345     break;
 346   case PatchingStub::load_mirror_id:
 347     target = Runtime1::entry_for(Runtime1::load_mirror_patching_id);
 348     reloc_type = relocInfo::oop_type;
 349     break;
 350   case PatchingStub::load_appendix_id:
 351     target = Runtime1::entry_for(Runtime1::load_appendix_patching_id);
 352     reloc_type = relocInfo::oop_type;
 353     break;
 354   default: ShouldNotReachHere();
 355   }
 356 
 357   __ far_call(RuntimeAddress(target));
 358   add_call_info_here(info);
 359 }
 360 
 361 void LIR_Assembler::jobject2reg_with_patching(Register reg, CodeEmitInfo *info) {
 362   deoptimize_trap(info);
 363 }
 364 
 365 
 366 // This specifies the rsp decrement needed to build the frame
 367 int LIR_Assembler::initial_frame_size_in_bytes() const {
 368   // if rounding, must let FrameMap know!
 369 
 370   // The frame_map records size in slots (32bit word)
 371 
 372   // subtract two words to account for return address and link
 373   return (frame_map()->framesize() - (2*VMRegImpl::slots_per_word))  * VMRegImpl::stack_slot_size;
 374 }
 375 
 376 
 377 int LIR_Assembler::emit_exception_handler() {
 378   // if the last instruction is a call (typically to do a throw which
 379   // is coming at the end after block reordering) the return address
 380   // must still point into the code area in order to avoid assertion
 381   // failures when searching for the corresponding bci => add a nop
 382   // (was bug 5/14/1999 - gri)
 383   __ nop();
 384 
 385   // generate code for exception handler
 386   address handler_base = __ start_a_stub(exception_handler_size());
 387   if (handler_base == NULL) {
 388     // not enough space left for the handler
 389     bailout("exception handler overflow");
 390     return -1;
 391   }
 392 
 393   int offset = code_offset();
 394 
 395   // the exception oop and pc are in r0, and r3
 396   // no other registers need to be preserved, so invalidate them
 397   __ invalidate_registers(false, true, true, false, true, true);
 398 
 399   // check that there is really an exception
 400   __ verify_not_null_oop(r0);
 401 
 402   // search an exception handler (r0: exception oop, r3: throwing pc)
 403   __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::handle_exception_from_callee_id)));  __ should_not_reach_here();
 404   guarantee(code_offset() - offset <= exception_handler_size(), "overflow");
 405   __ end_a_stub();
 406 
 407   return offset;
 408 }
 409 
 410 
 411 // Emit the code to remove the frame from the stack in the exception
 412 // unwind path.
 413 int LIR_Assembler::emit_unwind_handler() {
 414 #ifndef PRODUCT
 415   if (CommentedAssembly) {
 416     _masm->block_comment("Unwind handler");
 417   }
 418 #endif
 419 
 420   int offset = code_offset();
 421 
 422   // Fetch the exception from TLS and clear out exception related thread state
 423   __ ldr(r0, Address(rthread, JavaThread::exception_oop_offset()));
 424   __ str(zr, Address(rthread, JavaThread::exception_oop_offset()));
 425   __ str(zr, Address(rthread, JavaThread::exception_pc_offset()));
 426 
 427   __ bind(_unwind_handler_entry);
 428   __ verify_not_null_oop(r0);
 429   if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
 430     __ mov(r19, r0);  // Preserve the exception
 431   }
 432 
 433   // Preform needed unlocking
 434   MonitorExitStub* stub = NULL;
 435   if (method()->is_synchronized()) {
 436     monitor_address(0, FrameMap::r0_opr);
 437     stub = new MonitorExitStub(FrameMap::r0_opr, true, 0);
 438     __ unlock_object(r5, r4, r0, *stub->entry());
 439     __ bind(*stub->continuation());
 440   }
 441 
 442   if (compilation()->env()->dtrace_method_probes()) {
 443     __ call_Unimplemented();
 444 #if 0
 445     __ movptr(Address(rsp, 0), rax);
 446     __ mov_metadata(Address(rsp, sizeof(void*)), method()->constant_encoding());
 447     __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_exit)));
 448 #endif
 449   }
 450 
 451   if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
 452     __ mov(r0, r19);  // Restore the exception
 453   }
 454 
 455   // remove the activation and dispatch to the unwind handler
 456   __ block_comment("remove_frame and dispatch to the unwind handler");
 457   __ remove_frame(initial_frame_size_in_bytes(), needs_stack_repair());
 458   __ far_jump(RuntimeAddress(Runtime1::entry_for(Runtime1::unwind_exception_id)));
 459 
 460   // Emit the slow path assembly
 461   if (stub != NULL) {
 462     stub->emit_code(this);
 463   }
 464 
 465   return offset;
 466 }
 467 
 468 
 469 int LIR_Assembler::emit_deopt_handler() {
 470   // if the last instruction is a call (typically to do a throw which
 471   // is coming at the end after block reordering) the return address
 472   // must still point into the code area in order to avoid assertion
 473   // failures when searching for the corresponding bci => add a nop
 474   // (was bug 5/14/1999 - gri)
 475   __ nop();
 476 
 477   // generate code for exception handler
 478   address handler_base = __ start_a_stub(deopt_handler_size());
 479   if (handler_base == NULL) {
 480     // not enough space left for the handler
 481     bailout("deopt handler overflow");
 482     return -1;
 483   }
 484 
 485   int offset = code_offset();
 486 
 487   __ adr(lr, pc());
 488   __ far_jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
 489   guarantee(code_offset() - offset <= deopt_handler_size(), "overflow");
 490   __ end_a_stub();
 491 
 492   return offset;
 493 }
 494 
 495 void LIR_Assembler::add_debug_info_for_branch(address adr, CodeEmitInfo* info) {
 496   _masm->code_section()->relocate(adr, relocInfo::poll_type);
 497   int pc_offset = code_offset();
 498   flush_debug_info(pc_offset);
 499   info->record_debug_info(compilation()->debug_info_recorder(), pc_offset);
 500   if (info->exception_handlers() != NULL) {
 501     compilation()->add_exception_handlers_for_pco(pc_offset, info->exception_handlers());
 502   }
 503 }
 504 
 505 void LIR_Assembler::return_op(LIR_Opr result) {
 506   assert(result->is_illegal() || !result->is_single_cpu() || result->as_register() == r0, "word returns are in r0,");
 507 
 508   ciMethod* method = compilation()->method();
 509 
 510   if (ValueTypeReturnedAsFields && method->signature()->returns_never_null()) {
 511     ciType* return_type = method->return_type();
 512     if (return_type->is_valuetype()) {
 513       ciValueKlass* vk = return_type->as_value_klass();
 514       if (vk->can_be_returned_as_fields()) {
 515         address unpack_handler = vk->unpack_handler();
 516         assert(unpack_handler != NULL, "must be");
 517         __ far_call(RuntimeAddress(unpack_handler));
 518         // At this point, rax points to the value object (for interpreter or C1 caller).
 519         // The fields of the object are copied into registers (for C2 caller).
 520       }
 521     }
 522   }
 523 
 524   // Pop the stack before the safepoint code
 525   __ remove_frame(initial_frame_size_in_bytes(), needs_stack_repair());
 526 
 527   if (StackReservedPages > 0 && compilation()->has_reserved_stack_access()) {
 528     __ reserved_stack_check();
 529   }
 530 
 531   address polling_page(os::get_polling_page());
 532   __ read_polling_page(rscratch1, polling_page, relocInfo::poll_return_type);
 533   __ ret(lr);
 534 }
 535 
 536 int LIR_Assembler::store_value_type_fields_to_buf(ciValueKlass* vk) { 
 537  return (__ store_value_type_fields_to_buf(vk, false));
 538 }
 539 
 540 int LIR_Assembler::safepoint_poll(LIR_Opr tmp, CodeEmitInfo* info) {
 541   address polling_page(os::get_polling_page());
 542   guarantee(info != NULL, "Shouldn't be NULL");
 543   assert(os::is_poll_address(polling_page), "should be");
 544   __ get_polling_page(rscratch1, polling_page, relocInfo::poll_type);
 545   add_debug_info_for_branch(info);  // This isn't just debug info:
 546                                     // it's the oop map
 547   __ read_polling_page(rscratch1, relocInfo::poll_type);
 548   return __ offset();
 549 }
 550 
 551 
 552 void LIR_Assembler::move_regs(Register from_reg, Register to_reg) {
 553   if (from_reg == r31_sp)
 554     from_reg = sp;
 555   if (to_reg == r31_sp)
 556     to_reg = sp;
 557   __ mov(to_reg, from_reg);
 558 }
 559 
 560 void LIR_Assembler::swap_reg(Register a, Register b) { Unimplemented(); }
 561 
 562 
 563 void LIR_Assembler::const2reg(LIR_Opr src, LIR_Opr dest, LIR_PatchCode patch_code, CodeEmitInfo* info) {
 564   assert(src->is_constant(), "should not call otherwise");
 565   assert(dest->is_register(), "should not call otherwise");
 566   LIR_Const* c = src->as_constant_ptr();
 567 
 568   switch (c->type()) {
 569     case T_INT: {
 570       assert(patch_code == lir_patch_none, "no patching handled here");
 571       __ movw(dest->as_register(), c->as_jint());
 572       break;
 573     }
 574 
 575     case T_ADDRESS: {
 576       assert(patch_code == lir_patch_none, "no patching handled here");
 577       __ mov(dest->as_register(), c->as_jint());
 578       break;
 579     }
 580 
 581     case T_LONG: {
 582       assert(patch_code == lir_patch_none, "no patching handled here");
 583       __ mov(dest->as_register_lo(), (intptr_t)c->as_jlong());
 584       break;
 585     }
 586 
 587     case T_VALUETYPE:
 588     case T_OBJECT: {
 589         if (patch_code != lir_patch_none) {
 590           jobject2reg_with_patching(dest->as_register(), info);
 591         } else {
 592           jobject2reg(c->as_jobject(), dest->as_register());
 593         }
 594       break;
 595     }
 596 
 597     case T_METADATA: {
 598       if (patch_code != lir_patch_none) {
 599         klass2reg_with_patching(dest->as_register(), info);
 600       } else {
 601         __ mov_metadata(dest->as_register(), c->as_metadata());
 602       }
 603       break;
 604     }
 605 
 606     case T_FLOAT: {
 607       if (__ operand_valid_for_float_immediate(c->as_jfloat())) {
 608         __ fmovs(dest->as_float_reg(), (c->as_jfloat()));
 609       } else {
 610         __ adr(rscratch1, InternalAddress(float_constant(c->as_jfloat())));
 611         __ ldrs(dest->as_float_reg(), Address(rscratch1));
 612       }
 613       break;
 614     }
 615 
 616     case T_DOUBLE: {
 617       if (__ operand_valid_for_float_immediate(c->as_jdouble())) {
 618         __ fmovd(dest->as_double_reg(), (c->as_jdouble()));
 619       } else {
 620         __ adr(rscratch1, InternalAddress(double_constant(c->as_jdouble())));
 621         __ ldrd(dest->as_double_reg(), Address(rscratch1));
 622       }
 623       break;
 624     }
 625 
 626     default:
 627       ShouldNotReachHere();
 628   }
 629 }
 630 
 631 void LIR_Assembler::const2stack(LIR_Opr src, LIR_Opr dest) {
 632   LIR_Const* c = src->as_constant_ptr();
 633   switch (c->type()) {
 634   case T_VALUETYPE: 
 635   case T_OBJECT:
 636     {
 637       if (! c->as_jobject())
 638         __ str(zr, frame_map()->address_for_slot(dest->single_stack_ix()));
 639       else {
 640         const2reg(src, FrameMap::rscratch1_opr, lir_patch_none, NULL);
 641         reg2stack(FrameMap::rscratch1_opr, dest, c->type(), false);
 642       }
 643     }
 644     break;
 645   case T_ADDRESS:
 646     {
 647       const2reg(src, FrameMap::rscratch1_opr, lir_patch_none, NULL);
 648       reg2stack(FrameMap::rscratch1_opr, dest, c->type(), false);
 649     }
 650   case T_INT:
 651   case T_FLOAT:
 652     {
 653       Register reg = zr;
 654       if (c->as_jint_bits() == 0)
 655         __ strw(zr, frame_map()->address_for_slot(dest->single_stack_ix()));
 656       else {
 657         __ movw(rscratch1, c->as_jint_bits());
 658         __ strw(rscratch1, frame_map()->address_for_slot(dest->single_stack_ix()));
 659       }
 660     }
 661     break;
 662   case T_LONG:
 663   case T_DOUBLE:
 664     {
 665       Register reg = zr;
 666       if (c->as_jlong_bits() == 0)
 667         __ str(zr, frame_map()->address_for_slot(dest->double_stack_ix(),
 668                                                  lo_word_offset_in_bytes));
 669       else {
 670         __ mov(rscratch1, (intptr_t)c->as_jlong_bits());
 671         __ str(rscratch1, frame_map()->address_for_slot(dest->double_stack_ix(),
 672                                                         lo_word_offset_in_bytes));
 673       }
 674     }
 675     break;
 676   default:
 677     ShouldNotReachHere();
 678   }
 679 }
 680 
 681 void LIR_Assembler::const2mem(LIR_Opr src, LIR_Opr dest, BasicType type, CodeEmitInfo* info, bool wide) {
 682   assert(src->is_constant(), "should not call otherwise");
 683   LIR_Const* c = src->as_constant_ptr();
 684   LIR_Address* to_addr = dest->as_address_ptr();
 685 
 686   void (Assembler::* insn)(Register Rt, const Address &adr);
 687 
 688   switch (type) {
 689   case T_ADDRESS:
 690     assert(c->as_jint() == 0, "should be");
 691     insn = &Assembler::str;
 692     break;
 693   case T_LONG:
 694     assert(c->as_jlong() == 0, "should be");
 695     insn = &Assembler::str;
 696     break;
 697   case T_INT:
 698     assert(c->as_jint() == 0, "should be");
 699     insn = &Assembler::strw;
 700     break;
 701   case T_VALUETYPE: 
 702   case T_OBJECT:
 703   case T_ARRAY:
 704     // Non-null case is not handled on aarch64 but handled on x86
 705     // FIXME: do we need to add it here?
 706     assert(c->as_jobject() == 0, "should be");
 707     if (UseCompressedOops && !wide) {
 708       insn = &Assembler::strw;
 709     } else {
 710       insn = &Assembler::str;
 711     }
 712     break;
 713   case T_CHAR:
 714   case T_SHORT:
 715     assert(c->as_jint() == 0, "should be");
 716     insn = &Assembler::strh;
 717     break;
 718   case T_BOOLEAN:
 719   case T_BYTE:
 720     assert(c->as_jint() == 0, "should be");
 721     insn = &Assembler::strb;
 722     break;
 723   default:
 724     ShouldNotReachHere();
 725     insn = &Assembler::str;  // unreachable
 726   }
 727 
 728   if (info) add_debug_info_for_null_check_here(info);
 729   (_masm->*insn)(zr, as_Address(to_addr, rscratch1));
 730 }
 731 
 732 void LIR_Assembler::reg2reg(LIR_Opr src, LIR_Opr dest) {
 733   assert(src->is_register(), "should not call otherwise");
 734   assert(dest->is_register(), "should not call otherwise");
 735 
 736   // move between cpu-registers
 737   if (dest->is_single_cpu()) {
 738     if (src->type() == T_LONG) {
 739       // Can do LONG -> OBJECT
 740       move_regs(src->as_register_lo(), dest->as_register());
 741       return;
 742     }
 743     assert(src->is_single_cpu(), "must match");
 744     if (src->type() == T_OBJECT || src->type() == T_VALUETYPE) {
 745       __ verify_oop(src->as_register());
 746     }
 747     move_regs(src->as_register(), dest->as_register());
 748 
 749   } else if (dest->is_double_cpu()) {
 750     if (src->type() == T_OBJECT || src->type() == T_ARRAY || src->type() == T_VALUETYPE) {
 751       // Surprising to me but we can see move of a long to t_object
 752       __ verify_oop(src->as_register());
 753       move_regs(src->as_register(), dest->as_register_lo());
 754       return;
 755     }
 756     assert(src->is_double_cpu(), "must match");
 757     Register f_lo = src->as_register_lo();
 758     Register f_hi = src->as_register_hi();
 759     Register t_lo = dest->as_register_lo();
 760     Register t_hi = dest->as_register_hi();
 761     assert(f_hi == f_lo, "must be same");
 762     assert(t_hi == t_lo, "must be same");
 763     move_regs(f_lo, t_lo);
 764 
 765   } else if (dest->is_single_fpu()) {
 766     __ fmovs(dest->as_float_reg(), src->as_float_reg());
 767 
 768   } else if (dest->is_double_fpu()) {
 769     __ fmovd(dest->as_double_reg(), src->as_double_reg());
 770 
 771   } else {
 772     ShouldNotReachHere();
 773   }
 774 }
 775 
 776 void LIR_Assembler::reg2stack(LIR_Opr src, LIR_Opr dest, BasicType type, bool pop_fpu_stack) {
 777   if (src->is_single_cpu()) {
 778     if (type == T_ARRAY || type == T_OBJECT || type == T_VALUETYPE) {
 779       __ str(src->as_register(), frame_map()->address_for_slot(dest->single_stack_ix()));
 780       __ verify_oop(src->as_register());
 781     } else if (type == T_METADATA || type == T_DOUBLE) {
 782       __ str(src->as_register(), frame_map()->address_for_slot(dest->single_stack_ix()));
 783     } else {
 784       __ strw(src->as_register(), frame_map()->address_for_slot(dest->single_stack_ix()));
 785     }
 786 
 787   } else if (src->is_double_cpu()) {
 788     Address dest_addr_LO = frame_map()->address_for_slot(dest->double_stack_ix(), lo_word_offset_in_bytes);
 789     __ str(src->as_register_lo(), dest_addr_LO);
 790 
 791   } else if (src->is_single_fpu()) {
 792     Address dest_addr = frame_map()->address_for_slot(dest->single_stack_ix());
 793     __ strs(src->as_float_reg(), dest_addr);
 794 
 795   } else if (src->is_double_fpu()) {
 796     Address dest_addr = frame_map()->address_for_slot(dest->double_stack_ix());
 797     __ strd(src->as_double_reg(), dest_addr);
 798 
 799   } else {
 800     ShouldNotReachHere();
 801   }
 802 
 803 }
 804 
 805 
 806 void LIR_Assembler::reg2mem(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_PatchCode patch_code, CodeEmitInfo* info, bool pop_fpu_stack, bool wide, bool /* unaligned */) {
 807   LIR_Address* to_addr = dest->as_address_ptr();
 808   PatchingStub* patch = NULL;
 809   Register compressed_src = rscratch1;
 810 
 811   if (patch_code != lir_patch_none) {
 812     deoptimize_trap(info);
 813     return;
 814   }
 815 
 816   if (type == T_ARRAY || type == T_OBJECT || type == T_VALUETYPE) {
 817     __ verify_oop(src->as_register());
 818 
 819     if (UseCompressedOops && !wide) {
 820       __ encode_heap_oop(compressed_src, src->as_register());
 821     } else {
 822       compressed_src = src->as_register();
 823     }
 824   }
 825 
 826   int null_check_here = code_offset();
 827   switch (type) {
 828     case T_FLOAT: {
 829       __ strs(src->as_float_reg(), as_Address(to_addr));
 830       break;
 831     }
 832 
 833     case T_DOUBLE: {
 834       __ strd(src->as_double_reg(), as_Address(to_addr));
 835       break;
 836     }
 837 
 838     case T_VALUETYPE: // fall through
 839     case T_ARRAY:   // fall through
 840     case T_OBJECT:  // fall through
 841       if (UseCompressedOops && !wide) {
 842         __ strw(compressed_src, as_Address(to_addr, rscratch2));
 843       } else {
 844          __ str(compressed_src, as_Address(to_addr));
 845       }
 846       break;
 847     case T_METADATA:
 848       // We get here to store a method pointer to the stack to pass to
 849       // a dtrace runtime call. This can't work on 64 bit with
 850       // compressed klass ptrs: T_METADATA can be a compressed klass
 851       // ptr or a 64 bit method pointer.
 852       ShouldNotReachHere();
 853       __ str(src->as_register(), as_Address(to_addr));
 854       break;
 855     case T_ADDRESS:
 856       __ str(src->as_register(), as_Address(to_addr));
 857       break;
 858     case T_INT:
 859       __ strw(src->as_register(), as_Address(to_addr));
 860       break;
 861 
 862     case T_LONG: {
 863       __ str(src->as_register_lo(), as_Address_lo(to_addr));
 864       break;
 865     }
 866 
 867     case T_BYTE:    // fall through
 868     case T_BOOLEAN: {
 869       __ strb(src->as_register(), as_Address(to_addr));
 870       break;
 871     }
 872 
 873     case T_CHAR:    // fall through
 874     case T_SHORT:
 875       __ strh(src->as_register(), as_Address(to_addr));
 876       break;
 877 
 878     default:
 879       ShouldNotReachHere();
 880   }
 881   if (info != NULL) {
 882     add_debug_info_for_null_check(null_check_here, info);
 883   }
 884 }
 885 
 886 
 887 void LIR_Assembler::stack2reg(LIR_Opr src, LIR_Opr dest, BasicType type) {
 888   assert(src->is_stack(), "should not call otherwise");
 889   assert(dest->is_register(), "should not call otherwise");
 890 
 891   if (dest->is_single_cpu()) {
 892     if (type == T_ARRAY || type == T_OBJECT || type == T_VALUETYPE) {
 893       __ ldr(dest->as_register(), frame_map()->address_for_slot(src->single_stack_ix()));
 894       __ verify_oop(dest->as_register());
 895     } else if (type == T_METADATA) {
 896       __ ldr(dest->as_register(), frame_map()->address_for_slot(src->single_stack_ix()));
 897     } else {
 898       __ ldrw(dest->as_register(), frame_map()->address_for_slot(src->single_stack_ix()));
 899     }
 900 
 901   } else if (dest->is_double_cpu()) {
 902     Address src_addr_LO = frame_map()->address_for_slot(src->double_stack_ix(), lo_word_offset_in_bytes);
 903     __ ldr(dest->as_register_lo(), src_addr_LO);
 904 
 905   } else if (dest->is_single_fpu()) {
 906     Address src_addr = frame_map()->address_for_slot(src->single_stack_ix());
 907     __ ldrs(dest->as_float_reg(), src_addr);
 908 
 909   } else if (dest->is_double_fpu()) {
 910     Address src_addr = frame_map()->address_for_slot(src->double_stack_ix());
 911     __ ldrd(dest->as_double_reg(), src_addr);
 912 
 913   } else {
 914     ShouldNotReachHere();
 915   }
 916 }
 917 
 918 
 919 void LIR_Assembler::klass2reg_with_patching(Register reg, CodeEmitInfo* info) {
 920   address target = NULL;
 921   relocInfo::relocType reloc_type = relocInfo::none;
 922 
 923   switch (patching_id(info)) {
 924   case PatchingStub::access_field_id:
 925     target = Runtime1::entry_for(Runtime1::access_field_patching_id);
 926     reloc_type = relocInfo::section_word_type;
 927     break;
 928   case PatchingStub::load_klass_id:
 929     target = Runtime1::entry_for(Runtime1::load_klass_patching_id);
 930     reloc_type = relocInfo::metadata_type;
 931     break;
 932   case PatchingStub::load_mirror_id:
 933     target = Runtime1::entry_for(Runtime1::load_mirror_patching_id);
 934     reloc_type = relocInfo::oop_type;
 935     break;
 936   case PatchingStub::load_appendix_id:
 937     target = Runtime1::entry_for(Runtime1::load_appendix_patching_id);
 938     reloc_type = relocInfo::oop_type;
 939     break;
 940   default: ShouldNotReachHere();
 941   }
 942 
 943   __ far_call(RuntimeAddress(target));
 944   add_call_info_here(info);
 945 }
 946 
 947 void LIR_Assembler::stack2stack(LIR_Opr src, LIR_Opr dest, BasicType type) {
 948 
 949   LIR_Opr temp;
 950   if (type == T_LONG || type == T_DOUBLE)
 951     temp = FrameMap::rscratch1_long_opr;
 952   else
 953     temp = FrameMap::rscratch1_opr;
 954 
 955   stack2reg(src, temp, src->type());
 956   reg2stack(temp, dest, dest->type(), false);
 957 }
 958 
 959 
 960 void LIR_Assembler::mem2reg(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_PatchCode patch_code, CodeEmitInfo* info, bool wide, bool /* unaligned */) {
 961   LIR_Address* addr = src->as_address_ptr();
 962   LIR_Address* from_addr = src->as_address_ptr();
 963 
 964   if (addr->base()->type() == T_OBJECT || addr->base()->type() == T_VALUETYPE) { 
 965     __ verify_oop(addr->base()->as_pointer_register());
 966   }
 967 
 968   if (patch_code != lir_patch_none) {
 969     deoptimize_trap(info);
 970     return;
 971   }
 972 
 973   if (info != NULL) {
 974     add_debug_info_for_null_check_here(info);
 975   }
 976   int null_check_here = code_offset();
 977   switch (type) {
 978     case T_FLOAT: {
 979       __ ldrs(dest->as_float_reg(), as_Address(from_addr));
 980       break;
 981     }
 982 
 983     case T_DOUBLE: {
 984       __ ldrd(dest->as_double_reg(), as_Address(from_addr));
 985       break;
 986     }
 987 
 988     case T_VALUETYPE: // fall through
 989     case T_ARRAY:   // fall through
 990     case T_OBJECT:  // fall through
 991       if (UseCompressedOops && !wide) {
 992         __ ldrw(dest->as_register(), as_Address(from_addr));
 993       } else {
 994          __ ldr(dest->as_register(), as_Address(from_addr));
 995       }
 996       break;
 997     case T_METADATA:
 998       // We get here to store a method pointer to the stack to pass to
 999       // a dtrace runtime call. This can't work on 64 bit with
1000       // compressed klass ptrs: T_METADATA can be a compressed klass
1001       // ptr or a 64 bit method pointer.
1002       ShouldNotReachHere();
1003       __ ldr(dest->as_register(), as_Address(from_addr));
1004       break;
1005     case T_ADDRESS:
1006       // FIXME: OMG this is a horrible kludge.  Any offset from an
1007       // address that matches klass_offset_in_bytes() will be loaded
1008       // as a word, not a long.
1009       if (UseCompressedClassPointers && addr->disp() == oopDesc::klass_offset_in_bytes()) {
1010         __ ldrw(dest->as_register(), as_Address(from_addr));
1011       } else {
1012         __ ldr(dest->as_register(), as_Address(from_addr));
1013       }
1014       break;
1015     case T_INT:
1016       __ ldrw(dest->as_register(), as_Address(from_addr));
1017       break;
1018 
1019     case T_LONG: {
1020       __ ldr(dest->as_register_lo(), as_Address_lo(from_addr));
1021       break;
1022     }
1023 
1024     case T_BYTE:
1025       __ ldrsb(dest->as_register(), as_Address(from_addr));
1026       break;
1027     case T_BOOLEAN: {
1028       __ ldrb(dest->as_register(), as_Address(from_addr));
1029       break;
1030     }
1031 
1032     case T_CHAR:
1033       __ ldrh(dest->as_register(), as_Address(from_addr));
1034       break;
1035     case T_SHORT:
1036       __ ldrsh(dest->as_register(), as_Address(from_addr));
1037       break;
1038 
1039     default:
1040       ShouldNotReachHere();
1041   }
1042 
1043   if (type == T_ARRAY || type == T_OBJECT || type == T_VALUETYPE) {
1044     if (UseCompressedOops && !wide) {
1045       __ decode_heap_oop(dest->as_register());
1046     }
1047 
1048     if (!UseZGC) {
1049       // Load barrier has not yet been applied, so ZGC can't verify the oop here
1050       __ verify_oop(dest->as_register());
1051     }
1052   } else if (type == T_ADDRESS && addr->disp() == oopDesc::klass_offset_in_bytes()) {
1053     if (UseCompressedClassPointers) {
1054       __ andr(dest->as_register(), dest->as_register(), oopDesc::compressed_klass_mask());
1055       __ decode_klass_not_null(dest->as_register());
1056     } else {
1057       __   ubfm(dest->as_register(), dest->as_register(), 0, 63 - oopDesc::storage_props_nof_bits);
1058     }
1059   }
1060 }
1061 
1062 void LIR_Assembler::move(LIR_Opr src, LIR_Opr dst) {
1063   assert(dst->is_cpu_register(), "must be");
1064   assert(dst->type() == src->type(), "must be");
1065 
1066   if (src->is_cpu_register()) {
1067     reg2reg(src, dst);
1068   } else if (src->is_stack()) {
1069     stack2reg(src, dst, dst->type());
1070   } else if (src->is_constant()) {
1071     const2reg(src, dst, lir_patch_none, NULL);
1072   } else {
1073     ShouldNotReachHere();
1074   }
1075 }
1076 
1077 int LIR_Assembler::array_element_size(BasicType type) const {
1078   int elem_size = type2aelembytes(type);
1079   return exact_log2(elem_size);
1080 }
1081 
1082 
1083 void LIR_Assembler::emit_op3(LIR_Op3* op) {
1084   switch (op->code()) {
1085   case lir_idiv:
1086   case lir_irem:
1087     arithmetic_idiv(op->code(),
1088                     op->in_opr1(),
1089                     op->in_opr2(),
1090                     op->in_opr3(),
1091                     op->result_opr(),
1092                     op->info());
1093     break;
1094   case lir_fmad:
1095     __ fmaddd(op->result_opr()->as_double_reg(),
1096               op->in_opr1()->as_double_reg(),
1097               op->in_opr2()->as_double_reg(),
1098               op->in_opr3()->as_double_reg());
1099     break;
1100   case lir_fmaf:
1101     __ fmadds(op->result_opr()->as_float_reg(),
1102               op->in_opr1()->as_float_reg(),
1103               op->in_opr2()->as_float_reg(),
1104               op->in_opr3()->as_float_reg());
1105     break;
1106   default:      ShouldNotReachHere(); break;
1107   }
1108 }
1109 
1110 void LIR_Assembler::emit_opBranch(LIR_OpBranch* op) {
1111 #ifdef ASSERT
1112   assert(op->block() == NULL || op->block()->label() == op->label(), "wrong label");
1113   if (op->block() != NULL)  _branch_target_blocks.append(op->block());
1114   if (op->ublock() != NULL) _branch_target_blocks.append(op->ublock());
1115 #endif
1116 
1117   if (op->cond() == lir_cond_always) {
1118     if (op->info() != NULL) add_debug_info_for_branch(op->info());
1119     __ b(*(op->label()));
1120   } else {
1121     Assembler::Condition acond;
1122     if (op->code() == lir_cond_float_branch) {
1123       bool is_unordered = (op->ublock() == op->block());
1124       // Assembler::EQ does not permit unordered branches, so we add
1125       // another branch here.  Likewise, Assembler::NE does not permit
1126       // ordered branches.
1127       if ((is_unordered && op->cond() == lir_cond_equal)
1128           || (!is_unordered && op->cond() == lir_cond_notEqual))
1129         __ br(Assembler::VS, *(op->ublock()->label()));
1130       switch(op->cond()) {
1131       case lir_cond_equal:        acond = Assembler::EQ; break;
1132       case lir_cond_notEqual:     acond = Assembler::NE; break;
1133       case lir_cond_less:         acond = (is_unordered ? Assembler::LT : Assembler::LO); break;
1134       case lir_cond_lessEqual:    acond = (is_unordered ? Assembler::LE : Assembler::LS); break;
1135       case lir_cond_greaterEqual: acond = (is_unordered ? Assembler::HS : Assembler::GE); break;
1136       case lir_cond_greater:      acond = (is_unordered ? Assembler::HI : Assembler::GT); break;
1137       default:                    ShouldNotReachHere();
1138         acond = Assembler::EQ;  // unreachable
1139       }
1140     } else {
1141       switch (op->cond()) {
1142         case lir_cond_equal:        acond = Assembler::EQ; break;
1143         case lir_cond_notEqual:     acond = Assembler::NE; break;
1144         case lir_cond_less:         acond = Assembler::LT; break;
1145         case lir_cond_lessEqual:    acond = Assembler::LE; break;
1146         case lir_cond_greaterEqual: acond = Assembler::GE; break;
1147         case lir_cond_greater:      acond = Assembler::GT; break;
1148         case lir_cond_belowEqual:   acond = Assembler::LS; break;
1149         case lir_cond_aboveEqual:   acond = Assembler::HS; break;
1150         default:                    ShouldNotReachHere();
1151           acond = Assembler::EQ;  // unreachable
1152       }
1153     }
1154     __ br(acond,*(op->label()));
1155   }
1156 }
1157 
1158 
1159 
1160 void LIR_Assembler::emit_opConvert(LIR_OpConvert* op) {
1161   LIR_Opr src  = op->in_opr();
1162   LIR_Opr dest = op->result_opr();
1163 
1164   switch (op->bytecode()) {
1165     case Bytecodes::_i2f:
1166       {
1167         __ scvtfws(dest->as_float_reg(), src->as_register());
1168         break;
1169       }
1170     case Bytecodes::_i2d:
1171       {
1172         __ scvtfwd(dest->as_double_reg(), src->as_register());
1173         break;
1174       }
1175     case Bytecodes::_l2d:
1176       {
1177         __ scvtfd(dest->as_double_reg(), src->as_register_lo());
1178         break;
1179       }
1180     case Bytecodes::_l2f:
1181       {
1182         __ scvtfs(dest->as_float_reg(), src->as_register_lo());
1183         break;
1184       }
1185     case Bytecodes::_f2d:
1186       {
1187         __ fcvts(dest->as_double_reg(), src->as_float_reg());
1188         break;
1189       }
1190     case Bytecodes::_d2f:
1191       {
1192         __ fcvtd(dest->as_float_reg(), src->as_double_reg());
1193         break;
1194       }
1195     case Bytecodes::_i2c:
1196       {
1197         __ ubfx(dest->as_register(), src->as_register(), 0, 16);
1198         break;
1199       }
1200     case Bytecodes::_i2l:
1201       {
1202         __ sxtw(dest->as_register_lo(), src->as_register());
1203         break;
1204       }
1205     case Bytecodes::_i2s:
1206       {
1207         __ sxth(dest->as_register(), src->as_register());
1208         break;
1209       }
1210     case Bytecodes::_i2b:
1211       {
1212         __ sxtb(dest->as_register(), src->as_register());
1213         break;
1214       }
1215     case Bytecodes::_l2i:
1216       {
1217         _masm->block_comment("FIXME: This could be a no-op");
1218         __ uxtw(dest->as_register(), src->as_register_lo());
1219         break;
1220       }
1221     case Bytecodes::_d2l:
1222       {
1223         __ fcvtzd(dest->as_register_lo(), src->as_double_reg());
1224         break;
1225       }
1226     case Bytecodes::_f2i:
1227       {
1228         __ fcvtzsw(dest->as_register(), src->as_float_reg());
1229         break;
1230       }
1231     case Bytecodes::_f2l:
1232       {
1233         __ fcvtzs(dest->as_register_lo(), src->as_float_reg());
1234         break;
1235       }
1236     case Bytecodes::_d2i:
1237       {
1238         __ fcvtzdw(dest->as_register(), src->as_double_reg());
1239         break;
1240       }
1241     default: ShouldNotReachHere();
1242   }
1243 }
1244 
1245 void LIR_Assembler::emit_alloc_obj(LIR_OpAllocObj* op) {
1246   if (op->init_check()) {
1247     __ ldrb(rscratch1, Address(op->klass()->as_register(),
1248                                InstanceKlass::init_state_offset()));
1249     __ cmpw(rscratch1, InstanceKlass::fully_initialized);
1250     add_debug_info_for_null_check_here(op->stub()->info());
1251     __ br(Assembler::NE, *op->stub()->entry());
1252   }
1253   __ allocate_object(op->obj()->as_register(),
1254                      op->tmp1()->as_register(),
1255                      op->tmp2()->as_register(),
1256                      op->header_size(),
1257                      op->object_size(),
1258                      op->klass()->as_register(),
1259                      *op->stub()->entry());
1260   __ bind(*op->stub()->continuation());
1261 }
1262 
1263 void LIR_Assembler::emit_alloc_array(LIR_OpAllocArray* op) {
1264   Register len =  op->len()->as_register();
1265   __ uxtw(len, len);
1266 
1267   if (UseSlowPath || op->type() == T_VALUETYPE ||
1268       (!UseFastNewObjectArray && (op->type() == T_OBJECT || op->type() == T_ARRAY)) ||
1269       (!UseFastNewTypeArray   && (op->type() != T_OBJECT && op->type() != T_ARRAY))) {
1270     __ b(*op->stub()->entry());
1271   } else {
1272     Register tmp1 = op->tmp1()->as_register();
1273     Register tmp2 = op->tmp2()->as_register();
1274     Register tmp3 = op->tmp3()->as_register();
1275     if (len == tmp1) {
1276       tmp1 = tmp3;
1277     } else if (len == tmp2) {
1278       tmp2 = tmp3;
1279     } else if (len == tmp3) {
1280       // everything is ok
1281     } else {
1282       __ mov(tmp3, len);
1283     }
1284     __ allocate_array(op->obj()->as_register(),
1285                       len,
1286                       tmp1,
1287                       tmp2,
1288                       arrayOopDesc::header_size(op->type()),
1289                       array_element_size(op->type()),
1290                       op->klass()->as_register(),
1291                       *op->stub()->entry());
1292   }
1293   __ bind(*op->stub()->continuation());
1294 }
1295 
1296 void LIR_Assembler::type_profile_helper(Register mdo,
1297                                         ciMethodData *md, ciProfileData *data,
1298                                         Register recv, Label* update_done) {
1299   for (uint i = 0; i < ReceiverTypeData::row_limit(); i++) {
1300     Label next_test;
1301     // See if the receiver is receiver[n].
1302     __ lea(rscratch2, Address(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_offset(i))));
1303     __ ldr(rscratch1, Address(rscratch2));
1304     __ cmp(recv, rscratch1);
1305     __ br(Assembler::NE, next_test);
1306     Address data_addr(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_count_offset(i)));
1307     __ addptr(data_addr, DataLayout::counter_increment);
1308     __ b(*update_done);
1309     __ bind(next_test);
1310   }
1311 
1312   // Didn't find receiver; find next empty slot and fill it in
1313   for (uint i = 0; i < ReceiverTypeData::row_limit(); i++) {
1314     Label next_test;
1315     __ lea(rscratch2,
1316            Address(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_offset(i))));
1317     Address recv_addr(rscratch2);
1318     __ ldr(rscratch1, recv_addr);
1319     __ cbnz(rscratch1, next_test);
1320     __ str(recv, recv_addr);
1321     __ mov(rscratch1, DataLayout::counter_increment);
1322     __ lea(rscratch2, Address(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_count_offset(i))));
1323     __ str(rscratch1, Address(rscratch2));
1324     __ b(*update_done);
1325     __ bind(next_test);
1326   }
1327 }
1328 
1329 void LIR_Assembler::emit_typecheck_helper(LIR_OpTypeCheck *op, Label* success, Label* failure, Label* obj_is_null) {
1330   // we always need a stub for the failure case.
1331   CodeStub* stub = op->stub();
1332   Register obj = op->object()->as_register();
1333   Register k_RInfo = op->tmp1()->as_register();
1334   Register klass_RInfo = op->tmp2()->as_register();
1335   Register dst = op->result_opr()->as_register();
1336   ciKlass* k = op->klass();
1337   Register Rtmp1 = noreg;
1338 
1339   // check if it needs to be profiled
1340   ciMethodData* md;
1341   ciProfileData* data;
1342 
1343   const bool should_profile = op->should_profile();
1344 
1345   if (should_profile) {
1346     ciMethod* method = op->profiled_method();
1347     assert(method != NULL, "Should have method");
1348     int bci = op->profiled_bci();
1349     md = method->method_data_or_null();
1350     assert(md != NULL, "Sanity");
1351     data = md->bci_to_data(bci);
1352     assert(data != NULL,                "need data for type check");
1353     assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
1354   }
1355   Label profile_cast_success, profile_cast_failure;
1356   Label *success_target = should_profile ? &profile_cast_success : success;
1357   Label *failure_target = should_profile ? &profile_cast_failure : failure;
1358 
1359   if (obj == k_RInfo) {
1360     k_RInfo = dst;
1361   } else if (obj == klass_RInfo) {
1362     klass_RInfo = dst;
1363   }
1364   if (k->is_loaded() && !UseCompressedClassPointers) {
1365     select_different_registers(obj, dst, k_RInfo, klass_RInfo);
1366   } else {
1367     Rtmp1 = op->tmp3()->as_register();
1368     select_different_registers(obj, dst, k_RInfo, klass_RInfo, Rtmp1);
1369   }
1370 
1371   assert_different_registers(obj, k_RInfo, klass_RInfo);
1372 
1373     if (should_profile) {
1374       Label not_null;
1375       __ cbnz(obj, not_null);
1376       // Object is null; update MDO and exit
1377       Register mdo  = klass_RInfo;
1378       __ mov_metadata(mdo, md->constant_encoding());
1379       Address data_addr
1380         = __ form_address(rscratch2, mdo,
1381                           md->byte_offset_of_slot(data, DataLayout::flags_offset()),
1382                           0);
1383       __ ldrb(rscratch1, data_addr);
1384       __ orr(rscratch1, rscratch1, BitData::null_seen_byte_constant());
1385       __ strb(rscratch1, data_addr);
1386       __ b(*obj_is_null);
1387       __ bind(not_null);
1388     } else {
1389       __ cbz(obj, *obj_is_null);
1390     }
1391 
1392   if (!k->is_loaded()) {
1393     klass2reg_with_patching(k_RInfo, op->info_for_patch());
1394   } else {
1395     __ mov_metadata(k_RInfo, k->constant_encoding());
1396   }
1397   __ verify_oop(obj);
1398 
1399   if (op->fast_check()) {
1400     // get object class
1401     // not a safepoint as obj null check happens earlier
1402     __ load_klass(rscratch1, obj);
1403     __ cmp( rscratch1, k_RInfo);
1404 
1405     __ br(Assembler::NE, *failure_target);
1406     // successful cast, fall through to profile or jump
1407   } else {
1408     // get object class
1409     // not a safepoint as obj null check happens earlier
1410     __ load_klass(klass_RInfo, obj);
1411     if (k->is_loaded()) {
1412       // See if we get an immediate positive hit
1413       __ ldr(rscratch1, Address(klass_RInfo, long(k->super_check_offset())));
1414       __ cmp(k_RInfo, rscratch1);
1415       if ((juint)in_bytes(Klass::secondary_super_cache_offset()) != k->super_check_offset()) {
1416         __ br(Assembler::NE, *failure_target);
1417         // successful cast, fall through to profile or jump
1418       } else {
1419         // See if we get an immediate positive hit
1420         __ br(Assembler::EQ, *success_target);
1421         // check for self
1422         __ cmp(klass_RInfo, k_RInfo);
1423         __ br(Assembler::EQ, *success_target);
1424 
1425         __ stp(klass_RInfo, k_RInfo, Address(__ pre(sp, -2 * wordSize)));
1426         __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
1427         __ ldr(klass_RInfo, Address(__ post(sp, 2 * wordSize)));
1428         // result is a boolean
1429         __ cbzw(klass_RInfo, *failure_target);
1430         // successful cast, fall through to profile or jump
1431       }
1432     } else {
1433       // perform the fast part of the checking logic
1434       __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, NULL);
1435       // call out-of-line instance of __ check_klass_subtype_slow_path(...):
1436       __ stp(klass_RInfo, k_RInfo, Address(__ pre(sp, -2 * wordSize)));
1437       __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
1438       __ ldp(k_RInfo, klass_RInfo, Address(__ post(sp, 2 * wordSize)));
1439       // result is a boolean
1440       __ cbz(k_RInfo, *failure_target);
1441       // successful cast, fall through to profile or jump
1442     }
1443   }
1444   if (should_profile) {
1445     Register mdo  = klass_RInfo, recv = k_RInfo;
1446     __ bind(profile_cast_success);
1447     __ mov_metadata(mdo, md->constant_encoding());
1448     __ load_klass(recv, obj);
1449     Label update_done;
1450     type_profile_helper(mdo, md, data, recv, success);
1451     __ b(*success);
1452 
1453     __ bind(profile_cast_failure);
1454     __ mov_metadata(mdo, md->constant_encoding());
1455     Address counter_addr
1456       = __ form_address(rscratch2, mdo,
1457                         md->byte_offset_of_slot(data, CounterData::count_offset()),
1458                         0);
1459     __ ldr(rscratch1, counter_addr);
1460     __ sub(rscratch1, rscratch1, DataLayout::counter_increment);
1461     __ str(rscratch1, counter_addr);
1462     __ b(*failure);
1463   }
1464   __ b(*success);
1465 }
1466 
1467 
1468 void LIR_Assembler::emit_opTypeCheck(LIR_OpTypeCheck* op) {
1469   const bool should_profile = op->should_profile();
1470 
1471   LIR_Code code = op->code();
1472   if (code == lir_store_check) {
1473     Register value = op->object()->as_register();
1474     Register array = op->array()->as_register();
1475     Register k_RInfo = op->tmp1()->as_register();
1476     Register klass_RInfo = op->tmp2()->as_register();
1477     Register Rtmp1 = op->tmp3()->as_register();
1478 
1479     CodeStub* stub = op->stub();
1480 
1481     // check if it needs to be profiled
1482     ciMethodData* md;
1483     ciProfileData* data;
1484 
1485     if (should_profile) {
1486       ciMethod* method = op->profiled_method();
1487       assert(method != NULL, "Should have method");
1488       int bci = op->profiled_bci();
1489       md = method->method_data_or_null();
1490       assert(md != NULL, "Sanity");
1491       data = md->bci_to_data(bci);
1492       assert(data != NULL,                "need data for type check");
1493       assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
1494     }
1495     Label profile_cast_success, profile_cast_failure, done;
1496     Label *success_target = should_profile ? &profile_cast_success : &done;
1497     Label *failure_target = should_profile ? &profile_cast_failure : stub->entry();
1498 
1499     if (should_profile) {
1500       Label not_null;
1501       __ cbnz(value, not_null);
1502       // Object is null; update MDO and exit
1503       Register mdo  = klass_RInfo;
1504       __ mov_metadata(mdo, md->constant_encoding());
1505       Address data_addr
1506         = __ form_address(rscratch2, mdo,
1507                           md->byte_offset_of_slot(data, DataLayout::flags_offset()),
1508                           0);
1509       __ ldrb(rscratch1, data_addr);
1510       __ orr(rscratch1, rscratch1, BitData::null_seen_byte_constant());
1511       __ strb(rscratch1, data_addr);
1512       __ b(done);
1513       __ bind(not_null);
1514     } else {
1515       __ cbz(value, done);
1516     }
1517 
1518     add_debug_info_for_null_check_here(op->info_for_exception());
1519     __ load_klass(k_RInfo, array);
1520     __ load_klass(klass_RInfo, value);
1521 
1522     // get instance klass (it's already uncompressed)
1523     __ ldr(k_RInfo, Address(k_RInfo, ObjArrayKlass::element_klass_offset()));
1524     // perform the fast part of the checking logic
1525     __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, NULL);
1526     // call out-of-line instance of __ check_klass_subtype_slow_path(...):
1527     __ stp(klass_RInfo, k_RInfo, Address(__ pre(sp, -2 * wordSize)));
1528     __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
1529     __ ldp(k_RInfo, klass_RInfo, Address(__ post(sp, 2 * wordSize)));
1530     // result is a boolean
1531     __ cbzw(k_RInfo, *failure_target);
1532     // fall through to the success case
1533 
1534     if (should_profile) {
1535       Register mdo  = klass_RInfo, recv = k_RInfo;
1536       __ bind(profile_cast_success);
1537       __ mov_metadata(mdo, md->constant_encoding());
1538       __ load_klass(recv, value);
1539       Label update_done;
1540       type_profile_helper(mdo, md, data, recv, &done);
1541       __ b(done);
1542 
1543       __ bind(profile_cast_failure);
1544       __ mov_metadata(mdo, md->constant_encoding());
1545       Address counter_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
1546       __ lea(rscratch2, counter_addr);
1547       __ ldr(rscratch1, Address(rscratch2));
1548       __ sub(rscratch1, rscratch1, DataLayout::counter_increment);
1549       __ str(rscratch1, Address(rscratch2));
1550       __ b(*stub->entry());
1551     }
1552 
1553     __ bind(done);
1554   } else if (code == lir_checkcast) {
1555     Register obj = op->object()->as_register();
1556     Register dst = op->result_opr()->as_register();
1557     Label success;
1558     emit_typecheck_helper(op, &success, op->stub()->entry(), &success);
1559     __ bind(success);
1560     if (dst != obj) {
1561       __ mov(dst, obj);
1562     }
1563   } else if (code == lir_instanceof) {
1564     Register obj = op->object()->as_register();
1565     Register dst = op->result_opr()->as_register();
1566     Label success, failure, done;
1567     emit_typecheck_helper(op, &success, &failure, &failure);
1568     __ bind(failure);
1569     __ mov(dst, zr);
1570     __ b(done);
1571     __ bind(success);
1572     __ mov(dst, 1);
1573     __ bind(done);
1574   } else {
1575     ShouldNotReachHere();
1576   }
1577 }
1578 
1579 void LIR_Assembler::emit_opFlattenedArrayCheck(LIR_OpFlattenedArrayCheck* op) {
1580   // We are loading/storing an array that *may* be a flattened array (the declared type
1581   // Object[], interface[], or VT?[]). If this array is flattened, take slow path.
1582 
1583   __ load_storage_props(op->tmp()->as_register(), op->array()->as_register());
1584   __ tst(op->tmp()->as_register(), ArrayStorageProperties::flattened_value);
1585   __ br(Assembler::NE, *op->stub()->entry());
1586   if (!op->value()->is_illegal()) {
1587     // We are storing into the array.
1588     Label skip;
1589     __ tst(op->tmp()->as_register(), ArrayStorageProperties::null_free_value);
1590     __ br(Assembler::EQ, skip);
1591     // The array is not flattened, but it is null_free. If we are storing
1592     // a null, take the slow path (which will throw NPE).
1593     __ cbz(op->value()->as_register(), *op->stub()->entry());
1594     __ bind(skip);
1595   }
1596 
1597 }
1598 
1599 void LIR_Assembler::emit_opNullFreeArrayCheck(LIR_OpNullFreeArrayCheck* op) {
1600   // This is called when we use aastore into a an array declared as "[LVT;",
1601   // where we know VT is not flattenable (due to ValueArrayElemMaxFlatOops, etc).
1602   // However, we need to do a NULL check if the actual array is a "[QVT;".
1603 
1604   __ load_storage_props(op->tmp()->as_register(), op->array()->as_register());
1605   __ mov(rscratch1, (uint64_t) ArrayStorageProperties::null_free_value);
1606   __ cmp(op->tmp()->as_register(), rscratch1);
1607 }
1608 
1609 void LIR_Assembler::emit_opSubstitutabilityCheck(LIR_OpSubstitutabilityCheck* op) {
1610   Label L_oops_equal;
1611   Label L_oops_not_equal;
1612   Label L_end;
1613 
1614   Register left  = op->left()->as_register();
1615   Register right = op->right()->as_register();
1616 
1617   __ cmp(left, right);
1618   __ br(Assembler::EQ, L_oops_equal);
1619 
1620   // (1) Null check -- if one of the operands is null, the other must not be null (because
1621   //     the two references are not equal), so they are not substitutable,
1622   //     FIXME: do null check only if the operand is nullable
1623   {
1624     __ cbz(left, L_oops_not_equal);
1625     __ cbz(right, L_oops_not_equal);
1626   }
1627 
1628 
1629   ciKlass* left_klass = op->left_klass();
1630   ciKlass* right_klass = op->right_klass();
1631 
1632   // (2) Value object check -- if either of the operands is not a value object,
1633   //     they are not substitutable. We do this only if we are not sure that the
1634   //     operands are value objects
1635   if ((left_klass == NULL || right_klass == NULL) ||// The klass is still unloaded, or came from a Phi node.
1636       !left_klass->is_valuetype() || !right_klass->is_valuetype()) {
1637     Register tmp1  = rscratch1; /* op->tmp1()->as_register(); */
1638     Register tmp2  = rscratch2; /* op->tmp2()->as_register(); */
1639 
1640     __ mov(tmp1, (intptr_t)markOopDesc::always_locked_pattern);
1641 
1642     __ ldr(tmp2, Address(left, oopDesc::mark_offset_in_bytes()));
1643     __ andr(tmp1, tmp1, tmp2);
1644 
1645     __ ldr(tmp2, Address(right, oopDesc::mark_offset_in_bytes()));
1646     __ andr(tmp1, tmp1, tmp2); 
1647 
1648     __ mov(tmp2, (intptr_t)markOopDesc::always_locked_pattern);
1649     __ cmp(tmp1, tmp2); 
1650     __ br(Assembler::NE, L_oops_not_equal);
1651   }
1652 
1653   // (3) Same klass check: if the operands are of different klasses, they are not substitutable.
1654   if (left_klass != NULL && left_klass->is_valuetype() && left_klass == right_klass) {
1655     // No need to load klass -- the operands are statically known to be the same value klass.
1656     __ b(*op->stub()->entry());
1657   } else {
1658     Register left_klass_op = op->left_klass_op()->as_register();
1659     Register right_klass_op = op->right_klass_op()->as_register();
1660 
1661     if (UseCompressedOops) {
1662       __ ldrw(left_klass_op,  Address(left,  oopDesc::klass_offset_in_bytes()));
1663       __ ldrw(right_klass_op, Address(right, oopDesc::klass_offset_in_bytes()));
1664       __ cmpw(left_klass_op, right_klass_op);
1665     } else {
1666       __ ldr(left_klass_op,  Address(left,  oopDesc::klass_offset_in_bytes()));
1667       __ ldr(right_klass_op, Address(right, oopDesc::klass_offset_in_bytes()));
1668       __ cmp(left_klass_op, right_klass_op);
1669     }
1670 
1671     __ br(Assembler::EQ, *op->stub()->entry()); // same klass -> do slow check
1672     // fall through to L_oops_not_equal
1673   }
1674 
1675   __ bind(L_oops_not_equal);
1676   move(op->not_equal_result(), op->result_opr());
1677   __ b(L_end);
1678 
1679   __ bind(L_oops_equal);
1680   move(op->equal_result(), op->result_opr());
1681   __ b(L_end);
1682 
1683   // We've returned from the stub. op->result_opr() contains 0x0 IFF the two
1684   // operands are not substitutable. (Don't compare against 0x1 in case the
1685   // C compiler is naughty)
1686   __ bind(*op->stub()->continuation());
1687 
1688   if (op->result_opr()->type() == T_LONG) {
1689     __ cbzw(op->result_opr()->as_register(), L_oops_not_equal); // (call_stub() == 0x0) -> not_equal
1690   } else {
1691     __ cbz(op->result_opr()->as_register(), L_oops_not_equal); // (call_stub() == 0x0) -> not_equal
1692   }
1693 
1694   move(op->equal_result(), op->result_opr()); // (call_stub() != 0x0) -> equal
1695   // fall-through
1696   __ bind(L_end);
1697 
1698 }
1699 
1700 
1701 void LIR_Assembler::casw(Register addr, Register newval, Register cmpval) {
1702   __ cmpxchg(addr, cmpval, newval, Assembler::word, /* acquire*/ true, /* release*/ true, /* weak*/ false, rscratch1);
1703   __ cset(rscratch1, Assembler::NE);
1704   __ membar(__ AnyAny);
1705 }
1706 
1707 void LIR_Assembler::casl(Register addr, Register newval, Register cmpval) {
1708   __ cmpxchg(addr, cmpval, newval, Assembler::xword, /* acquire*/ true, /* release*/ true, /* weak*/ false, rscratch1);
1709   __ cset(rscratch1, Assembler::NE);
1710   __ membar(__ AnyAny);
1711 }
1712 
1713 
1714 void LIR_Assembler::emit_compare_and_swap(LIR_OpCompareAndSwap* op) {
1715   assert(VM_Version::supports_cx8(), "wrong machine");
1716   Register addr;
1717   if (op->addr()->is_register()) {
1718     addr = as_reg(op->addr());
1719   } else {
1720     assert(op->addr()->is_address(), "what else?");
1721     LIR_Address* addr_ptr = op->addr()->as_address_ptr();
1722     assert(addr_ptr->disp() == 0, "need 0 disp");
1723     assert(addr_ptr->index() == LIR_OprDesc::illegalOpr(), "need 0 index");
1724     addr = as_reg(addr_ptr->base());
1725   }
1726   Register newval = as_reg(op->new_value());
1727   Register cmpval = as_reg(op->cmp_value());
1728 
1729   if (op->code() == lir_cas_obj) {
1730     if (UseCompressedOops) {
1731       Register t1 = op->tmp1()->as_register();
1732       assert(op->tmp1()->is_valid(), "must be");
1733       __ encode_heap_oop(t1, cmpval);
1734       cmpval = t1;
1735       __ encode_heap_oop(rscratch2, newval);
1736       newval = rscratch2;
1737       casw(addr, newval, cmpval);
1738     } else {
1739       casl(addr, newval, cmpval);
1740     }
1741   } else if (op->code() == lir_cas_int) {
1742     casw(addr, newval, cmpval);
1743   } else {
1744     casl(addr, newval, cmpval);
1745   }
1746 }
1747 
1748 
1749 void LIR_Assembler::cmove(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Opr result, BasicType type) {
1750 
1751   Assembler::Condition acond, ncond;
1752   switch (condition) {
1753   case lir_cond_equal:        acond = Assembler::EQ; ncond = Assembler::NE; break;
1754   case lir_cond_notEqual:     acond = Assembler::NE; ncond = Assembler::EQ; break;
1755   case lir_cond_less:         acond = Assembler::LT; ncond = Assembler::GE; break;
1756   case lir_cond_lessEqual:    acond = Assembler::LE; ncond = Assembler::GT; break;
1757   case lir_cond_greaterEqual: acond = Assembler::GE; ncond = Assembler::LT; break;
1758   case lir_cond_greater:      acond = Assembler::GT; ncond = Assembler::LE; break;
1759   case lir_cond_belowEqual:
1760   case lir_cond_aboveEqual:
1761   default:                    ShouldNotReachHere();
1762     acond = Assembler::EQ; ncond = Assembler::NE;  // unreachable
1763   }
1764 
1765   assert(result->is_single_cpu() || result->is_double_cpu(),
1766          "expect single register for result");
1767   if (opr1->is_constant() && opr2->is_constant()
1768       && opr1->type() == T_INT && opr2->type() == T_INT) {
1769     jint val1 = opr1->as_jint();
1770     jint val2 = opr2->as_jint();
1771     if (val1 == 0 && val2 == 1) {
1772       __ cset(result->as_register(), ncond);
1773       return;
1774     } else if (val1 == 1 && val2 == 0) {
1775       __ cset(result->as_register(), acond);
1776       return;
1777     }
1778   }
1779 
1780   if (opr1->is_constant() && opr2->is_constant()
1781       && opr1->type() == T_LONG && opr2->type() == T_LONG) {
1782     jlong val1 = opr1->as_jlong();
1783     jlong val2 = opr2->as_jlong();
1784     if (val1 == 0 && val2 == 1) {
1785       __ cset(result->as_register_lo(), ncond);
1786       return;
1787     } else if (val1 == 1 && val2 == 0) {
1788       __ cset(result->as_register_lo(), acond);
1789       return;
1790     }
1791   }
1792 
1793   if (opr1->is_stack()) {
1794     stack2reg(opr1, FrameMap::rscratch1_opr, result->type());
1795     opr1 = FrameMap::rscratch1_opr;
1796   } else if (opr1->is_constant()) {
1797     LIR_Opr tmp
1798       = opr1->type() == T_LONG ? FrameMap::rscratch1_long_opr : FrameMap::rscratch1_opr;
1799     const2reg(opr1, tmp, lir_patch_none, NULL);
1800     opr1 = tmp;
1801   }
1802 
1803   if (opr2->is_stack()) {
1804     stack2reg(opr2, FrameMap::rscratch2_opr, result->type());
1805     opr2 = FrameMap::rscratch2_opr;
1806   } else if (opr2->is_constant()) {
1807     LIR_Opr tmp
1808       = opr2->type() == T_LONG ? FrameMap::rscratch2_long_opr : FrameMap::rscratch2_opr;
1809     const2reg(opr2, tmp, lir_patch_none, NULL);
1810     opr2 = tmp;
1811   }
1812 
1813   if (result->type() == T_LONG)
1814     __ csel(result->as_register_lo(), opr1->as_register_lo(), opr2->as_register_lo(), acond);
1815   else
1816     __ csel(result->as_register(), opr1->as_register(), opr2->as_register(), acond);
1817 }
1818 
1819 void LIR_Assembler::arith_op(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dest, CodeEmitInfo* info, bool pop_fpu_stack) {
1820   assert(info == NULL, "should never be used, idiv/irem and ldiv/lrem not handled by this method");
1821 
1822   if (left->is_single_cpu()) {
1823     Register lreg = left->as_register();
1824     Register dreg = as_reg(dest);
1825 
1826     if (right->is_single_cpu()) {
1827       // cpu register - cpu register
1828 
1829       assert(left->type() == T_INT && right->type() == T_INT && dest->type() == T_INT,
1830              "should be");
1831       Register rreg = right->as_register();
1832       switch (code) {
1833       case lir_add: __ addw (dest->as_register(), lreg, rreg); break;
1834       case lir_sub: __ subw (dest->as_register(), lreg, rreg); break;
1835       case lir_mul: __ mulw (dest->as_register(), lreg, rreg); break;
1836       default:      ShouldNotReachHere();
1837       }
1838 
1839     } else if (right->is_double_cpu()) {
1840       Register rreg = right->as_register_lo();
1841       // single_cpu + double_cpu: can happen with obj+long
1842       assert(code == lir_add || code == lir_sub, "mismatched arithmetic op");
1843       switch (code) {
1844       case lir_add: __ add(dreg, lreg, rreg); break;
1845       case lir_sub: __ sub(dreg, lreg, rreg); break;
1846       default: ShouldNotReachHere();
1847       }
1848     } else if (right->is_constant()) {
1849       // cpu register - constant
1850       jlong c;
1851 
1852       // FIXME.  This is fugly: we really need to factor all this logic.
1853       switch(right->type()) {
1854       case T_LONG:
1855         c = right->as_constant_ptr()->as_jlong();
1856         break;
1857       case T_INT:
1858       case T_ADDRESS:
1859         c = right->as_constant_ptr()->as_jint();
1860         break;
1861       default:
1862         ShouldNotReachHere();
1863         c = 0;  // unreachable
1864         break;
1865       }
1866 
1867       assert(code == lir_add || code == lir_sub, "mismatched arithmetic op");
1868       if (c == 0 && dreg == lreg) {
1869         COMMENT("effective nop elided");
1870         return;
1871       }
1872       switch(left->type()) {
1873       case T_INT:
1874         switch (code) {
1875         case lir_add: __ addw(dreg, lreg, c); break;
1876         case lir_sub: __ subw(dreg, lreg, c); break;
1877         default: ShouldNotReachHere();
1878         }
1879         break;
1880       case T_OBJECT:
1881       case T_ADDRESS:
1882         switch (code) {
1883         case lir_add: __ add(dreg, lreg, c); break;
1884         case lir_sub: __ sub(dreg, lreg, c); break;
1885         default: ShouldNotReachHere();
1886         }
1887         break;
1888       default:
1889         ShouldNotReachHere();
1890       }
1891     } else {
1892       ShouldNotReachHere();
1893     }
1894 
1895   } else if (left->is_double_cpu()) {
1896     Register lreg_lo = left->as_register_lo();
1897 
1898     if (right->is_double_cpu()) {
1899       // cpu register - cpu register
1900       Register rreg_lo = right->as_register_lo();
1901       switch (code) {
1902       case lir_add: __ add (dest->as_register_lo(), lreg_lo, rreg_lo); break;
1903       case lir_sub: __ sub (dest->as_register_lo(), lreg_lo, rreg_lo); break;
1904       case lir_mul: __ mul (dest->as_register_lo(), lreg_lo, rreg_lo); break;
1905       case lir_div: __ corrected_idivq(dest->as_register_lo(), lreg_lo, rreg_lo, false, rscratch1); break;
1906       case lir_rem: __ corrected_idivq(dest->as_register_lo(), lreg_lo, rreg_lo, true, rscratch1); break;
1907       default:
1908         ShouldNotReachHere();
1909       }
1910 
1911     } else if (right->is_constant()) {
1912       jlong c = right->as_constant_ptr()->as_jlong();
1913       Register dreg = as_reg(dest);
1914       switch (code) {
1915         case lir_add:
1916         case lir_sub:
1917           if (c == 0 && dreg == lreg_lo) {
1918             COMMENT("effective nop elided");
1919             return;
1920           }
1921           code == lir_add ? __ add(dreg, lreg_lo, c) : __ sub(dreg, lreg_lo, c);
1922           break;
1923         case lir_div:
1924           assert(c > 0 && is_power_of_2_long(c), "divisor must be power-of-2 constant");
1925           if (c == 1) {
1926             // move lreg_lo to dreg if divisor is 1
1927             __ mov(dreg, lreg_lo);
1928           } else {
1929             unsigned int shift = exact_log2_long(c);
1930             // use rscratch1 as intermediate result register
1931             __ asr(rscratch1, lreg_lo, 63);
1932             __ add(rscratch1, lreg_lo, rscratch1, Assembler::LSR, 64 - shift);
1933             __ asr(dreg, rscratch1, shift);
1934           }
1935           break;
1936         case lir_rem:
1937           assert(c > 0 && is_power_of_2_long(c), "divisor must be power-of-2 constant");
1938           if (c == 1) {
1939             // move 0 to dreg if divisor is 1
1940             __ mov(dreg, zr);
1941           } else {
1942             // use rscratch1 as intermediate result register
1943             __ negs(rscratch1, lreg_lo);
1944             __ andr(dreg, lreg_lo, c - 1);
1945             __ andr(rscratch1, rscratch1, c - 1);
1946             __ csneg(dreg, dreg, rscratch1, Assembler::MI);
1947           }
1948           break;
1949         default:
1950           ShouldNotReachHere();
1951       }
1952     } else {
1953       ShouldNotReachHere();
1954     }
1955   } else if (left->is_single_fpu()) {
1956     assert(right->is_single_fpu(), "right hand side of float arithmetics needs to be float register");
1957     switch (code) {
1958     case lir_add: __ fadds (dest->as_float_reg(), left->as_float_reg(), right->as_float_reg()); break;
1959     case lir_sub: __ fsubs (dest->as_float_reg(), left->as_float_reg(), right->as_float_reg()); break;
1960     case lir_mul_strictfp: // fall through
1961     case lir_mul: __ fmuls (dest->as_float_reg(), left->as_float_reg(), right->as_float_reg()); break;
1962     case lir_div_strictfp: // fall through
1963     case lir_div: __ fdivs (dest->as_float_reg(), left->as_float_reg(), right->as_float_reg()); break;
1964     default:
1965       ShouldNotReachHere();
1966     }
1967   } else if (left->is_double_fpu()) {
1968     if (right->is_double_fpu()) {
1969       // fpu register - fpu register
1970       switch (code) {
1971       case lir_add: __ faddd (dest->as_double_reg(), left->as_double_reg(), right->as_double_reg()); break;
1972       case lir_sub: __ fsubd (dest->as_double_reg(), left->as_double_reg(), right->as_double_reg()); break;
1973       case lir_mul_strictfp: // fall through
1974       case lir_mul: __ fmuld (dest->as_double_reg(), left->as_double_reg(), right->as_double_reg()); break;
1975       case lir_div_strictfp: // fall through
1976       case lir_div: __ fdivd (dest->as_double_reg(), left->as_double_reg(), right->as_double_reg()); break;
1977       default:
1978         ShouldNotReachHere();
1979       }
1980     } else {
1981       if (right->is_constant()) {
1982         ShouldNotReachHere();
1983       }
1984       ShouldNotReachHere();
1985     }
1986   } else if (left->is_single_stack() || left->is_address()) {
1987     assert(left == dest, "left and dest must be equal");
1988     ShouldNotReachHere();
1989   } else {
1990     ShouldNotReachHere();
1991   }
1992 }
1993 
1994 void LIR_Assembler::arith_fpu_implementation(LIR_Code code, int left_index, int right_index, int dest_index, bool pop_fpu_stack) { Unimplemented(); }
1995 
1996 
1997 void LIR_Assembler::intrinsic_op(LIR_Code code, LIR_Opr value, LIR_Opr unused, LIR_Opr dest, LIR_Op* op) {
1998   switch(code) {
1999   case lir_abs : __ fabsd(dest->as_double_reg(), value->as_double_reg()); break;
2000   case lir_sqrt: __ fsqrtd(dest->as_double_reg(), value->as_double_reg()); break;
2001   default      : ShouldNotReachHere();
2002   }
2003 }
2004 
2005 void LIR_Assembler::logic_op(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dst) {
2006 
2007   assert(left->is_single_cpu() || left->is_double_cpu(), "expect single or double register");
2008   Register Rleft = left->is_single_cpu() ? left->as_register() :
2009                                            left->as_register_lo();
2010    if (dst->is_single_cpu()) {
2011      Register Rdst = dst->as_register();
2012      if (right->is_constant()) {
2013        switch (code) {
2014          case lir_logic_and: __ andw (Rdst, Rleft, right->as_jint()); break;
2015          case lir_logic_or:  __ orrw (Rdst, Rleft, right->as_jint()); break;
2016          case lir_logic_xor: __ eorw (Rdst, Rleft, right->as_jint()); break;
2017          default: ShouldNotReachHere(); break;
2018        }
2019      } else {
2020        Register Rright = right->is_single_cpu() ? right->as_register() :
2021                                                   right->as_register_lo();
2022        switch (code) {
2023          case lir_logic_and: __ andw (Rdst, Rleft, Rright); break;
2024          case lir_logic_or:  __ orrw (Rdst, Rleft, Rright); break;
2025          case lir_logic_xor: __ eorw (Rdst, Rleft, Rright); break;
2026          default: ShouldNotReachHere(); break;
2027        }
2028      }
2029    } else {
2030      Register Rdst = dst->as_register_lo();
2031      if (right->is_constant()) {
2032        switch (code) {
2033          case lir_logic_and: __ andr (Rdst, Rleft, right->as_jlong()); break;
2034          case lir_logic_or:  __ orr (Rdst, Rleft, right->as_jlong()); break;
2035          case lir_logic_xor: __ eor (Rdst, Rleft, right->as_jlong()); break;
2036          default: ShouldNotReachHere(); break;
2037        }
2038      } else {
2039        Register Rright = right->is_single_cpu() ? right->as_register() :
2040                                                   right->as_register_lo();
2041        switch (code) {
2042          case lir_logic_and: __ andr (Rdst, Rleft, Rright); break;
2043          case lir_logic_or:  __ orr (Rdst, Rleft, Rright); break;
2044          case lir_logic_xor: __ eor (Rdst, Rleft, Rright); break;
2045          default: ShouldNotReachHere(); break;
2046        }
2047      }
2048    }
2049 }
2050 
2051 
2052 
2053 void LIR_Assembler::arithmetic_idiv(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr illegal, LIR_Opr result, CodeEmitInfo* info) {
2054 
2055   // opcode check
2056   assert((code == lir_idiv) || (code == lir_irem), "opcode must be idiv or irem");
2057   bool is_irem = (code == lir_irem);
2058 
2059   // operand check
2060   assert(left->is_single_cpu(),   "left must be register");
2061   assert(right->is_single_cpu() || right->is_constant(),  "right must be register or constant");
2062   assert(result->is_single_cpu(), "result must be register");
2063   Register lreg = left->as_register();
2064   Register dreg = result->as_register();
2065 
2066   // power-of-2 constant check and codegen
2067   if (right->is_constant()) {
2068     int c = right->as_constant_ptr()->as_jint();
2069     assert(c > 0 && is_power_of_2(c), "divisor must be power-of-2 constant");
2070     if (is_irem) {
2071       if (c == 1) {
2072         // move 0 to dreg if divisor is 1
2073         __ movw(dreg, zr);
2074       } else {
2075         // use rscratch1 as intermediate result register
2076         __ negsw(rscratch1, lreg);
2077         __ andw(dreg, lreg, c - 1);
2078         __ andw(rscratch1, rscratch1, c - 1);
2079         __ csnegw(dreg, dreg, rscratch1, Assembler::MI);
2080       }
2081     } else {
2082       if (c == 1) {
2083         // move lreg to dreg if divisor is 1
2084         __ movw(dreg, lreg);
2085       } else {
2086         unsigned int shift = exact_log2(c);
2087         // use rscratch1 as intermediate result register
2088         __ asrw(rscratch1, lreg, 31);
2089         __ addw(rscratch1, lreg, rscratch1, Assembler::LSR, 32 - shift);
2090         __ asrw(dreg, rscratch1, shift);
2091       }
2092     }
2093   } else {
2094     Register rreg = right->as_register();
2095     __ corrected_idivl(dreg, lreg, rreg, is_irem, rscratch1);
2096   }
2097 }
2098 
2099 
2100 void LIR_Assembler::comp_op(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Op2* op) {
2101   if (opr1->is_constant() && opr2->is_single_cpu()) {
2102     // tableswitch
2103     Register reg = as_reg(opr2);
2104     struct tableswitch &table = switches[opr1->as_constant_ptr()->as_jint()];
2105     __ tableswitch(reg, table._first_key, table._last_key, table._branches, table._after);
2106   } else if (opr1->is_single_cpu() || opr1->is_double_cpu()) {
2107     Register reg1 = as_reg(opr1);
2108     if (opr2->is_single_cpu()) {
2109       // cpu register - cpu register
2110       Register reg2 = opr2->as_register();
2111       if (opr1->type() == T_OBJECT || opr1->type() == T_ARRAY || opr1->type() == T_VALUETYPE) {
2112         __ cmpoop(reg1, reg2);
2113       } else {
2114         assert(opr2->type() != T_OBJECT && opr2->type() != T_ARRAY && opr2->type() != T_VALUETYPE,  "cmp int, oop?");
2115         __ cmpw(reg1, reg2);
2116       }
2117       return;
2118     }
2119     if (opr2->is_double_cpu()) {
2120       // cpu register - cpu register
2121       Register reg2 = opr2->as_register_lo();
2122       __ cmp(reg1, reg2);
2123       return;
2124     }
2125 
2126     if (opr2->is_constant()) {
2127       bool is_32bit = false; // width of register operand
2128       jlong imm;
2129 
2130       switch(opr2->type()) {
2131       case T_INT:
2132         imm = opr2->as_constant_ptr()->as_jint();
2133         is_32bit = true;
2134         break;
2135       case T_LONG:
2136         imm = opr2->as_constant_ptr()->as_jlong();
2137         break;
2138       case T_ADDRESS:
2139         imm = opr2->as_constant_ptr()->as_jint();
2140         break;
2141       case T_VALUETYPE:
2142       case T_OBJECT:
2143       case T_ARRAY:
2144         jobject2reg(opr2->as_constant_ptr()->as_jobject(), rscratch1);
2145         __ cmpoop(reg1, rscratch1);
2146         return;
2147       default:
2148         ShouldNotReachHere();
2149         imm = 0;  // unreachable
2150         break;
2151       }
2152 
2153       if (Assembler::operand_valid_for_add_sub_immediate(imm)) {
2154         if (is_32bit)
2155           __ cmpw(reg1, imm);
2156         else
2157           __ subs(zr, reg1, imm);
2158         return;
2159       } else {
2160         __ mov(rscratch1, imm);
2161         if (is_32bit)
2162           __ cmpw(reg1, rscratch1);
2163         else
2164           __ cmp(reg1, rscratch1);
2165         return;
2166       }
2167     } else
2168       ShouldNotReachHere();
2169   } else if (opr1->is_single_fpu()) {
2170     FloatRegister reg1 = opr1->as_float_reg();
2171     assert(opr2->is_single_fpu(), "expect single float register");
2172     FloatRegister reg2 = opr2->as_float_reg();
2173     __ fcmps(reg1, reg2);
2174   } else if (opr1->is_double_fpu()) {
2175     FloatRegister reg1 = opr1->as_double_reg();
2176     assert(opr2->is_double_fpu(), "expect double float register");
2177     FloatRegister reg2 = opr2->as_double_reg();
2178     __ fcmpd(reg1, reg2);
2179   } else {
2180     ShouldNotReachHere();
2181   }
2182 }
2183 
2184 void LIR_Assembler::comp_fl2i(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dst, LIR_Op2* op){
2185   if (code == lir_cmp_fd2i || code == lir_ucmp_fd2i) {
2186     bool is_unordered_less = (code == lir_ucmp_fd2i);
2187     if (left->is_single_fpu()) {
2188       __ float_cmp(true, is_unordered_less ? -1 : 1, left->as_float_reg(), right->as_float_reg(), dst->as_register());
2189     } else if (left->is_double_fpu()) {
2190       __ float_cmp(false, is_unordered_less ? -1 : 1, left->as_double_reg(), right->as_double_reg(), dst->as_register());
2191     } else {
2192       ShouldNotReachHere();
2193     }
2194   } else if (code == lir_cmp_l2i) {
2195     Label done;
2196     __ cmp(left->as_register_lo(), right->as_register_lo());
2197     __ mov(dst->as_register(), (u_int64_t)-1L);
2198     __ br(Assembler::LT, done);
2199     __ csinc(dst->as_register(), zr, zr, Assembler::EQ);
2200     __ bind(done);
2201   } else {
2202     ShouldNotReachHere();
2203   }
2204 }
2205 
2206 
2207 void LIR_Assembler::align_call(LIR_Code code) {  }
2208 
2209 
2210 void LIR_Assembler::call(LIR_OpJavaCall* op, relocInfo::relocType rtype) {
2211   address call = __ trampoline_call(Address(op->addr(), rtype));
2212   if (call == NULL) {
2213     bailout("trampoline stub overflow");
2214     return;
2215   }
2216   add_call_info(code_offset(), op->info());
2217 }
2218 
2219 
2220 void LIR_Assembler::ic_call(LIR_OpJavaCall* op) {
2221   address call = __ ic_call(op->addr());
2222   if (call == NULL) {
2223     bailout("trampoline stub overflow");
2224     return;
2225   }
2226   add_call_info(code_offset(), op->info());
2227 }
2228 
2229 
2230 /* Currently, vtable-dispatch is only enabled for sparc platforms */
2231 void LIR_Assembler::vtable_call(LIR_OpJavaCall* op) {
2232   ShouldNotReachHere();
2233 }
2234 
2235 
2236 void LIR_Assembler::emit_static_call_stub() {
2237   address call_pc = __ pc();
2238   address stub = __ start_a_stub(call_stub_size());
2239   if (stub == NULL) {
2240     bailout("static call stub overflow");
2241     return;
2242   }
2243 
2244   int start = __ offset();
2245 
2246   __ relocate(static_stub_Relocation::spec(call_pc));
2247   __ emit_static_call_stub();
2248 
2249   assert(__ offset() - start + CompiledStaticCall::to_trampoline_stub_size()
2250         <= call_stub_size(), "stub too big");
2251   __ end_a_stub();
2252 }
2253 
2254 
2255 void LIR_Assembler::throw_op(LIR_Opr exceptionPC, LIR_Opr exceptionOop, CodeEmitInfo* info) {
2256   assert(exceptionOop->as_register() == r0, "must match");
2257   assert(exceptionPC->as_register() == r3, "must match");
2258 
2259   // exception object is not added to oop map by LinearScan
2260   // (LinearScan assumes that no oops are in fixed registers)
2261   info->add_register_oop(exceptionOop);
2262   Runtime1::StubID unwind_id;
2263 
2264   // get current pc information
2265   // pc is only needed if the method has an exception handler, the unwind code does not need it.
2266   int pc_for_athrow_offset = __ offset();
2267   InternalAddress pc_for_athrow(__ pc());
2268   __ adr(exceptionPC->as_register(), pc_for_athrow);
2269   add_call_info(pc_for_athrow_offset, info); // for exception handler
2270 
2271   __ verify_not_null_oop(r0);
2272   // search an exception handler (r0: exception oop, r3: throwing pc)
2273   if (compilation()->has_fpu_code()) {
2274     unwind_id = Runtime1::handle_exception_id;
2275   } else {
2276     unwind_id = Runtime1::handle_exception_nofpu_id;
2277   }
2278   __ far_call(RuntimeAddress(Runtime1::entry_for(unwind_id)));
2279 
2280   // FIXME: enough room for two byte trap   ????
2281   __ nop();
2282 }
2283 
2284 
2285 void LIR_Assembler::unwind_op(LIR_Opr exceptionOop) {
2286   assert(exceptionOop->as_register() == r0, "must match");
2287 
2288   __ b(_unwind_handler_entry);
2289 }
2290 
2291 
2292 void LIR_Assembler::shift_op(LIR_Code code, LIR_Opr left, LIR_Opr count, LIR_Opr dest, LIR_Opr tmp) {
2293   Register lreg = left->is_single_cpu() ? left->as_register() : left->as_register_lo();
2294   Register dreg = dest->is_single_cpu() ? dest->as_register() : dest->as_register_lo();
2295 
2296   switch (left->type()) {
2297     case T_INT: {
2298       switch (code) {
2299       case lir_shl:  __ lslvw (dreg, lreg, count->as_register()); break;
2300       case lir_shr:  __ asrvw (dreg, lreg, count->as_register()); break;
2301       case lir_ushr: __ lsrvw (dreg, lreg, count->as_register()); break;
2302       default:
2303         ShouldNotReachHere();
2304         break;
2305       }
2306       break;
2307     case T_LONG:
2308     case T_VALUETYPE: 
2309     case T_ADDRESS:
2310     case T_OBJECT:
2311       switch (code) {
2312       case lir_shl:  __ lslv (dreg, lreg, count->as_register()); break;
2313       case lir_shr:  __ asrv (dreg, lreg, count->as_register()); break;
2314       case lir_ushr: __ lsrv (dreg, lreg, count->as_register()); break;
2315       default:
2316         ShouldNotReachHere();
2317         break;
2318       }
2319       break;
2320     default:
2321       ShouldNotReachHere();
2322       break;
2323     }
2324   }
2325 }
2326 
2327 
2328 void LIR_Assembler::shift_op(LIR_Code code, LIR_Opr left, jint count, LIR_Opr dest) {
2329   Register dreg = dest->is_single_cpu() ? dest->as_register() : dest->as_register_lo();
2330   Register lreg = left->is_single_cpu() ? left->as_register() : left->as_register_lo();
2331 
2332   switch (left->type()) {
2333     case T_INT: {
2334       switch (code) {
2335       case lir_shl:  __ lslw (dreg, lreg, count); break;
2336       case lir_shr:  __ asrw (dreg, lreg, count); break;
2337       case lir_ushr: __ lsrw (dreg, lreg, count); break;
2338       default:
2339         ShouldNotReachHere();
2340         break;
2341       }
2342       break;
2343     case T_LONG:
2344     case T_ADDRESS:
2345     case T_VALUETYPE:
2346     case T_OBJECT:
2347       switch (code) {
2348       case lir_shl:  __ lsl (dreg, lreg, count); break;
2349       case lir_shr:  __ asr (dreg, lreg, count); break;
2350       case lir_ushr: __ lsr (dreg, lreg, count); break;
2351       default:
2352         ShouldNotReachHere();
2353         break;
2354       }
2355       break;
2356     default:
2357       ShouldNotReachHere();
2358       break;
2359     }
2360   }
2361 }
2362 
2363 
2364 void LIR_Assembler::store_parameter(Register r, int offset_from_rsp_in_words) {
2365   assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2366   int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2367   assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2368   __ str (r, Address(sp, offset_from_rsp_in_bytes));
2369 }
2370 
2371 
2372 void LIR_Assembler::store_parameter(jint c,     int offset_from_rsp_in_words) {
2373   assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2374   int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2375   assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2376   __ mov (rscratch1, c);
2377   __ str (rscratch1, Address(sp, offset_from_rsp_in_bytes));
2378 }
2379 
2380 
2381 void LIR_Assembler::store_parameter(jobject o,  int offset_from_rsp_in_words) {
2382   ShouldNotReachHere();
2383   assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2384   int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2385   assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2386   __ lea(rscratch1, __ constant_oop_address(o));
2387   __ str(rscratch1, Address(sp, offset_from_rsp_in_bytes));
2388 }
2389 
2390 void LIR_Assembler::arraycopy_valuetype_check(Register obj, Register tmp, CodeStub* slow_path, bool is_dest) {
2391   __ load_storage_props(tmp, obj);
2392   if (is_dest) {
2393     // We also take slow path if it's a null_free destination array, just in case the source array
2394     // contains NULLs.
2395     __ tst(tmp, ArrayStorageProperties::flattened_value | ArrayStorageProperties::null_free_value);
2396   } else {
2397     __ tst(tmp, ArrayStorageProperties::flattened_value);
2398   }
2399   __ br(Assembler::NE, *slow_path->entry());
2400 }
2401 
2402 
2403 
2404 // This code replaces a call to arraycopy; no exception may
2405 // be thrown in this code, they must be thrown in the System.arraycopy
2406 // activation frame; we could save some checks if this would not be the case
2407 void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) {
2408   ciArrayKlass* default_type = op->expected_type();
2409   Register src = op->src()->as_register();
2410   Register dst = op->dst()->as_register();
2411   Register src_pos = op->src_pos()->as_register();
2412   Register dst_pos = op->dst_pos()->as_register();
2413   Register length  = op->length()->as_register();
2414   Register tmp = op->tmp()->as_register();
2415 
2416   __ resolve(ACCESS_READ, src);
2417   __ resolve(ACCESS_WRITE, dst);
2418 
2419   CodeStub* stub = op->stub();
2420   int flags = op->flags();
2421   BasicType basic_type = default_type != NULL ? default_type->element_type()->basic_type() : T_ILLEGAL;
2422   if (basic_type == T_ARRAY || basic_type == T_VALUETYPE) basic_type = T_OBJECT;
2423 
2424   if (flags & LIR_OpArrayCopy::always_slow_path) {
2425     __ b(*stub->entry());
2426     __ bind(*stub->continuation());
2427     return;
2428   }
2429 
2430   if (flags & LIR_OpArrayCopy::src_valuetype_check) {
2431     arraycopy_valuetype_check(src, tmp, stub, false);
2432   }
2433 
2434   if (flags & LIR_OpArrayCopy::dst_valuetype_check) {
2435     arraycopy_valuetype_check(dst, tmp, stub, true);
2436   }
2437 
2438 
2439 
2440   // if we don't know anything, just go through the generic arraycopy
2441   if (default_type == NULL // || basic_type == T_OBJECT
2442       ) {
2443     Label done;
2444     assert(src == r1 && src_pos == r2, "mismatch in calling convention");
2445 
2446     // Save the arguments in case the generic arraycopy fails and we
2447     // have to fall back to the JNI stub
2448     __ stp(dst,     dst_pos, Address(sp, 0*BytesPerWord));
2449     __ stp(length,  src_pos, Address(sp, 2*BytesPerWord));
2450     __ str(src,              Address(sp, 4*BytesPerWord));
2451 
2452     address copyfunc_addr = StubRoutines::generic_arraycopy();
2453     assert(copyfunc_addr != NULL, "generic arraycopy stub required");
2454 
2455     // The arguments are in java calling convention so we shift them
2456     // to C convention
2457     assert_different_registers(c_rarg0, j_rarg1, j_rarg2, j_rarg3, j_rarg4);
2458     __ mov(c_rarg0, j_rarg0);
2459     assert_different_registers(c_rarg1, j_rarg2, j_rarg3, j_rarg4);
2460     __ mov(c_rarg1, j_rarg1);
2461     assert_different_registers(c_rarg2, j_rarg3, j_rarg4);
2462     __ mov(c_rarg2, j_rarg2);
2463     assert_different_registers(c_rarg3, j_rarg4);
2464     __ mov(c_rarg3, j_rarg3);
2465     __ mov(c_rarg4, j_rarg4);
2466 #ifndef PRODUCT
2467     if (PrintC1Statistics) {
2468       __ incrementw(ExternalAddress((address)&Runtime1::_generic_arraycopystub_cnt));
2469     }
2470 #endif
2471     __ far_call(RuntimeAddress(copyfunc_addr));
2472 
2473     __ cbz(r0, *stub->continuation());
2474 
2475     // Reload values from the stack so they are where the stub
2476     // expects them.
2477     __ ldp(dst,     dst_pos, Address(sp, 0*BytesPerWord));
2478     __ ldp(length,  src_pos, Address(sp, 2*BytesPerWord));
2479     __ ldr(src,              Address(sp, 4*BytesPerWord));
2480 
2481     // r0 is -1^K where K == partial copied count
2482     __ eonw(rscratch1, r0, zr);
2483     // adjust length down and src/end pos up by partial copied count
2484     __ subw(length, length, rscratch1);
2485     __ addw(src_pos, src_pos, rscratch1);
2486     __ addw(dst_pos, dst_pos, rscratch1);
2487     __ b(*stub->entry());
2488 
2489     __ bind(*stub->continuation());
2490     return;
2491   }
2492 
2493   assert(default_type != NULL && default_type->is_array_klass() && default_type->is_loaded(), "must be true at this point");
2494 
2495   int elem_size = type2aelembytes(basic_type);
2496   int shift_amount;
2497   int scale = exact_log2(elem_size);
2498 
2499   Address src_length_addr = Address(src, arrayOopDesc::length_offset_in_bytes());
2500   Address dst_length_addr = Address(dst, arrayOopDesc::length_offset_in_bytes());
2501   Address src_klass_addr = Address(src, oopDesc::klass_offset_in_bytes());
2502   Address dst_klass_addr = Address(dst, oopDesc::klass_offset_in_bytes());
2503 
2504   // test for NULL
2505   if (flags & LIR_OpArrayCopy::src_null_check) {
2506     __ cbz(src, *stub->entry());
2507   }
2508   if (flags & LIR_OpArrayCopy::dst_null_check) {
2509     __ cbz(dst, *stub->entry());
2510   }
2511 
2512   // If the compiler was not able to prove that exact type of the source or the destination
2513   // of the arraycopy is an array type, check at runtime if the source or the destination is
2514   // an instance type.
2515   if (flags & LIR_OpArrayCopy::type_check) {
2516     if (!(flags & LIR_OpArrayCopy::LIR_OpArrayCopy::dst_objarray)) {
2517       __ load_klass(tmp, dst);
2518       __ ldrw(rscratch1, Address(tmp, in_bytes(Klass::layout_helper_offset())));
2519       __ cmpw(rscratch1, Klass::_lh_neutral_value);
2520       __ br(Assembler::GE, *stub->entry());
2521     }
2522 
2523     if (!(flags & LIR_OpArrayCopy::LIR_OpArrayCopy::src_objarray)) {
2524       __ load_klass(tmp, src);
2525       __ ldrw(rscratch1, Address(tmp, in_bytes(Klass::layout_helper_offset())));
2526       __ cmpw(rscratch1, Klass::_lh_neutral_value);
2527       __ br(Assembler::GE, *stub->entry());
2528     }
2529   }
2530 
2531   // check if negative
2532   if (flags & LIR_OpArrayCopy::src_pos_positive_check) {
2533     __ cmpw(src_pos, 0);
2534     __ br(Assembler::LT, *stub->entry());
2535   }
2536   if (flags & LIR_OpArrayCopy::dst_pos_positive_check) {
2537     __ cmpw(dst_pos, 0);
2538     __ br(Assembler::LT, *stub->entry());
2539   }
2540 
2541   if (flags & LIR_OpArrayCopy::length_positive_check) {
2542     __ cmpw(length, 0);
2543     __ br(Assembler::LT, *stub->entry());
2544   }
2545 
2546   if (flags & LIR_OpArrayCopy::src_range_check) {
2547     __ addw(tmp, src_pos, length);
2548     __ ldrw(rscratch1, src_length_addr);
2549     __ cmpw(tmp, rscratch1);
2550     __ br(Assembler::HI, *stub->entry());
2551   }
2552   if (flags & LIR_OpArrayCopy::dst_range_check) {
2553     __ addw(tmp, dst_pos, length);
2554     __ ldrw(rscratch1, dst_length_addr);
2555     __ cmpw(tmp, rscratch1);
2556     __ br(Assembler::HI, *stub->entry());
2557   }
2558 
2559   if (flags & LIR_OpArrayCopy::type_check) {
2560     // We don't know the array types are compatible
2561     if (basic_type != T_OBJECT) {
2562       // Simple test for basic type arrays
2563       if (UseCompressedClassPointers) {
2564         __ ldrw(tmp, src_klass_addr);
2565         __ ldrw(rscratch1, dst_klass_addr);
2566         __ cmpw(tmp, rscratch1);
2567       } else {
2568         __ ldr(tmp, src_klass_addr);
2569         __ ldr(rscratch1, dst_klass_addr);
2570         __ cmp(tmp, rscratch1);
2571       }
2572       __ br(Assembler::NE, *stub->entry());
2573     } else {
2574       // For object arrays, if src is a sub class of dst then we can
2575       // safely do the copy.
2576       Label cont, slow;
2577 
2578 #define PUSH(r1, r2)                                    \
2579       stp(r1, r2, __ pre(sp, -2 * wordSize));
2580 
2581 #define POP(r1, r2)                                     \
2582       ldp(r1, r2, __ post(sp, 2 * wordSize));
2583 
2584       __ PUSH(src, dst);
2585 
2586       __ load_klass(src, src);
2587       __ load_klass(dst, dst);
2588 
2589       __ check_klass_subtype_fast_path(src, dst, tmp, &cont, &slow, NULL);
2590 
2591       __ PUSH(src, dst);
2592       __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
2593       __ POP(src, dst);
2594 
2595       __ cbnz(src, cont);
2596 
2597       __ bind(slow);
2598       __ POP(src, dst);
2599 
2600       address copyfunc_addr = StubRoutines::checkcast_arraycopy();
2601       if (copyfunc_addr != NULL) { // use stub if available
2602         // src is not a sub class of dst so we have to do a
2603         // per-element check.
2604 
2605         int mask = LIR_OpArrayCopy::src_objarray|LIR_OpArrayCopy::dst_objarray;
2606         if ((flags & mask) != mask) {
2607           // Check that at least both of them object arrays.
2608           assert(flags & mask, "one of the two should be known to be an object array");
2609 
2610           if (!(flags & LIR_OpArrayCopy::src_objarray)) {
2611             __ load_klass(tmp, src);
2612           } else if (!(flags & LIR_OpArrayCopy::dst_objarray)) {
2613             __ load_klass(tmp, dst);
2614           }
2615           int lh_offset = in_bytes(Klass::layout_helper_offset());
2616           Address klass_lh_addr(tmp, lh_offset);
2617           jint objArray_lh = Klass::array_layout_helper(T_OBJECT);
2618           __ ldrw(rscratch1, klass_lh_addr);
2619           __ mov(rscratch2, objArray_lh);
2620           __ eorw(rscratch1, rscratch1, rscratch2);
2621           __ cbnzw(rscratch1, *stub->entry());
2622         }
2623 
2624        // Spill because stubs can use any register they like and it's
2625        // easier to restore just those that we care about.
2626         __ stp(dst,     dst_pos, Address(sp, 0*BytesPerWord));
2627         __ stp(length,  src_pos, Address(sp, 2*BytesPerWord));
2628         __ str(src,              Address(sp, 4*BytesPerWord));
2629 
2630         __ lea(c_rarg0, Address(src, src_pos, Address::uxtw(scale)));
2631         __ add(c_rarg0, c_rarg0, arrayOopDesc::base_offset_in_bytes(basic_type));
2632         assert_different_registers(c_rarg0, dst, dst_pos, length);
2633         __ lea(c_rarg1, Address(dst, dst_pos, Address::uxtw(scale)));
2634         __ add(c_rarg1, c_rarg1, arrayOopDesc::base_offset_in_bytes(basic_type));
2635         assert_different_registers(c_rarg1, dst, length);
2636         __ uxtw(c_rarg2, length);
2637         assert_different_registers(c_rarg2, dst);
2638 
2639         __ load_klass(c_rarg4, dst);
2640         __ ldr(c_rarg4, Address(c_rarg4, ObjArrayKlass::element_klass_offset()));
2641         __ ldrw(c_rarg3, Address(c_rarg4, Klass::super_check_offset_offset()));
2642         __ far_call(RuntimeAddress(copyfunc_addr));
2643 
2644 #ifndef PRODUCT
2645         if (PrintC1Statistics) {
2646           Label failed;
2647           __ cbnz(r0, failed);
2648           __ incrementw(ExternalAddress((address)&Runtime1::_arraycopy_checkcast_cnt));
2649           __ bind(failed);
2650         }
2651 #endif
2652 
2653         __ cbz(r0, *stub->continuation());
2654 
2655 #ifndef PRODUCT
2656         if (PrintC1Statistics) {
2657           __ incrementw(ExternalAddress((address)&Runtime1::_arraycopy_checkcast_attempt_cnt));
2658         }
2659 #endif
2660         assert_different_registers(dst, dst_pos, length, src_pos, src, r0, rscratch1);
2661 
2662         // Restore previously spilled arguments
2663         __ ldp(dst,     dst_pos, Address(sp, 0*BytesPerWord));
2664         __ ldp(length,  src_pos, Address(sp, 2*BytesPerWord));
2665         __ ldr(src,              Address(sp, 4*BytesPerWord));
2666 
2667         // return value is -1^K where K is partial copied count
2668         __ eonw(rscratch1, r0, zr);
2669         // adjust length down and src/end pos up by partial copied count
2670         __ subw(length, length, rscratch1);
2671         __ addw(src_pos, src_pos, rscratch1);
2672         __ addw(dst_pos, dst_pos, rscratch1);
2673       }
2674 
2675       __ b(*stub->entry());
2676 
2677       __ bind(cont);
2678       __ POP(src, dst);
2679     }
2680   }
2681 
2682 #ifdef ASSERT
2683   if (basic_type != T_OBJECT || !(flags & LIR_OpArrayCopy::type_check)) {
2684     // Sanity check the known type with the incoming class.  For the
2685     // primitive case the types must match exactly with src.klass and
2686     // dst.klass each exactly matching the default type.  For the
2687     // object array case, if no type check is needed then either the
2688     // dst type is exactly the expected type and the src type is a
2689     // subtype which we can't check or src is the same array as dst
2690     // but not necessarily exactly of type default_type.
2691     Label known_ok, halt;
2692     __ mov_metadata(tmp, default_type->constant_encoding());
2693     if (UseCompressedClassPointers) {
2694       __ encode_klass_not_null(tmp);
2695     }
2696 
2697     if (basic_type != T_OBJECT) {
2698 
2699       if (UseCompressedClassPointers) {
2700         __ ldrw(rscratch1, dst_klass_addr);
2701         __ cmpw(tmp, rscratch1);
2702       } else {
2703         __ ldr(rscratch1, dst_klass_addr);
2704         __ cmp(tmp, rscratch1);
2705       }
2706       __ br(Assembler::NE, halt);
2707       if (UseCompressedClassPointers) {
2708         __ ldrw(rscratch1, src_klass_addr);
2709         __ cmpw(tmp, rscratch1);
2710       } else {
2711         __ ldr(rscratch1, src_klass_addr);
2712         __ cmp(tmp, rscratch1);
2713       }
2714       __ br(Assembler::EQ, known_ok);
2715     } else {
2716       if (UseCompressedClassPointers) {
2717         __ ldrw(rscratch1, dst_klass_addr);
2718         __ cmpw(tmp, rscratch1);
2719       } else {
2720         __ ldr(rscratch1, dst_klass_addr);
2721         __ cmp(tmp, rscratch1);
2722       }
2723       __ br(Assembler::EQ, known_ok);
2724       __ cmp(src, dst);
2725       __ br(Assembler::EQ, known_ok);
2726     }
2727     __ bind(halt);
2728     __ stop("incorrect type information in arraycopy");
2729     __ bind(known_ok);
2730   }
2731 #endif
2732 
2733 #ifndef PRODUCT
2734   if (PrintC1Statistics) {
2735     __ incrementw(ExternalAddress(Runtime1::arraycopy_count_address(basic_type)));
2736   }
2737 #endif
2738 
2739   __ lea(c_rarg0, Address(src, src_pos, Address::uxtw(scale)));
2740   __ add(c_rarg0, c_rarg0, arrayOopDesc::base_offset_in_bytes(basic_type));
2741   assert_different_registers(c_rarg0, dst, dst_pos, length);
2742   __ lea(c_rarg1, Address(dst, dst_pos, Address::uxtw(scale)));
2743   __ add(c_rarg1, c_rarg1, arrayOopDesc::base_offset_in_bytes(basic_type));
2744   assert_different_registers(c_rarg1, dst, length);
2745   __ uxtw(c_rarg2, length);
2746   assert_different_registers(c_rarg2, dst);
2747 
2748   bool disjoint = (flags & LIR_OpArrayCopy::overlapping) == 0;
2749   bool aligned = (flags & LIR_OpArrayCopy::unaligned) == 0;
2750   const char *name;
2751   address entry = StubRoutines::select_arraycopy_function(basic_type, aligned, disjoint, name, false);
2752 
2753  CodeBlob *cb = CodeCache::find_blob(entry);
2754  if (cb) {
2755    __ far_call(RuntimeAddress(entry));
2756  } else {
2757    __ call_VM_leaf(entry, 3);
2758  }
2759 
2760   __ bind(*stub->continuation());
2761 }
2762 
2763 
2764 
2765 
2766 void LIR_Assembler::emit_lock(LIR_OpLock* op) {
2767   Register obj = op->obj_opr()->as_register();  // may not be an oop
2768   Register hdr = op->hdr_opr()->as_register();
2769   Register lock = op->lock_opr()->as_register();
2770   if (!UseFastLocking) {
2771     __ b(*op->stub()->entry());
2772   } else if (op->code() == lir_lock) {
2773     Register scratch = noreg;
2774     if (UseBiasedLocking) {
2775       scratch = op->scratch_opr()->as_register();
2776     }
2777     assert(BasicLock::displaced_header_offset_in_bytes() == 0, "lock_reg must point to the displaced header");
2778     __ resolve(ACCESS_READ | ACCESS_WRITE, obj);
2779     // add debug info for NullPointerException only if one is possible
2780     int null_check_offset = __ lock_object(hdr, obj, lock, scratch, *op->stub()->entry());
2781     if (op->info() != NULL) {
2782       add_debug_info_for_null_check(null_check_offset, op->info());
2783     }
2784     // done
2785   } else if (op->code() == lir_unlock) {
2786     assert(BasicLock::displaced_header_offset_in_bytes() == 0, "lock_reg must point to the displaced header");
2787     __ unlock_object(hdr, obj, lock, *op->stub()->entry());
2788   } else {
2789     Unimplemented();
2790   }
2791   __ bind(*op->stub()->continuation());
2792 }
2793 
2794 
2795 void LIR_Assembler::emit_profile_call(LIR_OpProfileCall* op) {
2796   ciMethod* method = op->profiled_method();
2797   int bci          = op->profiled_bci();
2798   ciMethod* callee = op->profiled_callee();
2799 
2800   // Update counter for all call types
2801   ciMethodData* md = method->method_data_or_null();
2802   assert(md != NULL, "Sanity");
2803   ciProfileData* data = md->bci_to_data(bci);
2804   assert(data != NULL && data->is_CounterData(), "need CounterData for calls");
2805   assert(op->mdo()->is_single_cpu(),  "mdo must be allocated");
2806   Register mdo  = op->mdo()->as_register();
2807   __ mov_metadata(mdo, md->constant_encoding());
2808   Address counter_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
2809   // Perform additional virtual call profiling for invokevirtual and
2810   // invokeinterface bytecodes
2811   if (op->should_profile_receiver_type()) {
2812     assert(op->recv()->is_single_cpu(), "recv must be allocated");
2813     Register recv = op->recv()->as_register();
2814     assert_different_registers(mdo, recv);
2815     assert(data->is_VirtualCallData(), "need VirtualCallData for virtual calls");
2816     ciKlass* known_klass = op->known_holder();
2817     if (C1OptimizeVirtualCallProfiling && known_klass != NULL) {
2818       // We know the type that will be seen at this call site; we can
2819       // statically update the MethodData* rather than needing to do
2820       // dynamic tests on the receiver type
2821 
2822       // NOTE: we should probably put a lock around this search to
2823       // avoid collisions by concurrent compilations
2824       ciVirtualCallData* vc_data = (ciVirtualCallData*) data;
2825       uint i;
2826       for (i = 0; i < VirtualCallData::row_limit(); i++) {
2827         ciKlass* receiver = vc_data->receiver(i);
2828         if (known_klass->equals(receiver)) {
2829           Address data_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_count_offset(i)));
2830           __ addptr(data_addr, DataLayout::counter_increment);
2831           return;
2832         }
2833       }
2834 
2835       // Receiver type not found in profile data; select an empty slot
2836 
2837       // Note that this is less efficient than it should be because it
2838       // always does a write to the receiver part of the
2839       // VirtualCallData rather than just the first time
2840       for (i = 0; i < VirtualCallData::row_limit(); i++) {
2841         ciKlass* receiver = vc_data->receiver(i);
2842         if (receiver == NULL) {
2843           Address recv_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_offset(i)));
2844           __ mov_metadata(rscratch1, known_klass->constant_encoding());
2845           __ lea(rscratch2, recv_addr);
2846           __ str(rscratch1, Address(rscratch2));
2847           Address data_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_count_offset(i)));
2848           __ addptr(data_addr, DataLayout::counter_increment);
2849           return;
2850         }
2851       }
2852     } else {
2853       __ load_klass(recv, recv);
2854       Label update_done;
2855       type_profile_helper(mdo, md, data, recv, &update_done);
2856       // Receiver did not match any saved receiver and there is no empty row for it.
2857       // Increment total counter to indicate polymorphic case.
2858       __ addptr(counter_addr, DataLayout::counter_increment);
2859 
2860       __ bind(update_done);
2861     }
2862   } else {
2863     // Static call
2864     __ addptr(counter_addr, DataLayout::counter_increment);
2865   }
2866 }
2867 
2868 
2869 void LIR_Assembler::emit_delay(LIR_OpDelay*) {
2870   Unimplemented();
2871 }
2872 
2873 
2874 void LIR_Assembler::monitor_address(int monitor_no, LIR_Opr dst) {
2875   __ lea(dst->as_register(), frame_map()->address_for_monitor_lock(monitor_no));
2876 }
2877 
2878 void LIR_Assembler::emit_updatecrc32(LIR_OpUpdateCRC32* op) {
2879   assert(op->crc()->is_single_cpu(),  "crc must be register");
2880   assert(op->val()->is_single_cpu(),  "byte value must be register");
2881   assert(op->result_opr()->is_single_cpu(), "result must be register");
2882   Register crc = op->crc()->as_register();
2883   Register val = op->val()->as_register();
2884   Register res = op->result_opr()->as_register();
2885 
2886   assert_different_registers(val, crc, res);
2887   unsigned long offset;
2888   __ adrp(res, ExternalAddress(StubRoutines::crc_table_addr()), offset);
2889   if (offset) __ add(res, res, offset);
2890 
2891   __ mvnw(crc, crc); // ~crc
2892   __ update_byte_crc32(crc, val, res);
2893   __ mvnw(res, crc); // ~crc
2894 }
2895 
2896 void LIR_Assembler::emit_profile_type(LIR_OpProfileType* op) {
2897   COMMENT("emit_profile_type {");
2898   Register obj = op->obj()->as_register();
2899   Register tmp = op->tmp()->as_pointer_register();
2900   Address mdo_addr = as_Address(op->mdp()->as_address_ptr());
2901   ciKlass* exact_klass = op->exact_klass();
2902   intptr_t current_klass = op->current_klass();
2903   bool not_null = op->not_null();
2904   bool no_conflict = op->no_conflict();
2905 
2906   Label update, next, none;
2907 
2908   bool do_null = !not_null;
2909   bool exact_klass_set = exact_klass != NULL && ciTypeEntries::valid_ciklass(current_klass) == exact_klass;
2910   bool do_update = !TypeEntries::is_type_unknown(current_klass) && !exact_klass_set;
2911 
2912   assert(do_null || do_update, "why are we here?");
2913   assert(!TypeEntries::was_null_seen(current_klass) || do_update, "why are we here?");
2914   assert(mdo_addr.base() != rscratch1, "wrong register");
2915 
2916   __ verify_oop(obj);
2917 
2918   if (tmp != obj) {
2919     __ mov(tmp, obj);
2920   }
2921   if (do_null) {
2922     __ cbnz(tmp, update);
2923     if (!TypeEntries::was_null_seen(current_klass)) {
2924       __ ldr(rscratch2, mdo_addr);
2925       __ orr(rscratch2, rscratch2, TypeEntries::null_seen);
2926       __ str(rscratch2, mdo_addr);
2927     }
2928     if (do_update) {
2929 #ifndef ASSERT
2930       __ b(next);
2931     }
2932 #else
2933       __ b(next);
2934     }
2935   } else {
2936     __ cbnz(tmp, update);
2937     __ stop("unexpected null obj");
2938 #endif
2939   }
2940 
2941   __ bind(update);
2942 
2943   if (do_update) {
2944 #ifdef ASSERT
2945     if (exact_klass != NULL) {
2946       Label ok;
2947       __ load_klass(tmp, tmp);
2948       __ mov_metadata(rscratch1, exact_klass->constant_encoding());
2949       __ eor(rscratch1, tmp, rscratch1);
2950       __ cbz(rscratch1, ok);
2951       __ stop("exact klass and actual klass differ");
2952       __ bind(ok);
2953     }
2954 #endif
2955     if (!no_conflict) {
2956       if (exact_klass == NULL || TypeEntries::is_type_none(current_klass)) {
2957         if (exact_klass != NULL) {
2958           __ mov_metadata(tmp, exact_klass->constant_encoding());
2959         } else {
2960           __ load_klass(tmp, tmp);
2961         }
2962 
2963         __ ldr(rscratch2, mdo_addr);
2964         __ eor(tmp, tmp, rscratch2);
2965         __ andr(rscratch1, tmp, TypeEntries::type_klass_mask);
2966         // klass seen before, nothing to do. The unknown bit may have been
2967         // set already but no need to check.
2968         __ cbz(rscratch1, next);
2969 
2970         __ tbnz(tmp, exact_log2(TypeEntries::type_unknown), next); // already unknown. Nothing to do anymore.
2971 
2972         if (TypeEntries::is_type_none(current_klass)) {
2973           __ cbz(rscratch2, none);
2974           __ cmp(rscratch2, (u1)TypeEntries::null_seen);
2975           __ br(Assembler::EQ, none);
2976           // There is a chance that the checks above (re-reading profiling
2977           // data from memory) fail if another thread has just set the
2978           // profiling to this obj's klass
2979           __ dmb(Assembler::ISHLD);
2980           __ ldr(rscratch2, mdo_addr);
2981           __ eor(tmp, tmp, rscratch2);
2982           __ andr(rscratch1, tmp, TypeEntries::type_klass_mask);
2983           __ cbz(rscratch1, next);
2984         }
2985       } else {
2986         assert(ciTypeEntries::valid_ciklass(current_klass) != NULL &&
2987                ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "conflict only");
2988 
2989         __ ldr(tmp, mdo_addr);
2990         __ tbnz(tmp, exact_log2(TypeEntries::type_unknown), next); // already unknown. Nothing to do anymore.
2991       }
2992 
2993       // different than before. Cannot keep accurate profile.
2994       __ ldr(rscratch2, mdo_addr);
2995       __ orr(rscratch2, rscratch2, TypeEntries::type_unknown);
2996       __ str(rscratch2, mdo_addr);
2997 
2998       if (TypeEntries::is_type_none(current_klass)) {
2999         __ b(next);
3000 
3001         __ bind(none);
3002         // first time here. Set profile type.
3003         __ str(tmp, mdo_addr);
3004       }
3005     } else {
3006       // There's a single possible klass at this profile point
3007       assert(exact_klass != NULL, "should be");
3008       if (TypeEntries::is_type_none(current_klass)) {
3009         __ mov_metadata(tmp, exact_klass->constant_encoding());
3010         __ ldr(rscratch2, mdo_addr);
3011         __ eor(tmp, tmp, rscratch2);
3012         __ andr(rscratch1, tmp, TypeEntries::type_klass_mask);
3013         __ cbz(rscratch1, next);
3014 #ifdef ASSERT
3015         {
3016           Label ok;
3017           __ ldr(rscratch1, mdo_addr);
3018           __ cbz(rscratch1, ok);
3019           __ cmp(rscratch1, (u1)TypeEntries::null_seen);
3020           __ br(Assembler::EQ, ok);
3021           // may have been set by another thread
3022           __ dmb(Assembler::ISHLD);
3023           __ mov_metadata(rscratch1, exact_klass->constant_encoding());
3024           __ ldr(rscratch2, mdo_addr);
3025           __ eor(rscratch2, rscratch1, rscratch2);
3026           __ andr(rscratch2, rscratch2, TypeEntries::type_mask);
3027           __ cbz(rscratch2, ok);
3028 
3029           __ stop("unexpected profiling mismatch");
3030           __ bind(ok);
3031         }
3032 #endif
3033         // first time here. Set profile type.
3034         __ ldr(tmp, mdo_addr);
3035       } else {
3036         assert(ciTypeEntries::valid_ciklass(current_klass) != NULL &&
3037                ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "inconsistent");
3038 
3039         __ ldr(tmp, mdo_addr);
3040         __ tbnz(tmp, exact_log2(TypeEntries::type_unknown), next); // already unknown. Nothing to do anymore.
3041 
3042         __ orr(tmp, tmp, TypeEntries::type_unknown);
3043         __ str(tmp, mdo_addr);
3044         // FIXME: Write barrier needed here?
3045       }
3046     }
3047 
3048     __ bind(next);
3049   }
3050   COMMENT("} emit_profile_type");
3051 }
3052 
3053 
3054 void LIR_Assembler::align_backward_branch_target() {
3055 }
3056 
3057 
3058 void LIR_Assembler::negate(LIR_Opr left, LIR_Opr dest, LIR_Opr tmp) {
3059   // tmp must be unused
3060   assert(tmp->is_illegal(), "wasting a register if tmp is allocated");
3061 
3062   if (left->is_single_cpu()) {
3063     assert(dest->is_single_cpu(), "expect single result reg");
3064     __ negw(dest->as_register(), left->as_register());
3065   } else if (left->is_double_cpu()) {
3066     assert(dest->is_double_cpu(), "expect double result reg");
3067     __ neg(dest->as_register_lo(), left->as_register_lo());
3068   } else if (left->is_single_fpu()) {
3069     assert(dest->is_single_fpu(), "expect single float result reg");
3070     __ fnegs(dest->as_float_reg(), left->as_float_reg());
3071   } else {
3072     assert(left->is_double_fpu(), "expect double float operand reg");
3073     assert(dest->is_double_fpu(), "expect double float result reg");
3074     __ fnegd(dest->as_double_reg(), left->as_double_reg());
3075   }
3076 }
3077 
3078 
3079 void LIR_Assembler::leal(LIR_Opr addr, LIR_Opr dest, LIR_PatchCode patch_code, CodeEmitInfo* info) {
3080   if (patch_code != lir_patch_none) {
3081     deoptimize_trap(info);
3082     return;
3083   }
3084 
3085   __ lea(dest->as_register_lo(), as_Address(addr->as_address_ptr()));
3086 }
3087 
3088 
3089 void LIR_Assembler::rt_call(LIR_Opr result, address dest, const LIR_OprList* args, LIR_Opr tmp, CodeEmitInfo* info) {
3090   assert(!tmp->is_valid(), "don't need temporary");
3091 
3092   CodeBlob *cb = CodeCache::find_blob(dest);
3093   if (cb) {
3094     __ far_call(RuntimeAddress(dest));
3095   } else {
3096     __ mov(rscratch1, RuntimeAddress(dest));
3097     int len = args->length();
3098     int type = 0;
3099     if (! result->is_illegal()) {
3100       switch (result->type()) {
3101       case T_VOID:
3102         type = 0;
3103         break;
3104       case T_INT:
3105       case T_LONG:
3106       case T_OBJECT:
3107       case T_VALUETYPE:
3108         type = 1;
3109         break;
3110       case T_FLOAT:
3111         type = 2;
3112         break;
3113       case T_DOUBLE:
3114         type = 3;
3115         break;
3116       default:
3117         ShouldNotReachHere();
3118         break;
3119       }
3120     }
3121     int num_gpargs = 0;
3122     int num_fpargs = 0;
3123     for (int i = 0; i < args->length(); i++) {
3124       LIR_Opr arg = args->at(i);
3125       if (arg->type() == T_FLOAT || arg->type() == T_DOUBLE) {
3126         num_fpargs++;
3127       } else {
3128         num_gpargs++;
3129       }
3130     }
3131     __ blrt(rscratch1, num_gpargs, num_fpargs, type);
3132   }
3133 
3134   if (info != NULL) {
3135     add_call_info_here(info);
3136   }
3137   __ maybe_isb();
3138 }
3139 
3140 void LIR_Assembler::volatile_move_op(LIR_Opr src, LIR_Opr dest, BasicType type, CodeEmitInfo* info) {
3141   if (dest->is_address() || src->is_address()) {
3142     move_op(src, dest, type, lir_patch_none, info,
3143             /*pop_fpu_stack*/false, /*unaligned*/false, /*wide*/false);
3144   } else {
3145     ShouldNotReachHere();
3146   }
3147 }
3148 
3149 #ifdef ASSERT
3150 // emit run-time assertion
3151 void LIR_Assembler::emit_assert(LIR_OpAssert* op) {
3152   assert(op->code() == lir_assert, "must be");
3153 
3154   if (op->in_opr1()->is_valid()) {
3155     assert(op->in_opr2()->is_valid(), "both operands must be valid");
3156     comp_op(op->condition(), op->in_opr1(), op->in_opr2(), op);
3157   } else {
3158     assert(op->in_opr2()->is_illegal(), "both operands must be illegal");
3159     assert(op->condition() == lir_cond_always, "no other conditions allowed");
3160   }
3161 
3162   Label ok;
3163   if (op->condition() != lir_cond_always) {
3164     Assembler::Condition acond = Assembler::AL;
3165     switch (op->condition()) {
3166       case lir_cond_equal:        acond = Assembler::EQ;  break;
3167       case lir_cond_notEqual:     acond = Assembler::NE;  break;
3168       case lir_cond_less:         acond = Assembler::LT;  break;
3169       case lir_cond_lessEqual:    acond = Assembler::LE;  break;
3170       case lir_cond_greaterEqual: acond = Assembler::GE;  break;
3171       case lir_cond_greater:      acond = Assembler::GT;  break;
3172       case lir_cond_belowEqual:   acond = Assembler::LS;  break;
3173       case lir_cond_aboveEqual:   acond = Assembler::HS;  break;
3174       default:                    ShouldNotReachHere();
3175     }
3176     __ br(acond, ok);
3177   }
3178   if (op->halt()) {
3179     const char* str = __ code_string(op->msg());
3180     __ stop(str);
3181   } else {
3182     breakpoint();
3183   }
3184   __ bind(ok);
3185 }
3186 #endif
3187 
3188 #ifndef PRODUCT
3189 #define COMMENT(x)   do { __ block_comment(x); } while (0)
3190 #else
3191 #define COMMENT(x)
3192 #endif
3193 
3194 void LIR_Assembler::membar() {
3195   COMMENT("membar");
3196   __ membar(MacroAssembler::AnyAny);
3197 }
3198 
3199 void LIR_Assembler::membar_acquire() {
3200   __ membar(Assembler::LoadLoad|Assembler::LoadStore);
3201 }
3202 
3203 void LIR_Assembler::membar_release() {
3204   __ membar(Assembler::LoadStore|Assembler::StoreStore);
3205 }
3206 
3207 void LIR_Assembler::membar_loadload() {
3208   __ membar(Assembler::LoadLoad);
3209 }
3210 
3211 void LIR_Assembler::membar_storestore() {
3212   __ membar(MacroAssembler::StoreStore);
3213 }
3214 
3215 void LIR_Assembler::membar_loadstore() { __ membar(MacroAssembler::LoadStore); }
3216 
3217 void LIR_Assembler::membar_storeload() { __ membar(MacroAssembler::StoreLoad); }
3218 
3219 void LIR_Assembler::on_spin_wait() {
3220   Unimplemented();
3221 }
3222 
3223 void LIR_Assembler::get_thread(LIR_Opr result_reg) {
3224   __ mov(result_reg->as_register(), rthread);
3225 }
3226 
3227 
3228 void LIR_Assembler::peephole(LIR_List *lir) {
3229 #if 0
3230   if (tableswitch_count >= max_tableswitches)
3231     return;
3232 
3233   /*
3234     This finite-state automaton recognizes sequences of compare-and-
3235     branch instructions.  We will turn them into a tableswitch.  You
3236     could argue that C1 really shouldn't be doing this sort of
3237     optimization, but without it the code is really horrible.
3238   */
3239 
3240   enum { start_s, cmp1_s, beq_s, cmp_s } state;
3241   int first_key, last_key = -2147483648;
3242   int next_key = 0;
3243   int start_insn = -1;
3244   int last_insn = -1;
3245   Register reg = noreg;
3246   LIR_Opr reg_opr;
3247   state = start_s;
3248 
3249   LIR_OpList* inst = lir->instructions_list();
3250   for (int i = 0; i < inst->length(); i++) {
3251     LIR_Op* op = inst->at(i);
3252     switch (state) {
3253     case start_s:
3254       first_key = -1;
3255       start_insn = i;
3256       switch (op->code()) {
3257       case lir_cmp:
3258         LIR_Opr opr1 = op->as_Op2()->in_opr1();
3259         LIR_Opr opr2 = op->as_Op2()->in_opr2();
3260         if (opr1->is_cpu_register() && opr1->is_single_cpu()
3261             && opr2->is_constant()
3262             && opr2->type() == T_INT) {
3263           reg_opr = opr1;
3264           reg = opr1->as_register();
3265           first_key = opr2->as_constant_ptr()->as_jint();
3266           next_key = first_key + 1;
3267           state = cmp_s;
3268           goto next_state;
3269         }
3270         break;
3271       }
3272       break;
3273     case cmp_s:
3274       switch (op->code()) {
3275       case lir_branch:
3276         if (op->as_OpBranch()->cond() == lir_cond_equal) {
3277           state = beq_s;
3278           last_insn = i;
3279           goto next_state;
3280         }
3281       }
3282       state = start_s;
3283       break;
3284     case beq_s:
3285       switch (op->code()) {
3286       case lir_cmp: {
3287         LIR_Opr opr1 = op->as_Op2()->in_opr1();
3288         LIR_Opr opr2 = op->as_Op2()->in_opr2();
3289         if (opr1->is_cpu_register() && opr1->is_single_cpu()
3290             && opr1->as_register() == reg
3291             && opr2->is_constant()
3292             && opr2->type() == T_INT
3293             && opr2->as_constant_ptr()->as_jint() == next_key) {
3294           last_key = next_key;
3295           next_key++;
3296           state = cmp_s;
3297           goto next_state;
3298         }
3299       }
3300       }
3301       last_key = next_key;
3302       state = start_s;
3303       break;
3304     default:
3305       assert(false, "impossible state");
3306     }
3307     if (state == start_s) {
3308       if (first_key < last_key - 5L && reg != noreg) {
3309         {
3310           // printf("found run register %d starting at insn %d low value %d high value %d\n",
3311           //        reg->encoding(),
3312           //        start_insn, first_key, last_key);
3313           //   for (int i = 0; i < inst->length(); i++) {
3314           //     inst->at(i)->print();
3315           //     tty->print("\n");
3316           //   }
3317           //   tty->print("\n");
3318         }
3319 
3320         struct tableswitch *sw = &switches[tableswitch_count];
3321         sw->_insn_index = start_insn, sw->_first_key = first_key,
3322           sw->_last_key = last_key, sw->_reg = reg;
3323         inst->insert_before(last_insn + 1, new LIR_OpLabel(&sw->_after));
3324         {
3325           // Insert the new table of branches
3326           int offset = last_insn;
3327           for (int n = first_key; n < last_key; n++) {
3328             inst->insert_before
3329               (last_insn + 1,
3330                new LIR_OpBranch(lir_cond_always, T_ILLEGAL,
3331                                 inst->at(offset)->as_OpBranch()->label()));
3332             offset -= 2, i++;
3333           }
3334         }
3335         // Delete all the old compare-and-branch instructions
3336         for (int n = first_key; n < last_key; n++) {
3337           inst->remove_at(start_insn);
3338           inst->remove_at(start_insn);
3339         }
3340         // Insert the tableswitch instruction
3341         inst->insert_before(start_insn,
3342                             new LIR_Op2(lir_cmp, lir_cond_always,
3343                                         LIR_OprFact::intConst(tableswitch_count),
3344                                         reg_opr));
3345         inst->insert_before(start_insn + 1, new LIR_OpLabel(&sw->_branches));
3346         tableswitch_count++;
3347       }
3348       reg = noreg;
3349       last_key = -2147483648;
3350     }
3351   next_state:
3352     ;
3353   }
3354 #endif
3355 }
3356 
3357 void LIR_Assembler::atomic_op(LIR_Code code, LIR_Opr src, LIR_Opr data, LIR_Opr dest, LIR_Opr tmp_op) {
3358   Address addr = as_Address(src->as_address_ptr());
3359   BasicType type = src->type();
3360   bool is_oop = type == T_OBJECT || type == T_ARRAY;
3361 
3362   void (MacroAssembler::* add)(Register prev, RegisterOrConstant incr, Register addr);
3363   void (MacroAssembler::* xchg)(Register prev, Register newv, Register addr);
3364 
3365   switch(type) {
3366   case T_INT:
3367     xchg = &MacroAssembler::atomic_xchgalw;
3368     add = &MacroAssembler::atomic_addalw;
3369     break;
3370   case T_LONG:
3371     xchg = &MacroAssembler::atomic_xchgal;
3372     add = &MacroAssembler::atomic_addal;
3373     break;
3374   case T_VALUETYPE:
3375   case T_OBJECT:
3376   case T_ARRAY:
3377     if (UseCompressedOops) {
3378       xchg = &MacroAssembler::atomic_xchgalw;
3379       add = &MacroAssembler::atomic_addalw;
3380     } else {
3381       xchg = &MacroAssembler::atomic_xchgal;
3382       add = &MacroAssembler::atomic_addal;
3383     }
3384     break;
3385   default:
3386     ShouldNotReachHere();
3387     xchg = &MacroAssembler::atomic_xchgal;
3388     add = &MacroAssembler::atomic_addal; // unreachable
3389   }
3390 
3391   switch (code) {
3392   case lir_xadd:
3393     {
3394       RegisterOrConstant inc;
3395       Register tmp = as_reg(tmp_op);
3396       Register dst = as_reg(dest);
3397       if (data->is_constant()) {
3398         inc = RegisterOrConstant(as_long(data));
3399         assert_different_registers(dst, addr.base(), tmp,
3400                                    rscratch1, rscratch2);
3401       } else {
3402         inc = RegisterOrConstant(as_reg(data));
3403         assert_different_registers(inc.as_register(), dst, addr.base(), tmp,
3404                                    rscratch1, rscratch2);
3405       }
3406       __ lea(tmp, addr);
3407       (_masm->*add)(dst, inc, tmp);
3408       break;
3409     }
3410   case lir_xchg:
3411     {
3412       Register tmp = tmp_op->as_register();
3413       Register obj = as_reg(data);
3414       Register dst = as_reg(dest);
3415       if (is_oop && UseCompressedOops) {
3416         __ encode_heap_oop(rscratch2, obj);
3417         obj = rscratch2;
3418       }
3419       assert_different_registers(obj, addr.base(), tmp, rscratch1, dst);
3420       __ lea(tmp, addr);
3421       (_masm->*xchg)(dst, obj, tmp);
3422       if (is_oop && UseCompressedOops) {
3423         __ decode_heap_oop(dst);
3424       }
3425     }
3426     break;
3427   default:
3428     ShouldNotReachHere();
3429   }
3430   __ membar(__ AnyAny);
3431 }
3432 
3433 #undef __