1 /*
   2  * Copyright (c) 2000, 2020, Oracle and/or its affiliates. All rights reserved.
   3  * Copyright (c) 2014, Red Hat Inc. All rights reserved.
   4  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   5  *
   6  * This code is free software; you can redistribute it and/or modify it
   7  * under the terms of the GNU General Public License version 2 only, as
   8  * published by the Free Software Foundation.
   9  *
  10  * This code is distributed in the hope that it will be useful, but WITHOUT
  11  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  12  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  13  * version 2 for more details (a copy is included in the LICENSE file that
  14  * accompanied this code).
  15  *
  16  * You should have received a copy of the GNU General Public License version
  17  * 2 along with this work; if not, write to the Free Software Foundation,
  18  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  19  *
  20  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  21  * or visit www.oracle.com if you need additional information or have any
  22  * questions.
  23  *
  24  */
  25 
  26 #include "precompiled.hpp"
  27 #include "asm/macroAssembler.inline.hpp"
  28 #include "asm/assembler.hpp"
  29 #include "c1/c1_CodeStubs.hpp"
  30 #include "c1/c1_Compilation.hpp"
  31 #include "c1/c1_LIRAssembler.hpp"
  32 #include "c1/c1_MacroAssembler.hpp"
  33 #include "c1/c1_Runtime1.hpp"
  34 #include "c1/c1_ValueStack.hpp"
  35 #include "ci/ciArrayKlass.hpp"
  36 #include "ci/ciInstance.hpp"
  37 #include "code/compiledIC.hpp"
  38 #include "gc/shared/collectedHeap.hpp"
  39 #include "nativeInst_aarch64.hpp"
  40 #include "oops/objArrayKlass.hpp"
  41 #include "runtime/frame.inline.hpp"
  42 #include "runtime/sharedRuntime.hpp"
  43 #include "utilities/powerOfTwo.hpp"
  44 #include "vmreg_aarch64.inline.hpp"
  45 
  46 
  47 #ifndef PRODUCT
  48 #define COMMENT(x)   do { __ block_comment(x); } while (0)
  49 #else
  50 #define COMMENT(x)
  51 #endif
  52 
  53 NEEDS_CLEANUP // remove this definitions ?
  54 const Register IC_Klass    = rscratch2;   // where the IC klass is cached
  55 const Register SYNC_header = r0;   // synchronization header
  56 const Register SHIFT_count = r0;   // where count for shift operations must be
  57 
  58 #define __ _masm->
  59 
  60 
  61 static void select_different_registers(Register preserve,
  62                                        Register extra,
  63                                        Register &tmp1,
  64                                        Register &tmp2) {
  65   if (tmp1 == preserve) {
  66     assert_different_registers(tmp1, tmp2, extra);
  67     tmp1 = extra;
  68   } else if (tmp2 == preserve) {
  69     assert_different_registers(tmp1, tmp2, extra);
  70     tmp2 = extra;
  71   }
  72   assert_different_registers(preserve, tmp1, tmp2);
  73 }
  74 
  75 
  76 
  77 static void select_different_registers(Register preserve,
  78                                        Register extra,
  79                                        Register &tmp1,
  80                                        Register &tmp2,
  81                                        Register &tmp3) {
  82   if (tmp1 == preserve) {
  83     assert_different_registers(tmp1, tmp2, tmp3, extra);
  84     tmp1 = extra;
  85   } else if (tmp2 == preserve) {
  86     assert_different_registers(tmp1, tmp2, tmp3, extra);
  87     tmp2 = extra;
  88   } else if (tmp3 == preserve) {
  89     assert_different_registers(tmp1, tmp2, tmp3, extra);
  90     tmp3 = extra;
  91   }
  92   assert_different_registers(preserve, tmp1, tmp2, tmp3);
  93 }
  94 
  95 
  96 bool LIR_Assembler::is_small_constant(LIR_Opr opr) { Unimplemented(); return false; }
  97 
  98 
  99 LIR_Opr LIR_Assembler::receiverOpr() {
 100   return FrameMap::receiver_opr;
 101 }
 102 
 103 LIR_Opr LIR_Assembler::osrBufferPointer() {
 104   return FrameMap::as_pointer_opr(receiverOpr()->as_register());
 105 }
 106 
 107 //--------------fpu register translations-----------------------
 108 
 109 
 110 address LIR_Assembler::float_constant(float f) {
 111   address const_addr = __ float_constant(f);
 112   if (const_addr == NULL) {
 113     bailout("const section overflow");
 114     return __ code()->consts()->start();
 115   } else {
 116     return const_addr;
 117   }
 118 }
 119 
 120 
 121 address LIR_Assembler::double_constant(double d) {
 122   address const_addr = __ double_constant(d);
 123   if (const_addr == NULL) {
 124     bailout("const section overflow");
 125     return __ code()->consts()->start();
 126   } else {
 127     return const_addr;
 128   }
 129 }
 130 
 131 address LIR_Assembler::int_constant(jlong n) {
 132   address const_addr = __ long_constant(n);
 133   if (const_addr == NULL) {
 134     bailout("const section overflow");
 135     return __ code()->consts()->start();
 136   } else {
 137     return const_addr;
 138   }
 139 }
 140 
 141 void LIR_Assembler::breakpoint() { Unimplemented(); }
 142 
 143 void LIR_Assembler::push(LIR_Opr opr) { Unimplemented(); }
 144 
 145 void LIR_Assembler::pop(LIR_Opr opr) { Unimplemented(); }
 146 
 147 bool LIR_Assembler::is_literal_address(LIR_Address* addr) { Unimplemented(); return false; }
 148 //-------------------------------------------
 149 
 150 static Register as_reg(LIR_Opr op) {
 151   return op->is_double_cpu() ? op->as_register_lo() : op->as_register();
 152 }
 153 
 154 static jlong as_long(LIR_Opr data) {
 155   jlong result;
 156   switch (data->type()) {
 157   case T_INT:
 158     result = (data->as_jint());
 159     break;
 160   case T_LONG:
 161     result = (data->as_jlong());
 162     break;
 163   default:
 164     ShouldNotReachHere();
 165     result = 0;  // unreachable
 166   }
 167   return result;
 168 }
 169 
 170 Address LIR_Assembler::as_Address(LIR_Address* addr, Register tmp) {
 171   Register base = addr->base()->as_pointer_register();
 172   LIR_Opr opr = addr->index();
 173   if (opr->is_cpu_register()) {
 174     Register index;
 175     if (opr->is_single_cpu())
 176       index = opr->as_register();
 177     else
 178       index = opr->as_register_lo();
 179     assert(addr->disp() == 0, "must be");
 180     switch(opr->type()) {
 181       case T_INT:
 182         return Address(base, index, Address::sxtw(addr->scale()));
 183       case T_LONG:
 184         return Address(base, index, Address::lsl(addr->scale()));
 185       default:
 186         ShouldNotReachHere();
 187       }
 188   } else  {
 189     intptr_t addr_offset = intptr_t(addr->disp());
 190     if (Address::offset_ok_for_immed(addr_offset, addr->scale()))
 191       return Address(base, addr_offset, Address::lsl(addr->scale()));
 192     else {
 193       __ mov(tmp, addr_offset);
 194       return Address(base, tmp, Address::lsl(addr->scale()));
 195     }
 196   }
 197   return Address();
 198 }
 199 
 200 Address LIR_Assembler::as_Address_hi(LIR_Address* addr) {
 201   ShouldNotReachHere();
 202   return Address();
 203 }
 204 
 205 Address LIR_Assembler::as_Address(LIR_Address* addr) {
 206   return as_Address(addr, rscratch1);
 207 }
 208 
 209 Address LIR_Assembler::as_Address_lo(LIR_Address* addr) {
 210   return as_Address(addr, rscratch1);  // Ouch
 211   // FIXME: This needs to be much more clever.  See x86.
 212 }
 213 
 214 
 215 void LIR_Assembler::osr_entry() {
 216   offsets()->set_value(CodeOffsets::OSR_Entry, code_offset());
 217   BlockBegin* osr_entry = compilation()->hir()->osr_entry();
 218   ValueStack* entry_state = osr_entry->state();
 219   int number_of_locks = entry_state->locks_size();
 220 
 221   // we jump here if osr happens with the interpreter
 222   // state set up to continue at the beginning of the
 223   // loop that triggered osr - in particular, we have
 224   // the following registers setup:
 225   //
 226   // r2: osr buffer
 227   //
 228 
 229   // build frame
 230   ciMethod* m = compilation()->method();
 231   __ build_frame(initial_frame_size_in_bytes(), bang_size_in_bytes());
 232 
 233   // OSR buffer is
 234   //
 235   // locals[nlocals-1..0]
 236   // monitors[0..number_of_locks]
 237   //
 238   // locals is a direct copy of the interpreter frame so in the osr buffer
 239   // so first slot in the local array is the last local from the interpreter
 240   // and last slot is local[0] (receiver) from the interpreter
 241   //
 242   // Similarly with locks. The first lock slot in the osr buffer is the nth lock
 243   // from the interpreter frame, the nth lock slot in the osr buffer is 0th lock
 244   // in the interpreter frame (the method lock if a sync method)
 245 
 246   // Initialize monitors in the compiled activation.
 247   //   r2: pointer to osr buffer
 248   //
 249   // All other registers are dead at this point and the locals will be
 250   // copied into place by code emitted in the IR.
 251 
 252   Register OSR_buf = osrBufferPointer()->as_pointer_register();
 253   { assert(frame::interpreter_frame_monitor_size() == BasicObjectLock::size(), "adjust code below");
 254     int monitor_offset = BytesPerWord * method()->max_locals() +
 255       (2 * BytesPerWord) * (number_of_locks - 1);
 256     // SharedRuntime::OSR_migration_begin() packs BasicObjectLocks in
 257     // the OSR buffer using 2 word entries: first the lock and then
 258     // the oop.
 259     for (int i = 0; i < number_of_locks; i++) {
 260       int slot_offset = monitor_offset - ((i * 2) * BytesPerWord);
 261 #ifdef ASSERT
 262       // verify the interpreter's monitor has a non-null object
 263       {
 264         Label L;
 265         __ ldr(rscratch1, Address(OSR_buf, slot_offset + 1*BytesPerWord));
 266         __ cbnz(rscratch1, L);
 267         __ stop("locked object is NULL");
 268         __ bind(L);
 269       }
 270 #endif
 271       __ ldr(r19, Address(OSR_buf, slot_offset + 0));
 272       __ str(r19, frame_map()->address_for_monitor_lock(i));
 273       __ ldr(r19, Address(OSR_buf, slot_offset + 1*BytesPerWord));
 274       __ str(r19, frame_map()->address_for_monitor_object(i));
 275     }
 276   }
 277 }
 278 
 279 
 280 // inline cache check; done before the frame is built.
 281 int LIR_Assembler::check_icache() {
 282   Register receiver = FrameMap::receiver_opr->as_register();
 283   Register ic_klass = IC_Klass;
 284   int start_offset = __ offset();
 285   __ inline_cache_check(receiver, ic_klass);
 286 
 287   // if icache check fails, then jump to runtime routine
 288   // Note: RECEIVER must still contain the receiver!
 289   Label dont;
 290   __ br(Assembler::EQ, dont);
 291   __ far_jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
 292 
 293   // We align the verified entry point unless the method body
 294   // (including its inline cache check) will fit in a single 64-byte
 295   // icache line.
 296   if (! method()->is_accessor() || __ offset() - start_offset > 4 * 4) {
 297     // force alignment after the cache check.
 298     __ align(CodeEntryAlignment);
 299   }
 300 
 301   __ bind(dont);
 302   return start_offset;
 303 }
 304 
 305 void LIR_Assembler::clinit_barrier(ciMethod* method) {
 306   assert(VM_Version::supports_fast_class_init_checks(), "sanity");
 307   assert(!method->holder()->is_not_initialized(), "initialization should have been started");
 308 
 309   Label L_skip_barrier;
 310 
 311   __ mov_metadata(rscratch2, method->holder()->constant_encoding());
 312   __ clinit_barrier(rscratch2, rscratch1, &L_skip_barrier /*L_fast_path*/);
 313   __ far_jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub()));
 314   __ bind(L_skip_barrier);
 315 }
 316 
 317 void LIR_Assembler::jobject2reg(jobject o, Register reg) {
 318   if (o == NULL) {
 319     __ mov(reg, zr);
 320   } else {
 321     __ movoop(reg, o, /*immediate*/true);
 322   }
 323 }
 324 
 325 void LIR_Assembler::deoptimize_trap(CodeEmitInfo *info) {
 326   address target = NULL;
 327   relocInfo::relocType reloc_type = relocInfo::none;
 328 
 329   switch (patching_id(info)) {
 330   case PatchingStub::access_field_id:
 331     target = Runtime1::entry_for(Runtime1::access_field_patching_id);
 332     reloc_type = relocInfo::section_word_type;
 333     break;
 334   case PatchingStub::load_klass_id:
 335     target = Runtime1::entry_for(Runtime1::load_klass_patching_id);
 336     reloc_type = relocInfo::metadata_type;
 337     break;
 338   case PatchingStub::load_mirror_id:
 339     target = Runtime1::entry_for(Runtime1::load_mirror_patching_id);
 340     reloc_type = relocInfo::oop_type;
 341     break;
 342   case PatchingStub::load_appendix_id:
 343     target = Runtime1::entry_for(Runtime1::load_appendix_patching_id);
 344     reloc_type = relocInfo::oop_type;
 345     break;
 346   default: ShouldNotReachHere();
 347   }
 348 
 349   __ far_call(RuntimeAddress(target));
 350   add_call_info_here(info);
 351 }
 352 
 353 void LIR_Assembler::jobject2reg_with_patching(Register reg, CodeEmitInfo *info) {
 354   deoptimize_trap(info);
 355 }
 356 
 357 
 358 // This specifies the rsp decrement needed to build the frame
 359 int LIR_Assembler::initial_frame_size_in_bytes() const {
 360   // if rounding, must let FrameMap know!
 361 
 362   // The frame_map records size in slots (32bit word)
 363 
 364   // subtract two words to account for return address and link
 365   return (frame_map()->framesize() - (2*VMRegImpl::slots_per_word))  * VMRegImpl::stack_slot_size;
 366 }
 367 
 368 
 369 int LIR_Assembler::emit_exception_handler() {
 370   // if the last instruction is a call (typically to do a throw which
 371   // is coming at the end after block reordering) the return address
 372   // must still point into the code area in order to avoid assertion
 373   // failures when searching for the corresponding bci => add a nop
 374   // (was bug 5/14/1999 - gri)
 375   __ nop();
 376 
 377   // generate code for exception handler
 378   address handler_base = __ start_a_stub(exception_handler_size());
 379   if (handler_base == NULL) {
 380     // not enough space left for the handler
 381     bailout("exception handler overflow");
 382     return -1;
 383   }
 384 
 385   int offset = code_offset();
 386 
 387   // the exception oop and pc are in r0, and r3
 388   // no other registers need to be preserved, so invalidate them
 389   __ invalidate_registers(false, true, true, false, true, true);
 390 
 391   // check that there is really an exception
 392   __ verify_not_null_oop(r0);
 393 
 394   // search an exception handler (r0: exception oop, r3: throwing pc)
 395   __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::handle_exception_from_callee_id)));  __ should_not_reach_here();
 396   guarantee(code_offset() - offset <= exception_handler_size(), "overflow");
 397   __ end_a_stub();
 398 
 399   return offset;
 400 }
 401 
 402 
 403 // Emit the code to remove the frame from the stack in the exception
 404 // unwind path.
 405 int LIR_Assembler::emit_unwind_handler() {
 406 #ifndef PRODUCT
 407   if (CommentedAssembly) {
 408     _masm->block_comment("Unwind handler");
 409   }
 410 #endif
 411 
 412   int offset = code_offset();
 413 
 414   // Fetch the exception from TLS and clear out exception related thread state
 415   __ ldr(r0, Address(rthread, JavaThread::exception_oop_offset()));
 416   __ str(zr, Address(rthread, JavaThread::exception_oop_offset()));
 417   __ str(zr, Address(rthread, JavaThread::exception_pc_offset()));
 418 
 419   __ bind(_unwind_handler_entry);
 420   __ verify_not_null_oop(r0);
 421   if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
 422     __ mov(r19, r0);  // Preserve the exception
 423   }
 424 
 425   // Preform needed unlocking
 426   MonitorExitStub* stub = NULL;
 427   if (method()->is_synchronized()) {
 428     monitor_address(0, FrameMap::r0_opr);
 429     stub = new MonitorExitStub(FrameMap::r0_opr, true, 0);
 430     __ unlock_object(r5, r4, r0, *stub->entry());
 431     __ bind(*stub->continuation());
 432   }
 433 
 434   if (compilation()->env()->dtrace_method_probes()) {
 435     __ mov(c_rarg0, rthread);
 436     __ mov_metadata(c_rarg1, method()->constant_encoding());
 437     __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_exit), c_rarg0, c_rarg1);
 438   }
 439 
 440   if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
 441     __ mov(r0, r19);  // Restore the exception
 442   }
 443 
 444   // remove the activation and dispatch to the unwind handler
 445   __ block_comment("remove_frame and dispatch to the unwind handler");
 446   __ remove_frame(initial_frame_size_in_bytes());
 447   __ far_jump(RuntimeAddress(Runtime1::entry_for(Runtime1::unwind_exception_id)));
 448 
 449   // Emit the slow path assembly
 450   if (stub != NULL) {
 451     stub->emit_code(this);
 452   }
 453 
 454   return offset;
 455 }
 456 
 457 
 458 int LIR_Assembler::emit_deopt_handler() {
 459   // if the last instruction is a call (typically to do a throw which
 460   // is coming at the end after block reordering) the return address
 461   // must still point into the code area in order to avoid assertion
 462   // failures when searching for the corresponding bci => add a nop
 463   // (was bug 5/14/1999 - gri)
 464   __ nop();
 465 
 466   // generate code for exception handler
 467   address handler_base = __ start_a_stub(deopt_handler_size());
 468   if (handler_base == NULL) {
 469     // not enough space left for the handler
 470     bailout("deopt handler overflow");
 471     return -1;
 472   }
 473 
 474   int offset = code_offset();
 475 
 476   __ adr(lr, pc());
 477   __ far_jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
 478   guarantee(code_offset() - offset <= deopt_handler_size(), "overflow");
 479   __ end_a_stub();
 480 
 481   return offset;
 482 }
 483 
 484 void LIR_Assembler::add_debug_info_for_branch(address adr, CodeEmitInfo* info) {
 485   _masm->code_section()->relocate(adr, relocInfo::poll_type);
 486   int pc_offset = code_offset();
 487   flush_debug_info(pc_offset);
 488   info->record_debug_info(compilation()->debug_info_recorder(), pc_offset);
 489   if (info->exception_handlers() != NULL) {
 490     compilation()->add_exception_handlers_for_pco(pc_offset, info->exception_handlers());
 491   }
 492 }
 493 
 494 void LIR_Assembler::return_op(LIR_Opr result) {
 495   assert(result->is_illegal() || !result->is_single_cpu() || result->as_register() == r0, "word returns are in r0,");
 496 
 497   // Pop the stack before the safepoint code
 498   __ remove_frame(initial_frame_size_in_bytes());
 499 
 500   if (StackReservedPages > 0 && compilation()->has_reserved_stack_access()) {
 501     __ reserved_stack_check();
 502   }
 503 
 504   __ fetch_and_read_polling_page(rscratch1, relocInfo::poll_return_type);
 505   __ ret(lr);
 506 }
 507 
 508 int LIR_Assembler::safepoint_poll(LIR_Opr tmp, CodeEmitInfo* info) {
 509   guarantee(info != NULL, "Shouldn't be NULL");
 510   __ get_polling_page(rscratch1, relocInfo::poll_type);
 511   add_debug_info_for_branch(info);  // This isn't just debug info:
 512                                     // it's the oop map
 513   __ read_polling_page(rscratch1, relocInfo::poll_type);
 514   return __ offset();
 515 }
 516 
 517 
 518 void LIR_Assembler::move_regs(Register from_reg, Register to_reg) {
 519   if (from_reg == r31_sp)
 520     from_reg = sp;
 521   if (to_reg == r31_sp)
 522     to_reg = sp;
 523   __ mov(to_reg, from_reg);
 524 }
 525 
 526 void LIR_Assembler::swap_reg(Register a, Register b) { Unimplemented(); }
 527 
 528 
 529 void LIR_Assembler::const2reg(LIR_Opr src, LIR_Opr dest, LIR_PatchCode patch_code, CodeEmitInfo* info) {
 530   assert(src->is_constant(), "should not call otherwise");
 531   assert(dest->is_register(), "should not call otherwise");
 532   LIR_Const* c = src->as_constant_ptr();
 533 
 534   switch (c->type()) {
 535     case T_INT: {
 536       assert(patch_code == lir_patch_none, "no patching handled here");
 537       __ movw(dest->as_register(), c->as_jint());
 538       break;
 539     }
 540 
 541     case T_ADDRESS: {
 542       assert(patch_code == lir_patch_none, "no patching handled here");
 543       __ mov(dest->as_register(), c->as_jint());
 544       break;
 545     }
 546 
 547     case T_LONG: {
 548       assert(patch_code == lir_patch_none, "no patching handled here");
 549       __ mov(dest->as_register_lo(), (intptr_t)c->as_jlong());
 550       break;
 551     }
 552 
 553     case T_OBJECT: {
 554         if (patch_code == lir_patch_none) {
 555           jobject2reg(c->as_jobject(), dest->as_register());
 556         } else {
 557           jobject2reg_with_patching(dest->as_register(), info);
 558         }
 559       break;
 560     }
 561 
 562     case T_METADATA: {
 563       if (patch_code != lir_patch_none) {
 564         klass2reg_with_patching(dest->as_register(), info);
 565       } else {
 566         __ mov_metadata(dest->as_register(), c->as_metadata());
 567       }
 568       break;
 569     }
 570 
 571     case T_FLOAT: {
 572       if (__ operand_valid_for_float_immediate(c->as_jfloat())) {
 573         __ fmovs(dest->as_float_reg(), (c->as_jfloat()));
 574       } else {
 575         __ adr(rscratch1, InternalAddress(float_constant(c->as_jfloat())));
 576         __ ldrs(dest->as_float_reg(), Address(rscratch1));
 577       }
 578       break;
 579     }
 580 
 581     case T_DOUBLE: {
 582       if (__ operand_valid_for_float_immediate(c->as_jdouble())) {
 583         __ fmovd(dest->as_double_reg(), (c->as_jdouble()));
 584       } else {
 585         __ adr(rscratch1, InternalAddress(double_constant(c->as_jdouble())));
 586         __ ldrd(dest->as_double_reg(), Address(rscratch1));
 587       }
 588       break;
 589     }
 590 
 591     default:
 592       ShouldNotReachHere();
 593   }
 594 }
 595 
 596 void LIR_Assembler::const2stack(LIR_Opr src, LIR_Opr dest) {
 597   LIR_Const* c = src->as_constant_ptr();
 598   switch (c->type()) {
 599   case T_OBJECT:
 600     {
 601       if (! c->as_jobject())
 602         __ str(zr, frame_map()->address_for_slot(dest->single_stack_ix()));
 603       else {
 604         const2reg(src, FrameMap::rscratch1_opr, lir_patch_none, NULL);
 605         reg2stack(FrameMap::rscratch1_opr, dest, c->type(), false);
 606       }
 607     }
 608     break;
 609   case T_ADDRESS:
 610     {
 611       const2reg(src, FrameMap::rscratch1_opr, lir_patch_none, NULL);
 612       reg2stack(FrameMap::rscratch1_opr, dest, c->type(), false);
 613     }
 614   case T_INT:
 615   case T_FLOAT:
 616     {
 617       Register reg = zr;
 618       if (c->as_jint_bits() == 0)
 619         __ strw(zr, frame_map()->address_for_slot(dest->single_stack_ix()));
 620       else {
 621         __ movw(rscratch1, c->as_jint_bits());
 622         __ strw(rscratch1, frame_map()->address_for_slot(dest->single_stack_ix()));
 623       }
 624     }
 625     break;
 626   case T_LONG:
 627   case T_DOUBLE:
 628     {
 629       Register reg = zr;
 630       if (c->as_jlong_bits() == 0)
 631         __ str(zr, frame_map()->address_for_slot(dest->double_stack_ix(),
 632                                                  lo_word_offset_in_bytes));
 633       else {
 634         __ mov(rscratch1, (intptr_t)c->as_jlong_bits());
 635         __ str(rscratch1, frame_map()->address_for_slot(dest->double_stack_ix(),
 636                                                         lo_word_offset_in_bytes));
 637       }
 638     }
 639     break;
 640   default:
 641     ShouldNotReachHere();
 642   }
 643 }
 644 
 645 void LIR_Assembler::const2mem(LIR_Opr src, LIR_Opr dest, BasicType type, CodeEmitInfo* info, bool wide) {
 646   assert(src->is_constant(), "should not call otherwise");
 647   LIR_Const* c = src->as_constant_ptr();
 648   LIR_Address* to_addr = dest->as_address_ptr();
 649 
 650   void (Assembler::* insn)(Register Rt, const Address &adr);
 651 
 652   switch (type) {
 653   case T_ADDRESS:
 654     assert(c->as_jint() == 0, "should be");
 655     insn = &Assembler::str;
 656     break;
 657   case T_LONG:
 658     assert(c->as_jlong() == 0, "should be");
 659     insn = &Assembler::str;
 660     break;
 661   case T_INT:
 662     assert(c->as_jint() == 0, "should be");
 663     insn = &Assembler::strw;
 664     break;
 665   case T_OBJECT:
 666   case T_ARRAY:
 667     assert(c->as_jobject() == 0, "should be");
 668     if (UseCompressedOops && !wide) {
 669       insn = &Assembler::strw;
 670     } else {
 671       insn = &Assembler::str;
 672     }
 673     break;
 674   case T_CHAR:
 675   case T_SHORT:
 676     assert(c->as_jint() == 0, "should be");
 677     insn = &Assembler::strh;
 678     break;
 679   case T_BOOLEAN:
 680   case T_BYTE:
 681     assert(c->as_jint() == 0, "should be");
 682     insn = &Assembler::strb;
 683     break;
 684   default:
 685     ShouldNotReachHere();
 686     insn = &Assembler::str;  // unreachable
 687   }
 688 
 689   if (info) add_debug_info_for_null_check_here(info);
 690   (_masm->*insn)(zr, as_Address(to_addr, rscratch1));
 691 }
 692 
 693 void LIR_Assembler::reg2reg(LIR_Opr src, LIR_Opr dest) {
 694   assert(src->is_register(), "should not call otherwise");
 695   assert(dest->is_register(), "should not call otherwise");
 696 
 697   // move between cpu-registers
 698   if (dest->is_single_cpu()) {
 699     if (src->type() == T_LONG) {
 700       // Can do LONG -> OBJECT
 701       move_regs(src->as_register_lo(), dest->as_register());
 702       return;
 703     }
 704     assert(src->is_single_cpu(), "must match");
 705     if (src->type() == T_OBJECT) {
 706       __ verify_oop(src->as_register());
 707     }
 708     move_regs(src->as_register(), dest->as_register());
 709 
 710   } else if (dest->is_double_cpu()) {
 711     if (is_reference_type(src->type())) {
 712       // Surprising to me but we can see move of a long to t_object
 713       __ verify_oop(src->as_register());
 714       move_regs(src->as_register(), dest->as_register_lo());
 715       return;
 716     }
 717     assert(src->is_double_cpu(), "must match");
 718     Register f_lo = src->as_register_lo();
 719     Register f_hi = src->as_register_hi();
 720     Register t_lo = dest->as_register_lo();
 721     Register t_hi = dest->as_register_hi();
 722     assert(f_hi == f_lo, "must be same");
 723     assert(t_hi == t_lo, "must be same");
 724     move_regs(f_lo, t_lo);
 725 
 726   } else if (dest->is_single_fpu()) {
 727     __ fmovs(dest->as_float_reg(), src->as_float_reg());
 728 
 729   } else if (dest->is_double_fpu()) {
 730     __ fmovd(dest->as_double_reg(), src->as_double_reg());
 731 
 732   } else {
 733     ShouldNotReachHere();
 734   }
 735 }
 736 
 737 void LIR_Assembler::reg2stack(LIR_Opr src, LIR_Opr dest, BasicType type, bool pop_fpu_stack) {
 738   if (src->is_single_cpu()) {
 739     if (is_reference_type(type)) {
 740       __ str(src->as_register(), frame_map()->address_for_slot(dest->single_stack_ix()));
 741       __ verify_oop(src->as_register());
 742     } else if (type == T_METADATA || type == T_DOUBLE || type == T_ADDRESS) {
 743       __ str(src->as_register(), frame_map()->address_for_slot(dest->single_stack_ix()));
 744     } else {
 745       __ strw(src->as_register(), frame_map()->address_for_slot(dest->single_stack_ix()));
 746     }
 747 
 748   } else if (src->is_double_cpu()) {
 749     Address dest_addr_LO = frame_map()->address_for_slot(dest->double_stack_ix(), lo_word_offset_in_bytes);
 750     __ str(src->as_register_lo(), dest_addr_LO);
 751 
 752   } else if (src->is_single_fpu()) {
 753     Address dest_addr = frame_map()->address_for_slot(dest->single_stack_ix());
 754     __ strs(src->as_float_reg(), dest_addr);
 755 
 756   } else if (src->is_double_fpu()) {
 757     Address dest_addr = frame_map()->address_for_slot(dest->double_stack_ix());
 758     __ strd(src->as_double_reg(), dest_addr);
 759 
 760   } else {
 761     ShouldNotReachHere();
 762   }
 763 
 764 }
 765 
 766 
 767 void LIR_Assembler::reg2mem(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_PatchCode patch_code, CodeEmitInfo* info, bool pop_fpu_stack, bool wide, bool /* unaligned */) {
 768   LIR_Address* to_addr = dest->as_address_ptr();
 769   PatchingStub* patch = NULL;
 770   Register compressed_src = rscratch1;
 771 
 772   if (patch_code != lir_patch_none) {
 773     deoptimize_trap(info);
 774     return;
 775   }
 776 
 777   if (is_reference_type(type)) {
 778     __ verify_oop(src->as_register());
 779 
 780     if (UseCompressedOops && !wide) {
 781       __ encode_heap_oop(compressed_src, src->as_register());
 782     } else {
 783       compressed_src = src->as_register();
 784     }
 785   }
 786 
 787   int null_check_here = code_offset();
 788   switch (type) {
 789     case T_FLOAT: {
 790       __ strs(src->as_float_reg(), as_Address(to_addr));
 791       break;
 792     }
 793 
 794     case T_DOUBLE: {
 795       __ strd(src->as_double_reg(), as_Address(to_addr));
 796       break;
 797     }
 798 
 799     case T_ARRAY:   // fall through
 800     case T_OBJECT:  // fall through
 801       if (UseCompressedOops && !wide) {
 802         __ strw(compressed_src, as_Address(to_addr, rscratch2));
 803       } else {
 804          __ str(compressed_src, as_Address(to_addr));
 805       }
 806       break;
 807     case T_METADATA:
 808       // We get here to store a method pointer to the stack to pass to
 809       // a dtrace runtime call. This can't work on 64 bit with
 810       // compressed klass ptrs: T_METADATA can be a compressed klass
 811       // ptr or a 64 bit method pointer.
 812       ShouldNotReachHere();
 813       __ str(src->as_register(), as_Address(to_addr));
 814       break;
 815     case T_ADDRESS:
 816       __ str(src->as_register(), as_Address(to_addr));
 817       break;
 818     case T_INT:
 819       __ strw(src->as_register(), as_Address(to_addr));
 820       break;
 821 
 822     case T_LONG: {
 823       __ str(src->as_register_lo(), as_Address_lo(to_addr));
 824       break;
 825     }
 826 
 827     case T_BYTE:    // fall through
 828     case T_BOOLEAN: {
 829       __ strb(src->as_register(), as_Address(to_addr));
 830       break;
 831     }
 832 
 833     case T_CHAR:    // fall through
 834     case T_SHORT:
 835       __ strh(src->as_register(), as_Address(to_addr));
 836       break;
 837 
 838     default:
 839       ShouldNotReachHere();
 840   }
 841   if (info != NULL) {
 842     add_debug_info_for_null_check(null_check_here, info);
 843   }
 844 }
 845 
 846 
 847 void LIR_Assembler::stack2reg(LIR_Opr src, LIR_Opr dest, BasicType type) {
 848   assert(src->is_stack(), "should not call otherwise");
 849   assert(dest->is_register(), "should not call otherwise");
 850 
 851   if (dest->is_single_cpu()) {
 852     if (is_reference_type(type)) {
 853       __ ldr(dest->as_register(), frame_map()->address_for_slot(src->single_stack_ix()));
 854       __ verify_oop(dest->as_register());
 855     } else if (type == T_METADATA || type == T_ADDRESS) {
 856       __ ldr(dest->as_register(), frame_map()->address_for_slot(src->single_stack_ix()));
 857     } else {
 858       __ ldrw(dest->as_register(), frame_map()->address_for_slot(src->single_stack_ix()));
 859     }
 860 
 861   } else if (dest->is_double_cpu()) {
 862     Address src_addr_LO = frame_map()->address_for_slot(src->double_stack_ix(), lo_word_offset_in_bytes);
 863     __ ldr(dest->as_register_lo(), src_addr_LO);
 864 
 865   } else if (dest->is_single_fpu()) {
 866     Address src_addr = frame_map()->address_for_slot(src->single_stack_ix());
 867     __ ldrs(dest->as_float_reg(), src_addr);
 868 
 869   } else if (dest->is_double_fpu()) {
 870     Address src_addr = frame_map()->address_for_slot(src->double_stack_ix());
 871     __ ldrd(dest->as_double_reg(), src_addr);
 872 
 873   } else {
 874     ShouldNotReachHere();
 875   }
 876 }
 877 
 878 
 879 void LIR_Assembler::klass2reg_with_patching(Register reg, CodeEmitInfo* info) {
 880   address target = NULL;
 881   relocInfo::relocType reloc_type = relocInfo::none;
 882 
 883   switch (patching_id(info)) {
 884   case PatchingStub::access_field_id:
 885     target = Runtime1::entry_for(Runtime1::access_field_patching_id);
 886     reloc_type = relocInfo::section_word_type;
 887     break;
 888   case PatchingStub::load_klass_id:
 889     target = Runtime1::entry_for(Runtime1::load_klass_patching_id);
 890     reloc_type = relocInfo::metadata_type;
 891     break;
 892   case PatchingStub::load_mirror_id:
 893     target = Runtime1::entry_for(Runtime1::load_mirror_patching_id);
 894     reloc_type = relocInfo::oop_type;
 895     break;
 896   case PatchingStub::load_appendix_id:
 897     target = Runtime1::entry_for(Runtime1::load_appendix_patching_id);
 898     reloc_type = relocInfo::oop_type;
 899     break;
 900   default: ShouldNotReachHere();
 901   }
 902 
 903   __ far_call(RuntimeAddress(target));
 904   add_call_info_here(info);
 905 }
 906 
 907 void LIR_Assembler::stack2stack(LIR_Opr src, LIR_Opr dest, BasicType type) {
 908 
 909   LIR_Opr temp;
 910   if (type == T_LONG || type == T_DOUBLE)
 911     temp = FrameMap::rscratch1_long_opr;
 912   else
 913     temp = FrameMap::rscratch1_opr;
 914 
 915   stack2reg(src, temp, src->type());
 916   reg2stack(temp, dest, dest->type(), false);
 917 }
 918 
 919 
 920 void LIR_Assembler::mem2reg(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_PatchCode patch_code, CodeEmitInfo* info, bool wide, bool /* unaligned */) {
 921   LIR_Address* addr = src->as_address_ptr();
 922   LIR_Address* from_addr = src->as_address_ptr();
 923 
 924   if (addr->base()->type() == T_OBJECT) {
 925     __ verify_oop(addr->base()->as_pointer_register());
 926   }
 927 
 928   if (patch_code != lir_patch_none) {
 929     deoptimize_trap(info);
 930     return;
 931   }
 932 
 933   if (info != NULL) {
 934     add_debug_info_for_null_check_here(info);
 935   }
 936   int null_check_here = code_offset();
 937   switch (type) {
 938     case T_FLOAT: {
 939       __ ldrs(dest->as_float_reg(), as_Address(from_addr));
 940       break;
 941     }
 942 
 943     case T_DOUBLE: {
 944       __ ldrd(dest->as_double_reg(), as_Address(from_addr));
 945       break;
 946     }
 947 
 948     case T_ARRAY:   // fall through
 949     case T_OBJECT:  // fall through
 950       if (UseCompressedOops && !wide) {
 951         __ ldrw(dest->as_register(), as_Address(from_addr));
 952       } else {
 953          __ ldr(dest->as_register(), as_Address(from_addr));
 954       }
 955       break;
 956     case T_METADATA:
 957       // We get here to store a method pointer to the stack to pass to
 958       // a dtrace runtime call. This can't work on 64 bit with
 959       // compressed klass ptrs: T_METADATA can be a compressed klass
 960       // ptr or a 64 bit method pointer.
 961       ShouldNotReachHere();
 962       __ ldr(dest->as_register(), as_Address(from_addr));
 963       break;
 964     case T_ADDRESS:
 965       // FIXME: OMG this is a horrible kludge.  Any offset from an
 966       // address that matches klass_offset_in_bytes() will be loaded
 967       // as a word, not a long.
 968       if (UseCompressedClassPointers && addr->disp() == oopDesc::klass_offset_in_bytes()) {
 969         __ ldrw(dest->as_register(), as_Address(from_addr));
 970       } else {
 971         __ ldr(dest->as_register(), as_Address(from_addr));
 972       }
 973       break;
 974     case T_INT:
 975       __ ldrw(dest->as_register(), as_Address(from_addr));
 976       break;
 977 
 978     case T_LONG: {
 979       __ ldr(dest->as_register_lo(), as_Address_lo(from_addr));
 980       break;
 981     }
 982 
 983     case T_BYTE:
 984       __ ldrsb(dest->as_register(), as_Address(from_addr));
 985       break;
 986     case T_BOOLEAN: {
 987       __ ldrb(dest->as_register(), as_Address(from_addr));
 988       break;
 989     }
 990 
 991     case T_CHAR:
 992       __ ldrh(dest->as_register(), as_Address(from_addr));
 993       break;
 994     case T_SHORT:
 995       __ ldrsh(dest->as_register(), as_Address(from_addr));
 996       break;
 997 
 998     default:
 999       ShouldNotReachHere();
1000   }
1001 
1002   if (is_reference_type(type)) {
1003     if (UseCompressedOops && !wide) {
1004       __ decode_heap_oop(dest->as_register());
1005     }
1006 
1007     if (!UseZGC) {
1008       // Load barrier has not yet been applied, so ZGC can't verify the oop here
1009       __ verify_oop(dest->as_register());
1010     }
1011   } else if (type == T_ADDRESS && addr->disp() == oopDesc::klass_offset_in_bytes()) {
1012     if (UseCompressedClassPointers) {
1013       __ decode_klass_not_null(dest->as_register());
1014     }
1015   }
1016 }
1017 
1018 
1019 int LIR_Assembler::array_element_size(BasicType type) const {
1020   int elem_size = type2aelembytes(type);
1021   return exact_log2(elem_size);
1022 }
1023 
1024 
1025 void LIR_Assembler::emit_op3(LIR_Op3* op) {
1026   switch (op->code()) {
1027   case lir_idiv:
1028   case lir_irem:
1029     arithmetic_idiv(op->code(),
1030                     op->in_opr1(),
1031                     op->in_opr2(),
1032                     op->in_opr3(),
1033                     op->result_opr(),
1034                     op->info());
1035     break;
1036   case lir_fmad:
1037     __ fmaddd(op->result_opr()->as_double_reg(),
1038               op->in_opr1()->as_double_reg(),
1039               op->in_opr2()->as_double_reg(),
1040               op->in_opr3()->as_double_reg());
1041     break;
1042   case lir_fmaf:
1043     __ fmadds(op->result_opr()->as_float_reg(),
1044               op->in_opr1()->as_float_reg(),
1045               op->in_opr2()->as_float_reg(),
1046               op->in_opr3()->as_float_reg());
1047     break;
1048   default:      ShouldNotReachHere(); break;
1049   }
1050 }
1051 
1052 void LIR_Assembler::emit_opBranch(LIR_OpBranch* op) {
1053 #ifdef ASSERT
1054   assert(op->block() == NULL || op->block()->label() == op->label(), "wrong label");
1055   if (op->block() != NULL)  _branch_target_blocks.append(op->block());
1056   if (op->ublock() != NULL) _branch_target_blocks.append(op->ublock());
1057 #endif
1058 
1059   if (op->cond() == lir_cond_always) {
1060     if (op->info() != NULL) add_debug_info_for_branch(op->info());
1061     __ b(*(op->label()));
1062   } else {
1063     Assembler::Condition acond;
1064     if (op->code() == lir_cond_float_branch) {
1065       bool is_unordered = (op->ublock() == op->block());
1066       // Assembler::EQ does not permit unordered branches, so we add
1067       // another branch here.  Likewise, Assembler::NE does not permit
1068       // ordered branches.
1069       if ((is_unordered && op->cond() == lir_cond_equal)
1070           || (!is_unordered && op->cond() == lir_cond_notEqual))
1071         __ br(Assembler::VS, *(op->ublock()->label()));
1072       switch(op->cond()) {
1073       case lir_cond_equal:        acond = Assembler::EQ; break;
1074       case lir_cond_notEqual:     acond = Assembler::NE; break;
1075       case lir_cond_less:         acond = (is_unordered ? Assembler::LT : Assembler::LO); break;
1076       case lir_cond_lessEqual:    acond = (is_unordered ? Assembler::LE : Assembler::LS); break;
1077       case lir_cond_greaterEqual: acond = (is_unordered ? Assembler::HS : Assembler::GE); break;
1078       case lir_cond_greater:      acond = (is_unordered ? Assembler::HI : Assembler::GT); break;
1079       default:                    ShouldNotReachHere();
1080         acond = Assembler::EQ;  // unreachable
1081       }
1082     } else {
1083       switch (op->cond()) {
1084         case lir_cond_equal:        acond = Assembler::EQ; break;
1085         case lir_cond_notEqual:     acond = Assembler::NE; break;
1086         case lir_cond_less:         acond = Assembler::LT; break;
1087         case lir_cond_lessEqual:    acond = Assembler::LE; break;
1088         case lir_cond_greaterEqual: acond = Assembler::GE; break;
1089         case lir_cond_greater:      acond = Assembler::GT; break;
1090         case lir_cond_belowEqual:   acond = Assembler::LS; break;
1091         case lir_cond_aboveEqual:   acond = Assembler::HS; break;
1092         default:                    ShouldNotReachHere();
1093           acond = Assembler::EQ;  // unreachable
1094       }
1095     }
1096     __ br(acond,*(op->label()));
1097   }
1098 }
1099 
1100 
1101 
1102 void LIR_Assembler::emit_opConvert(LIR_OpConvert* op) {
1103   LIR_Opr src  = op->in_opr();
1104   LIR_Opr dest = op->result_opr();
1105 
1106   switch (op->bytecode()) {
1107     case Bytecodes::_i2f:
1108       {
1109         __ scvtfws(dest->as_float_reg(), src->as_register());
1110         break;
1111       }
1112     case Bytecodes::_i2d:
1113       {
1114         __ scvtfwd(dest->as_double_reg(), src->as_register());
1115         break;
1116       }
1117     case Bytecodes::_l2d:
1118       {
1119         __ scvtfd(dest->as_double_reg(), src->as_register_lo());
1120         break;
1121       }
1122     case Bytecodes::_l2f:
1123       {
1124         __ scvtfs(dest->as_float_reg(), src->as_register_lo());
1125         break;
1126       }
1127     case Bytecodes::_f2d:
1128       {
1129         __ fcvts(dest->as_double_reg(), src->as_float_reg());
1130         break;
1131       }
1132     case Bytecodes::_d2f:
1133       {
1134         __ fcvtd(dest->as_float_reg(), src->as_double_reg());
1135         break;
1136       }
1137     case Bytecodes::_i2c:
1138       {
1139         __ ubfx(dest->as_register(), src->as_register(), 0, 16);
1140         break;
1141       }
1142     case Bytecodes::_i2l:
1143       {
1144         __ sxtw(dest->as_register_lo(), src->as_register());
1145         break;
1146       }
1147     case Bytecodes::_i2s:
1148       {
1149         __ sxth(dest->as_register(), src->as_register());
1150         break;
1151       }
1152     case Bytecodes::_i2b:
1153       {
1154         __ sxtb(dest->as_register(), src->as_register());
1155         break;
1156       }
1157     case Bytecodes::_l2i:
1158       {
1159         _masm->block_comment("FIXME: This could be a no-op");
1160         __ uxtw(dest->as_register(), src->as_register_lo());
1161         break;
1162       }
1163     case Bytecodes::_d2l:
1164       {
1165         __ fcvtzd(dest->as_register_lo(), src->as_double_reg());
1166         break;
1167       }
1168     case Bytecodes::_f2i:
1169       {
1170         __ fcvtzsw(dest->as_register(), src->as_float_reg());
1171         break;
1172       }
1173     case Bytecodes::_f2l:
1174       {
1175         __ fcvtzs(dest->as_register_lo(), src->as_float_reg());
1176         break;
1177       }
1178     case Bytecodes::_d2i:
1179       {
1180         __ fcvtzdw(dest->as_register(), src->as_double_reg());
1181         break;
1182       }
1183     default: ShouldNotReachHere();
1184   }
1185 }
1186 
1187 void LIR_Assembler::emit_alloc_obj(LIR_OpAllocObj* op) {
1188   if (op->init_check()) {
1189     __ ldrb(rscratch1, Address(op->klass()->as_register(),
1190                                InstanceKlass::init_state_offset()));
1191     __ cmpw(rscratch1, InstanceKlass::fully_initialized);
1192     add_debug_info_for_null_check_here(op->stub()->info());
1193     __ br(Assembler::NE, *op->stub()->entry());
1194   }
1195   __ allocate_object(op->obj()->as_register(),
1196                      op->tmp1()->as_register(),
1197                      op->tmp2()->as_register(),
1198                      op->header_size(),
1199                      op->object_size(),
1200                      op->klass()->as_register(),
1201                      *op->stub()->entry());
1202   __ bind(*op->stub()->continuation());
1203 }
1204 
1205 void LIR_Assembler::emit_alloc_array(LIR_OpAllocArray* op) {
1206   Register len =  op->len()->as_register();
1207   __ uxtw(len, len);
1208 
1209   if (UseSlowPath ||
1210       (!UseFastNewObjectArray && is_reference_type(op->type())) ||
1211       (!UseFastNewTypeArray   && !is_reference_type(op->type()))) {
1212     __ b(*op->stub()->entry());
1213   } else {
1214     Register tmp1 = op->tmp1()->as_register();
1215     Register tmp2 = op->tmp2()->as_register();
1216     Register tmp3 = op->tmp3()->as_register();
1217     if (len == tmp1) {
1218       tmp1 = tmp3;
1219     } else if (len == tmp2) {
1220       tmp2 = tmp3;
1221     } else if (len == tmp3) {
1222       // everything is ok
1223     } else {
1224       __ mov(tmp3, len);
1225     }
1226     __ allocate_array(op->obj()->as_register(),
1227                       len,
1228                       tmp1,
1229                       tmp2,
1230                       arrayOopDesc::header_size(op->type()),
1231                       array_element_size(op->type()),
1232                       op->klass()->as_register(),
1233                       *op->stub()->entry());
1234   }
1235   __ bind(*op->stub()->continuation());
1236 }
1237 
1238 void LIR_Assembler::type_profile_helper(Register mdo,
1239                                         ciMethodData *md, ciProfileData *data,
1240                                         Register recv, Label* update_done) {
1241   for (uint i = 0; i < ReceiverTypeData::row_limit(); i++) {
1242     Label next_test;
1243     // See if the receiver is receiver[n].
1244     __ lea(rscratch2, Address(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_offset(i))));
1245     __ ldr(rscratch1, Address(rscratch2));
1246     __ cmp(recv, rscratch1);
1247     __ br(Assembler::NE, next_test);
1248     Address data_addr(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_count_offset(i)));
1249     __ addptr(data_addr, DataLayout::counter_increment);
1250     __ b(*update_done);
1251     __ bind(next_test);
1252   }
1253 
1254   // Didn't find receiver; find next empty slot and fill it in
1255   for (uint i = 0; i < ReceiverTypeData::row_limit(); i++) {
1256     Label next_test;
1257     __ lea(rscratch2,
1258            Address(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_offset(i))));
1259     Address recv_addr(rscratch2);
1260     __ ldr(rscratch1, recv_addr);
1261     __ cbnz(rscratch1, next_test);
1262     __ str(recv, recv_addr);
1263     __ mov(rscratch1, DataLayout::counter_increment);
1264     __ lea(rscratch2, Address(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_count_offset(i))));
1265     __ str(rscratch1, Address(rscratch2));
1266     __ b(*update_done);
1267     __ bind(next_test);
1268   }
1269 }
1270 
1271 void LIR_Assembler::emit_typecheck_helper(LIR_OpTypeCheck *op, Label* success, Label* failure, Label* obj_is_null) {
1272   // we always need a stub for the failure case.
1273   CodeStub* stub = op->stub();
1274   Register obj = op->object()->as_register();
1275   Register k_RInfo = op->tmp1()->as_register();
1276   Register klass_RInfo = op->tmp2()->as_register();
1277   Register dst = op->result_opr()->as_register();
1278   ciKlass* k = op->klass();
1279   Register Rtmp1 = noreg;
1280 
1281   // check if it needs to be profiled
1282   ciMethodData* md;
1283   ciProfileData* data;
1284 
1285   const bool should_profile = op->should_profile();
1286 
1287   if (should_profile) {
1288     ciMethod* method = op->profiled_method();
1289     assert(method != NULL, "Should have method");
1290     int bci = op->profiled_bci();
1291     md = method->method_data_or_null();
1292     assert(md != NULL, "Sanity");
1293     data = md->bci_to_data(bci);
1294     assert(data != NULL,                "need data for type check");
1295     assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
1296   }
1297   Label profile_cast_success, profile_cast_failure;
1298   Label *success_target = should_profile ? &profile_cast_success : success;
1299   Label *failure_target = should_profile ? &profile_cast_failure : failure;
1300 
1301   if (obj == k_RInfo) {
1302     k_RInfo = dst;
1303   } else if (obj == klass_RInfo) {
1304     klass_RInfo = dst;
1305   }
1306   if (k->is_loaded() && !UseCompressedClassPointers) {
1307     select_different_registers(obj, dst, k_RInfo, klass_RInfo);
1308   } else {
1309     Rtmp1 = op->tmp3()->as_register();
1310     select_different_registers(obj, dst, k_RInfo, klass_RInfo, Rtmp1);
1311   }
1312 
1313   assert_different_registers(obj, k_RInfo, klass_RInfo);
1314 
1315     if (should_profile) {
1316       Label not_null;
1317       __ cbnz(obj, not_null);
1318       // Object is null; update MDO and exit
1319       Register mdo  = klass_RInfo;
1320       __ mov_metadata(mdo, md->constant_encoding());
1321       Address data_addr
1322         = __ form_address(rscratch2, mdo,
1323                           md->byte_offset_of_slot(data, DataLayout::flags_offset()),
1324                           0);
1325       __ ldrb(rscratch1, data_addr);
1326       __ orr(rscratch1, rscratch1, BitData::null_seen_byte_constant());
1327       __ strb(rscratch1, data_addr);
1328       __ b(*obj_is_null);
1329       __ bind(not_null);
1330     } else {
1331       __ cbz(obj, *obj_is_null);
1332     }
1333 
1334   if (!k->is_loaded()) {
1335     klass2reg_with_patching(k_RInfo, op->info_for_patch());
1336   } else {
1337     __ mov_metadata(k_RInfo, k->constant_encoding());
1338   }
1339   __ verify_oop(obj);
1340 
1341   if (op->fast_check()) {
1342     // get object class
1343     // not a safepoint as obj null check happens earlier
1344     __ load_klass(rscratch1, obj);
1345     __ cmp( rscratch1, k_RInfo);
1346 
1347     __ br(Assembler::NE, *failure_target);
1348     // successful cast, fall through to profile or jump
1349   } else {
1350     // get object class
1351     // not a safepoint as obj null check happens earlier
1352     __ load_klass(klass_RInfo, obj);
1353     if (k->is_loaded()) {
1354       // See if we get an immediate positive hit
1355       __ ldr(rscratch1, Address(klass_RInfo, int64_t(k->super_check_offset())));
1356       __ cmp(k_RInfo, rscratch1);
1357       if ((juint)in_bytes(Klass::secondary_super_cache_offset()) != k->super_check_offset()) {
1358         __ br(Assembler::NE, *failure_target);
1359         // successful cast, fall through to profile or jump
1360       } else {
1361         // See if we get an immediate positive hit
1362         __ br(Assembler::EQ, *success_target);
1363         // check for self
1364         __ cmp(klass_RInfo, k_RInfo);
1365         __ br(Assembler::EQ, *success_target);
1366 
1367         __ stp(klass_RInfo, k_RInfo, Address(__ pre(sp, -2 * wordSize)));
1368         __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
1369         __ ldr(klass_RInfo, Address(__ post(sp, 2 * wordSize)));
1370         // result is a boolean
1371         __ cbzw(klass_RInfo, *failure_target);
1372         // successful cast, fall through to profile or jump
1373       }
1374     } else {
1375       // perform the fast part of the checking logic
1376       __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, NULL);
1377       // call out-of-line instance of __ check_klass_subtype_slow_path(...):
1378       __ stp(klass_RInfo, k_RInfo, Address(__ pre(sp, -2 * wordSize)));
1379       __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
1380       __ ldp(k_RInfo, klass_RInfo, Address(__ post(sp, 2 * wordSize)));
1381       // result is a boolean
1382       __ cbz(k_RInfo, *failure_target);
1383       // successful cast, fall through to profile or jump
1384     }
1385   }
1386   if (should_profile) {
1387     Register mdo  = klass_RInfo, recv = k_RInfo;
1388     __ bind(profile_cast_success);
1389     __ mov_metadata(mdo, md->constant_encoding());
1390     __ load_klass(recv, obj);
1391     Label update_done;
1392     type_profile_helper(mdo, md, data, recv, success);
1393     __ b(*success);
1394 
1395     __ bind(profile_cast_failure);
1396     __ mov_metadata(mdo, md->constant_encoding());
1397     Address counter_addr
1398       = __ form_address(rscratch2, mdo,
1399                         md->byte_offset_of_slot(data, CounterData::count_offset()),
1400                         0);
1401     __ ldr(rscratch1, counter_addr);
1402     __ sub(rscratch1, rscratch1, DataLayout::counter_increment);
1403     __ str(rscratch1, counter_addr);
1404     __ b(*failure);
1405   }
1406   __ b(*success);
1407 }
1408 
1409 
1410 void LIR_Assembler::emit_opTypeCheck(LIR_OpTypeCheck* op) {
1411   const bool should_profile = op->should_profile();
1412 
1413   LIR_Code code = op->code();
1414   if (code == lir_store_check) {
1415     Register value = op->object()->as_register();
1416     Register array = op->array()->as_register();
1417     Register k_RInfo = op->tmp1()->as_register();
1418     Register klass_RInfo = op->tmp2()->as_register();
1419     Register Rtmp1 = op->tmp3()->as_register();
1420 
1421     CodeStub* stub = op->stub();
1422 
1423     // check if it needs to be profiled
1424     ciMethodData* md;
1425     ciProfileData* data;
1426 
1427     if (should_profile) {
1428       ciMethod* method = op->profiled_method();
1429       assert(method != NULL, "Should have method");
1430       int bci = op->profiled_bci();
1431       md = method->method_data_or_null();
1432       assert(md != NULL, "Sanity");
1433       data = md->bci_to_data(bci);
1434       assert(data != NULL,                "need data for type check");
1435       assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
1436     }
1437     Label profile_cast_success, profile_cast_failure, done;
1438     Label *success_target = should_profile ? &profile_cast_success : &done;
1439     Label *failure_target = should_profile ? &profile_cast_failure : stub->entry();
1440 
1441     if (should_profile) {
1442       Label not_null;
1443       __ cbnz(value, not_null);
1444       // Object is null; update MDO and exit
1445       Register mdo  = klass_RInfo;
1446       __ mov_metadata(mdo, md->constant_encoding());
1447       Address data_addr
1448         = __ form_address(rscratch2, mdo,
1449                           md->byte_offset_of_slot(data, DataLayout::flags_offset()),
1450                           0);
1451       __ ldrb(rscratch1, data_addr);
1452       __ orr(rscratch1, rscratch1, BitData::null_seen_byte_constant());
1453       __ strb(rscratch1, data_addr);
1454       __ b(done);
1455       __ bind(not_null);
1456     } else {
1457       __ cbz(value, done);
1458     }
1459 
1460     add_debug_info_for_null_check_here(op->info_for_exception());
1461     __ load_klass(k_RInfo, array);
1462     __ load_klass(klass_RInfo, value);
1463 
1464     // get instance klass (it's already uncompressed)
1465     __ ldr(k_RInfo, Address(k_RInfo, ObjArrayKlass::element_klass_offset()));
1466     // perform the fast part of the checking logic
1467     __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, NULL);
1468     // call out-of-line instance of __ check_klass_subtype_slow_path(...):
1469     __ stp(klass_RInfo, k_RInfo, Address(__ pre(sp, -2 * wordSize)));
1470     __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
1471     __ ldp(k_RInfo, klass_RInfo, Address(__ post(sp, 2 * wordSize)));
1472     // result is a boolean
1473     __ cbzw(k_RInfo, *failure_target);
1474     // fall through to the success case
1475 
1476     if (should_profile) {
1477       Register mdo  = klass_RInfo, recv = k_RInfo;
1478       __ bind(profile_cast_success);
1479       __ mov_metadata(mdo, md->constant_encoding());
1480       __ load_klass(recv, value);
1481       Label update_done;
1482       type_profile_helper(mdo, md, data, recv, &done);
1483       __ b(done);
1484 
1485       __ bind(profile_cast_failure);
1486       __ mov_metadata(mdo, md->constant_encoding());
1487       Address counter_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
1488       __ lea(rscratch2, counter_addr);
1489       __ ldr(rscratch1, Address(rscratch2));
1490       __ sub(rscratch1, rscratch1, DataLayout::counter_increment);
1491       __ str(rscratch1, Address(rscratch2));
1492       __ b(*stub->entry());
1493     }
1494 
1495     __ bind(done);
1496   } else if (code == lir_checkcast) {
1497     Register obj = op->object()->as_register();
1498     Register dst = op->result_opr()->as_register();
1499     Label success;
1500     emit_typecheck_helper(op, &success, op->stub()->entry(), &success);
1501     __ bind(success);
1502     if (dst != obj) {
1503       __ mov(dst, obj);
1504     }
1505   } else if (code == lir_instanceof) {
1506     Register obj = op->object()->as_register();
1507     Register dst = op->result_opr()->as_register();
1508     Label success, failure, done;
1509     emit_typecheck_helper(op, &success, &failure, &failure);
1510     __ bind(failure);
1511     __ mov(dst, zr);
1512     __ b(done);
1513     __ bind(success);
1514     __ mov(dst, 1);
1515     __ bind(done);
1516   } else {
1517     ShouldNotReachHere();
1518   }
1519 }
1520 
1521 void LIR_Assembler::casw(Register addr, Register newval, Register cmpval) {
1522   __ cmpxchg(addr, cmpval, newval, Assembler::word, /* acquire*/ true, /* release*/ true, /* weak*/ false, rscratch1);
1523   __ cset(rscratch1, Assembler::NE);
1524   __ membar(__ AnyAny);
1525 }
1526 
1527 void LIR_Assembler::casl(Register addr, Register newval, Register cmpval) {
1528   __ cmpxchg(addr, cmpval, newval, Assembler::xword, /* acquire*/ true, /* release*/ true, /* weak*/ false, rscratch1);
1529   __ cset(rscratch1, Assembler::NE);
1530   __ membar(__ AnyAny);
1531 }
1532 
1533 
1534 void LIR_Assembler::emit_compare_and_swap(LIR_OpCompareAndSwap* op) {
1535   assert(VM_Version::supports_cx8(), "wrong machine");
1536   Register addr;
1537   if (op->addr()->is_register()) {
1538     addr = as_reg(op->addr());
1539   } else {
1540     assert(op->addr()->is_address(), "what else?");
1541     LIR_Address* addr_ptr = op->addr()->as_address_ptr();
1542     assert(addr_ptr->disp() == 0, "need 0 disp");
1543     assert(addr_ptr->index() == LIR_OprDesc::illegalOpr(), "need 0 index");
1544     addr = as_reg(addr_ptr->base());
1545   }
1546   Register newval = as_reg(op->new_value());
1547   Register cmpval = as_reg(op->cmp_value());
1548 
1549   if (op->code() == lir_cas_obj) {
1550     if (UseCompressedOops) {
1551       Register t1 = op->tmp1()->as_register();
1552       assert(op->tmp1()->is_valid(), "must be");
1553       __ encode_heap_oop(t1, cmpval);
1554       cmpval = t1;
1555       __ encode_heap_oop(rscratch2, newval);
1556       newval = rscratch2;
1557       casw(addr, newval, cmpval);
1558     } else {
1559       casl(addr, newval, cmpval);
1560     }
1561   } else if (op->code() == lir_cas_int) {
1562     casw(addr, newval, cmpval);
1563   } else {
1564     casl(addr, newval, cmpval);
1565   }
1566 }
1567 
1568 
1569 void LIR_Assembler::cmove(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Opr result, BasicType type) {
1570 
1571   Assembler::Condition acond, ncond;
1572   switch (condition) {
1573   case lir_cond_equal:        acond = Assembler::EQ; ncond = Assembler::NE; break;
1574   case lir_cond_notEqual:     acond = Assembler::NE; ncond = Assembler::EQ; break;
1575   case lir_cond_less:         acond = Assembler::LT; ncond = Assembler::GE; break;
1576   case lir_cond_lessEqual:    acond = Assembler::LE; ncond = Assembler::GT; break;
1577   case lir_cond_greaterEqual: acond = Assembler::GE; ncond = Assembler::LT; break;
1578   case lir_cond_greater:      acond = Assembler::GT; ncond = Assembler::LE; break;
1579   case lir_cond_belowEqual:
1580   case lir_cond_aboveEqual:
1581   default:                    ShouldNotReachHere();
1582     acond = Assembler::EQ; ncond = Assembler::NE;  // unreachable
1583   }
1584 
1585   assert(result->is_single_cpu() || result->is_double_cpu(),
1586          "expect single register for result");
1587   if (opr1->is_constant() && opr2->is_constant()
1588       && opr1->type() == T_INT && opr2->type() == T_INT) {
1589     jint val1 = opr1->as_jint();
1590     jint val2 = opr2->as_jint();
1591     if (val1 == 0 && val2 == 1) {
1592       __ cset(result->as_register(), ncond);
1593       return;
1594     } else if (val1 == 1 && val2 == 0) {
1595       __ cset(result->as_register(), acond);
1596       return;
1597     }
1598   }
1599 
1600   if (opr1->is_constant() && opr2->is_constant()
1601       && opr1->type() == T_LONG && opr2->type() == T_LONG) {
1602     jlong val1 = opr1->as_jlong();
1603     jlong val2 = opr2->as_jlong();
1604     if (val1 == 0 && val2 == 1) {
1605       __ cset(result->as_register_lo(), ncond);
1606       return;
1607     } else if (val1 == 1 && val2 == 0) {
1608       __ cset(result->as_register_lo(), acond);
1609       return;
1610     }
1611   }
1612 
1613   if (opr1->is_stack()) {
1614     stack2reg(opr1, FrameMap::rscratch1_opr, result->type());
1615     opr1 = FrameMap::rscratch1_opr;
1616   } else if (opr1->is_constant()) {
1617     LIR_Opr tmp
1618       = opr1->type() == T_LONG ? FrameMap::rscratch1_long_opr : FrameMap::rscratch1_opr;
1619     const2reg(opr1, tmp, lir_patch_none, NULL);
1620     opr1 = tmp;
1621   }
1622 
1623   if (opr2->is_stack()) {
1624     stack2reg(opr2, FrameMap::rscratch2_opr, result->type());
1625     opr2 = FrameMap::rscratch2_opr;
1626   } else if (opr2->is_constant()) {
1627     LIR_Opr tmp
1628       = opr2->type() == T_LONG ? FrameMap::rscratch2_long_opr : FrameMap::rscratch2_opr;
1629     const2reg(opr2, tmp, lir_patch_none, NULL);
1630     opr2 = tmp;
1631   }
1632 
1633   if (result->type() == T_LONG)
1634     __ csel(result->as_register_lo(), opr1->as_register_lo(), opr2->as_register_lo(), acond);
1635   else
1636     __ csel(result->as_register(), opr1->as_register(), opr2->as_register(), acond);
1637 }
1638 
1639 void LIR_Assembler::arith_op(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dest, CodeEmitInfo* info, bool pop_fpu_stack) {
1640   assert(info == NULL, "should never be used, idiv/irem and ldiv/lrem not handled by this method");
1641 
1642   if (left->is_single_cpu()) {
1643     Register lreg = left->as_register();
1644     Register dreg = as_reg(dest);
1645 
1646     if (right->is_single_cpu()) {
1647       // cpu register - cpu register
1648 
1649       assert(left->type() == T_INT && right->type() == T_INT && dest->type() == T_INT,
1650              "should be");
1651       Register rreg = right->as_register();
1652       switch (code) {
1653       case lir_add: __ addw (dest->as_register(), lreg, rreg); break;
1654       case lir_sub: __ subw (dest->as_register(), lreg, rreg); break;
1655       case lir_mul: __ mulw (dest->as_register(), lreg, rreg); break;
1656       default:      ShouldNotReachHere();
1657       }
1658 
1659     } else if (right->is_double_cpu()) {
1660       Register rreg = right->as_register_lo();
1661       // single_cpu + double_cpu: can happen with obj+long
1662       assert(code == lir_add || code == lir_sub, "mismatched arithmetic op");
1663       switch (code) {
1664       case lir_add: __ add(dreg, lreg, rreg); break;
1665       case lir_sub: __ sub(dreg, lreg, rreg); break;
1666       default: ShouldNotReachHere();
1667       }
1668     } else if (right->is_constant()) {
1669       // cpu register - constant
1670       jlong c;
1671 
1672       // FIXME.  This is fugly: we really need to factor all this logic.
1673       switch(right->type()) {
1674       case T_LONG:
1675         c = right->as_constant_ptr()->as_jlong();
1676         break;
1677       case T_INT:
1678       case T_ADDRESS:
1679         c = right->as_constant_ptr()->as_jint();
1680         break;
1681       default:
1682         ShouldNotReachHere();
1683         c = 0;  // unreachable
1684         break;
1685       }
1686 
1687       assert(code == lir_add || code == lir_sub, "mismatched arithmetic op");
1688       if (c == 0 && dreg == lreg) {
1689         COMMENT("effective nop elided");
1690         return;
1691       }
1692       switch(left->type()) {
1693       case T_INT:
1694         switch (code) {
1695         case lir_add: __ addw(dreg, lreg, c); break;
1696         case lir_sub: __ subw(dreg, lreg, c); break;
1697         default: ShouldNotReachHere();
1698         }
1699         break;
1700       case T_OBJECT:
1701       case T_ADDRESS:
1702         switch (code) {
1703         case lir_add: __ add(dreg, lreg, c); break;
1704         case lir_sub: __ sub(dreg, lreg, c); break;
1705         default: ShouldNotReachHere();
1706         }
1707         break;
1708       default:
1709         ShouldNotReachHere();
1710       }
1711     } else {
1712       ShouldNotReachHere();
1713     }
1714 
1715   } else if (left->is_double_cpu()) {
1716     Register lreg_lo = left->as_register_lo();
1717 
1718     if (right->is_double_cpu()) {
1719       // cpu register - cpu register
1720       Register rreg_lo = right->as_register_lo();
1721       switch (code) {
1722       case lir_add: __ add (dest->as_register_lo(), lreg_lo, rreg_lo); break;
1723       case lir_sub: __ sub (dest->as_register_lo(), lreg_lo, rreg_lo); break;
1724       case lir_mul: __ mul (dest->as_register_lo(), lreg_lo, rreg_lo); break;
1725       case lir_div: __ corrected_idivq(dest->as_register_lo(), lreg_lo, rreg_lo, false, rscratch1); break;
1726       case lir_rem: __ corrected_idivq(dest->as_register_lo(), lreg_lo, rreg_lo, true, rscratch1); break;
1727       default:
1728         ShouldNotReachHere();
1729       }
1730 
1731     } else if (right->is_constant()) {
1732       jlong c = right->as_constant_ptr()->as_jlong();
1733       Register dreg = as_reg(dest);
1734       switch (code) {
1735         case lir_add:
1736         case lir_sub:
1737           if (c == 0 && dreg == lreg_lo) {
1738             COMMENT("effective nop elided");
1739             return;
1740           }
1741           code == lir_add ? __ add(dreg, lreg_lo, c) : __ sub(dreg, lreg_lo, c);
1742           break;
1743         case lir_div:
1744           assert(c > 0 && is_power_of_2(c), "divisor must be power-of-2 constant");
1745           if (c == 1) {
1746             // move lreg_lo to dreg if divisor is 1
1747             __ mov(dreg, lreg_lo);
1748           } else {
1749             unsigned int shift = exact_log2_long(c);
1750             // use rscratch1 as intermediate result register
1751             __ asr(rscratch1, lreg_lo, 63);
1752             __ add(rscratch1, lreg_lo, rscratch1, Assembler::LSR, 64 - shift);
1753             __ asr(dreg, rscratch1, shift);
1754           }
1755           break;
1756         case lir_rem:
1757           assert(c > 0 && is_power_of_2(c), "divisor must be power-of-2 constant");
1758           if (c == 1) {
1759             // move 0 to dreg if divisor is 1
1760             __ mov(dreg, zr);
1761           } else {
1762             // use rscratch1 as intermediate result register
1763             __ negs(rscratch1, lreg_lo);
1764             __ andr(dreg, lreg_lo, c - 1);
1765             __ andr(rscratch1, rscratch1, c - 1);
1766             __ csneg(dreg, dreg, rscratch1, Assembler::MI);
1767           }
1768           break;
1769         default:
1770           ShouldNotReachHere();
1771       }
1772     } else {
1773       ShouldNotReachHere();
1774     }
1775   } else if (left->is_single_fpu()) {
1776     assert(right->is_single_fpu(), "right hand side of float arithmetics needs to be float register");
1777     switch (code) {
1778     case lir_add: __ fadds (dest->as_float_reg(), left->as_float_reg(), right->as_float_reg()); break;
1779     case lir_sub: __ fsubs (dest->as_float_reg(), left->as_float_reg(), right->as_float_reg()); break;
1780     case lir_mul_strictfp: // fall through
1781     case lir_mul: __ fmuls (dest->as_float_reg(), left->as_float_reg(), right->as_float_reg()); break;
1782     case lir_div_strictfp: // fall through
1783     case lir_div: __ fdivs (dest->as_float_reg(), left->as_float_reg(), right->as_float_reg()); break;
1784     default:
1785       ShouldNotReachHere();
1786     }
1787   } else if (left->is_double_fpu()) {
1788     if (right->is_double_fpu()) {
1789       // fpu register - fpu register
1790       switch (code) {
1791       case lir_add: __ faddd (dest->as_double_reg(), left->as_double_reg(), right->as_double_reg()); break;
1792       case lir_sub: __ fsubd (dest->as_double_reg(), left->as_double_reg(), right->as_double_reg()); break;
1793       case lir_mul_strictfp: // fall through
1794       case lir_mul: __ fmuld (dest->as_double_reg(), left->as_double_reg(), right->as_double_reg()); break;
1795       case lir_div_strictfp: // fall through
1796       case lir_div: __ fdivd (dest->as_double_reg(), left->as_double_reg(), right->as_double_reg()); break;
1797       default:
1798         ShouldNotReachHere();
1799       }
1800     } else {
1801       if (right->is_constant()) {
1802         ShouldNotReachHere();
1803       }
1804       ShouldNotReachHere();
1805     }
1806   } else if (left->is_single_stack() || left->is_address()) {
1807     assert(left == dest, "left and dest must be equal");
1808     ShouldNotReachHere();
1809   } else {
1810     ShouldNotReachHere();
1811   }
1812 }
1813 
1814 void LIR_Assembler::arith_fpu_implementation(LIR_Code code, int left_index, int right_index, int dest_index, bool pop_fpu_stack) { Unimplemented(); }
1815 
1816 
1817 void LIR_Assembler::intrinsic_op(LIR_Code code, LIR_Opr value, LIR_Opr unused, LIR_Opr dest, LIR_Op* op) {
1818   switch(code) {
1819   case lir_abs : __ fabsd(dest->as_double_reg(), value->as_double_reg()); break;
1820   case lir_sqrt: __ fsqrtd(dest->as_double_reg(), value->as_double_reg()); break;
1821   default      : ShouldNotReachHere();
1822   }
1823 }
1824 
1825 void LIR_Assembler::logic_op(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dst) {
1826 
1827   assert(left->is_single_cpu() || left->is_double_cpu(), "expect single or double register");
1828   Register Rleft = left->is_single_cpu() ? left->as_register() :
1829                                            left->as_register_lo();
1830    if (dst->is_single_cpu()) {
1831      Register Rdst = dst->as_register();
1832      if (right->is_constant()) {
1833        switch (code) {
1834          case lir_logic_and: __ andw (Rdst, Rleft, right->as_jint()); break;
1835          case lir_logic_or:  __ orrw (Rdst, Rleft, right->as_jint()); break;
1836          case lir_logic_xor: __ eorw (Rdst, Rleft, right->as_jint()); break;
1837          default: ShouldNotReachHere(); break;
1838        }
1839      } else {
1840        Register Rright = right->is_single_cpu() ? right->as_register() :
1841                                                   right->as_register_lo();
1842        switch (code) {
1843          case lir_logic_and: __ andw (Rdst, Rleft, Rright); break;
1844          case lir_logic_or:  __ orrw (Rdst, Rleft, Rright); break;
1845          case lir_logic_xor: __ eorw (Rdst, Rleft, Rright); break;
1846          default: ShouldNotReachHere(); break;
1847        }
1848      }
1849    } else {
1850      Register Rdst = dst->as_register_lo();
1851      if (right->is_constant()) {
1852        switch (code) {
1853          case lir_logic_and: __ andr (Rdst, Rleft, right->as_jlong()); break;
1854          case lir_logic_or:  __ orr (Rdst, Rleft, right->as_jlong()); break;
1855          case lir_logic_xor: __ eor (Rdst, Rleft, right->as_jlong()); break;
1856          default: ShouldNotReachHere(); break;
1857        }
1858      } else {
1859        Register Rright = right->is_single_cpu() ? right->as_register() :
1860                                                   right->as_register_lo();
1861        switch (code) {
1862          case lir_logic_and: __ andr (Rdst, Rleft, Rright); break;
1863          case lir_logic_or:  __ orr (Rdst, Rleft, Rright); break;
1864          case lir_logic_xor: __ eor (Rdst, Rleft, Rright); break;
1865          default: ShouldNotReachHere(); break;
1866        }
1867      }
1868    }
1869 }
1870 
1871 
1872 
1873 void LIR_Assembler::arithmetic_idiv(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr illegal, LIR_Opr result, CodeEmitInfo* info) {
1874 
1875   // opcode check
1876   assert((code == lir_idiv) || (code == lir_irem), "opcode must be idiv or irem");
1877   bool is_irem = (code == lir_irem);
1878 
1879   // operand check
1880   assert(left->is_single_cpu(),   "left must be register");
1881   assert(right->is_single_cpu() || right->is_constant(),  "right must be register or constant");
1882   assert(result->is_single_cpu(), "result must be register");
1883   Register lreg = left->as_register();
1884   Register dreg = result->as_register();
1885 
1886   // power-of-2 constant check and codegen
1887   if (right->is_constant()) {
1888     int c = right->as_constant_ptr()->as_jint();
1889     assert(c > 0 && is_power_of_2(c), "divisor must be power-of-2 constant");
1890     if (is_irem) {
1891       if (c == 1) {
1892         // move 0 to dreg if divisor is 1
1893         __ movw(dreg, zr);
1894       } else {
1895         // use rscratch1 as intermediate result register
1896         __ negsw(rscratch1, lreg);
1897         __ andw(dreg, lreg, c - 1);
1898         __ andw(rscratch1, rscratch1, c - 1);
1899         __ csnegw(dreg, dreg, rscratch1, Assembler::MI);
1900       }
1901     } else {
1902       if (c == 1) {
1903         // move lreg to dreg if divisor is 1
1904         __ movw(dreg, lreg);
1905       } else {
1906         unsigned int shift = exact_log2(c);
1907         // use rscratch1 as intermediate result register
1908         __ asrw(rscratch1, lreg, 31);
1909         __ addw(rscratch1, lreg, rscratch1, Assembler::LSR, 32 - shift);
1910         __ asrw(dreg, rscratch1, shift);
1911       }
1912     }
1913   } else {
1914     Register rreg = right->as_register();
1915     __ corrected_idivl(dreg, lreg, rreg, is_irem, rscratch1);
1916   }
1917 }
1918 
1919 
1920 void LIR_Assembler::comp_op(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Op2* op) {
1921   if (opr1->is_constant() && opr2->is_single_cpu()) {
1922     // tableswitch
1923     Register reg = as_reg(opr2);
1924     struct tableswitch &table = switches[opr1->as_constant_ptr()->as_jint()];
1925     __ tableswitch(reg, table._first_key, table._last_key, table._branches, table._after);
1926   } else if (opr1->is_single_cpu() || opr1->is_double_cpu()) {
1927     Register reg1 = as_reg(opr1);
1928     if (opr2->is_single_cpu()) {
1929       // cpu register - cpu register
1930       Register reg2 = opr2->as_register();
1931       if (is_reference_type(opr1->type())) {
1932         __ cmpoop(reg1, reg2);
1933       } else {
1934         assert(!is_reference_type(opr2->type()), "cmp int, oop?");
1935         __ cmpw(reg1, reg2);
1936       }
1937       return;
1938     }
1939     if (opr2->is_double_cpu()) {
1940       // cpu register - cpu register
1941       Register reg2 = opr2->as_register_lo();
1942       __ cmp(reg1, reg2);
1943       return;
1944     }
1945 
1946     if (opr2->is_constant()) {
1947       bool is_32bit = false; // width of register operand
1948       jlong imm;
1949 
1950       switch(opr2->type()) {
1951       case T_INT:
1952         imm = opr2->as_constant_ptr()->as_jint();
1953         is_32bit = true;
1954         break;
1955       case T_LONG:
1956         imm = opr2->as_constant_ptr()->as_jlong();
1957         break;
1958       case T_ADDRESS:
1959         imm = opr2->as_constant_ptr()->as_jint();
1960         break;
1961       case T_METADATA:
1962         imm = (intptr_t)(opr2->as_constant_ptr()->as_metadata());
1963         break;
1964       case T_OBJECT:
1965       case T_ARRAY:
1966         jobject2reg(opr2->as_constant_ptr()->as_jobject(), rscratch1);
1967         __ cmpoop(reg1, rscratch1);
1968         return;
1969       default:
1970         ShouldNotReachHere();
1971         imm = 0;  // unreachable
1972         break;
1973       }
1974 
1975       if (Assembler::operand_valid_for_add_sub_immediate(imm)) {
1976         if (is_32bit)
1977           __ cmpw(reg1, imm);
1978         else
1979           __ subs(zr, reg1, imm);
1980         return;
1981       } else {
1982         __ mov(rscratch1, imm);
1983         if (is_32bit)
1984           __ cmpw(reg1, rscratch1);
1985         else
1986           __ cmp(reg1, rscratch1);
1987         return;
1988       }
1989     } else
1990       ShouldNotReachHere();
1991   } else if (opr1->is_single_fpu()) {
1992     FloatRegister reg1 = opr1->as_float_reg();
1993     assert(opr2->is_single_fpu(), "expect single float register");
1994     FloatRegister reg2 = opr2->as_float_reg();
1995     __ fcmps(reg1, reg2);
1996   } else if (opr1->is_double_fpu()) {
1997     FloatRegister reg1 = opr1->as_double_reg();
1998     assert(opr2->is_double_fpu(), "expect double float register");
1999     FloatRegister reg2 = opr2->as_double_reg();
2000     __ fcmpd(reg1, reg2);
2001   } else {
2002     ShouldNotReachHere();
2003   }
2004 }
2005 
2006 void LIR_Assembler::comp_fl2i(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dst, LIR_Op2* op){
2007   if (code == lir_cmp_fd2i || code == lir_ucmp_fd2i) {
2008     bool is_unordered_less = (code == lir_ucmp_fd2i);
2009     if (left->is_single_fpu()) {
2010       __ float_cmp(true, is_unordered_less ? -1 : 1, left->as_float_reg(), right->as_float_reg(), dst->as_register());
2011     } else if (left->is_double_fpu()) {
2012       __ float_cmp(false, is_unordered_less ? -1 : 1, left->as_double_reg(), right->as_double_reg(), dst->as_register());
2013     } else {
2014       ShouldNotReachHere();
2015     }
2016   } else if (code == lir_cmp_l2i) {
2017     Label done;
2018     __ cmp(left->as_register_lo(), right->as_register_lo());
2019     __ mov(dst->as_register(), (uint64_t)-1L);
2020     __ br(Assembler::LT, done);
2021     __ csinc(dst->as_register(), zr, zr, Assembler::EQ);
2022     __ bind(done);
2023   } else {
2024     ShouldNotReachHere();
2025   }
2026 }
2027 
2028 
2029 void LIR_Assembler::align_call(LIR_Code code) {  }
2030 
2031 
2032 void LIR_Assembler::call(LIR_OpJavaCall* op, relocInfo::relocType rtype) {
2033   address call = __ trampoline_call(Address(op->addr(), rtype));
2034   if (call == NULL) {
2035     bailout("trampoline stub overflow");
2036     return;
2037   }
2038   add_call_info(code_offset(), op->info());
2039 }
2040 
2041 
2042 void LIR_Assembler::ic_call(LIR_OpJavaCall* op) {
2043   address call = __ ic_call(op->addr());
2044   if (call == NULL) {
2045     bailout("trampoline stub overflow");
2046     return;
2047   }
2048   add_call_info(code_offset(), op->info());
2049 }
2050 
2051 
2052 /* Currently, vtable-dispatch is only enabled for sparc platforms */
2053 void LIR_Assembler::vtable_call(LIR_OpJavaCall* op) {
2054   ShouldNotReachHere();
2055 }
2056 
2057 
2058 void LIR_Assembler::emit_static_call_stub() {
2059   address call_pc = __ pc();
2060   address stub = __ start_a_stub(call_stub_size());
2061   if (stub == NULL) {
2062     bailout("static call stub overflow");
2063     return;
2064   }
2065 
2066   int start = __ offset();
2067 
2068   __ relocate(static_stub_Relocation::spec(call_pc));
2069   __ emit_static_call_stub();
2070 
2071   assert(__ offset() - start + CompiledStaticCall::to_trampoline_stub_size()
2072         <= call_stub_size(), "stub too big");
2073   __ end_a_stub();
2074 }
2075 
2076 
2077 void LIR_Assembler::throw_op(LIR_Opr exceptionPC, LIR_Opr exceptionOop, CodeEmitInfo* info) {
2078   assert(exceptionOop->as_register() == r0, "must match");
2079   assert(exceptionPC->as_register() == r3, "must match");
2080 
2081   // exception object is not added to oop map by LinearScan
2082   // (LinearScan assumes that no oops are in fixed registers)
2083   info->add_register_oop(exceptionOop);
2084   Runtime1::StubID unwind_id;
2085 
2086   // get current pc information
2087   // pc is only needed if the method has an exception handler, the unwind code does not need it.
2088   int pc_for_athrow_offset = __ offset();
2089   InternalAddress pc_for_athrow(__ pc());
2090   __ adr(exceptionPC->as_register(), pc_for_athrow);
2091   add_call_info(pc_for_athrow_offset, info); // for exception handler
2092 
2093   __ verify_not_null_oop(r0);
2094   // search an exception handler (r0: exception oop, r3: throwing pc)
2095   if (compilation()->has_fpu_code()) {
2096     unwind_id = Runtime1::handle_exception_id;
2097   } else {
2098     unwind_id = Runtime1::handle_exception_nofpu_id;
2099   }
2100   __ far_call(RuntimeAddress(Runtime1::entry_for(unwind_id)));
2101 
2102   // FIXME: enough room for two byte trap   ????
2103   __ nop();
2104 }
2105 
2106 
2107 void LIR_Assembler::unwind_op(LIR_Opr exceptionOop) {
2108   assert(exceptionOop->as_register() == r0, "must match");
2109 
2110   __ b(_unwind_handler_entry);
2111 }
2112 
2113 
2114 void LIR_Assembler::shift_op(LIR_Code code, LIR_Opr left, LIR_Opr count, LIR_Opr dest, LIR_Opr tmp) {
2115   Register lreg = left->is_single_cpu() ? left->as_register() : left->as_register_lo();
2116   Register dreg = dest->is_single_cpu() ? dest->as_register() : dest->as_register_lo();
2117 
2118   switch (left->type()) {
2119     case T_INT: {
2120       switch (code) {
2121       case lir_shl:  __ lslvw (dreg, lreg, count->as_register()); break;
2122       case lir_shr:  __ asrvw (dreg, lreg, count->as_register()); break;
2123       case lir_ushr: __ lsrvw (dreg, lreg, count->as_register()); break;
2124       default:
2125         ShouldNotReachHere();
2126         break;
2127       }
2128       break;
2129     case T_LONG:
2130     case T_ADDRESS:
2131     case T_OBJECT:
2132       switch (code) {
2133       case lir_shl:  __ lslv (dreg, lreg, count->as_register()); break;
2134       case lir_shr:  __ asrv (dreg, lreg, count->as_register()); break;
2135       case lir_ushr: __ lsrv (dreg, lreg, count->as_register()); break;
2136       default:
2137         ShouldNotReachHere();
2138         break;
2139       }
2140       break;
2141     default:
2142       ShouldNotReachHere();
2143       break;
2144     }
2145   }
2146 }
2147 
2148 
2149 void LIR_Assembler::shift_op(LIR_Code code, LIR_Opr left, jint count, LIR_Opr dest) {
2150   Register dreg = dest->is_single_cpu() ? dest->as_register() : dest->as_register_lo();
2151   Register lreg = left->is_single_cpu() ? left->as_register() : left->as_register_lo();
2152 
2153   switch (left->type()) {
2154     case T_INT: {
2155       switch (code) {
2156       case lir_shl:  __ lslw (dreg, lreg, count); break;
2157       case lir_shr:  __ asrw (dreg, lreg, count); break;
2158       case lir_ushr: __ lsrw (dreg, lreg, count); break;
2159       default:
2160         ShouldNotReachHere();
2161         break;
2162       }
2163       break;
2164     case T_LONG:
2165     case T_ADDRESS:
2166     case T_OBJECT:
2167       switch (code) {
2168       case lir_shl:  __ lsl (dreg, lreg, count); break;
2169       case lir_shr:  __ asr (dreg, lreg, count); break;
2170       case lir_ushr: __ lsr (dreg, lreg, count); break;
2171       default:
2172         ShouldNotReachHere();
2173         break;
2174       }
2175       break;
2176     default:
2177       ShouldNotReachHere();
2178       break;
2179     }
2180   }
2181 }
2182 
2183 
2184 void LIR_Assembler::store_parameter(Register r, int offset_from_rsp_in_words) {
2185   assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2186   int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2187   assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2188   __ str (r, Address(sp, offset_from_rsp_in_bytes));
2189 }
2190 
2191 
2192 void LIR_Assembler::store_parameter(jint c,     int offset_from_rsp_in_words) {
2193   assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2194   int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2195   assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2196   __ mov (rscratch1, c);
2197   __ str (rscratch1, Address(sp, offset_from_rsp_in_bytes));
2198 }
2199 
2200 
2201 void LIR_Assembler::store_parameter(jobject o,  int offset_from_rsp_in_words) {
2202   ShouldNotReachHere();
2203   assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2204   int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2205   assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2206   __ lea(rscratch1, __ constant_oop_address(o));
2207   __ str(rscratch1, Address(sp, offset_from_rsp_in_bytes));
2208 }
2209 
2210 
2211 // This code replaces a call to arraycopy; no exception may
2212 // be thrown in this code, they must be thrown in the System.arraycopy
2213 // activation frame; we could save some checks if this would not be the case
2214 void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) {
2215   ciArrayKlass* default_type = op->expected_type();
2216   Register src = op->src()->as_register();
2217   Register dst = op->dst()->as_register();
2218   Register src_pos = op->src_pos()->as_register();
2219   Register dst_pos = op->dst_pos()->as_register();
2220   Register length  = op->length()->as_register();
2221   Register tmp = op->tmp()->as_register();
2222 
2223   __ resolve(ACCESS_READ, src);
2224   __ resolve(ACCESS_WRITE, dst);
2225 
2226   CodeStub* stub = op->stub();
2227   int flags = op->flags();
2228   BasicType basic_type = default_type != NULL ? default_type->element_type()->basic_type() : T_ILLEGAL;
2229   if (is_reference_type(basic_type)) basic_type = T_OBJECT;
2230 
2231   // if we don't know anything, just go through the generic arraycopy
2232   if (default_type == NULL // || basic_type == T_OBJECT
2233       ) {
2234     Label done;
2235     assert(src == r1 && src_pos == r2, "mismatch in calling convention");
2236 
2237     // Save the arguments in case the generic arraycopy fails and we
2238     // have to fall back to the JNI stub
2239     __ stp(dst,     dst_pos, Address(sp, 0*BytesPerWord));
2240     __ stp(length,  src_pos, Address(sp, 2*BytesPerWord));
2241     __ str(src,              Address(sp, 4*BytesPerWord));
2242 
2243     address copyfunc_addr = StubRoutines::generic_arraycopy();
2244     assert(copyfunc_addr != NULL, "generic arraycopy stub required");
2245 
2246     // The arguments are in java calling convention so we shift them
2247     // to C convention
2248     assert_different_registers(c_rarg0, j_rarg1, j_rarg2, j_rarg3, j_rarg4);
2249     __ mov(c_rarg0, j_rarg0);
2250     assert_different_registers(c_rarg1, j_rarg2, j_rarg3, j_rarg4);
2251     __ mov(c_rarg1, j_rarg1);
2252     assert_different_registers(c_rarg2, j_rarg3, j_rarg4);
2253     __ mov(c_rarg2, j_rarg2);
2254     assert_different_registers(c_rarg3, j_rarg4);
2255     __ mov(c_rarg3, j_rarg3);
2256     __ mov(c_rarg4, j_rarg4);
2257 #ifndef PRODUCT
2258     if (PrintC1Statistics) {
2259       __ incrementw(ExternalAddress((address)&Runtime1::_generic_arraycopystub_cnt));
2260     }
2261 #endif
2262     __ far_call(RuntimeAddress(copyfunc_addr));
2263 
2264     __ cbz(r0, *stub->continuation());
2265 
2266     // Reload values from the stack so they are where the stub
2267     // expects them.
2268     __ ldp(dst,     dst_pos, Address(sp, 0*BytesPerWord));
2269     __ ldp(length,  src_pos, Address(sp, 2*BytesPerWord));
2270     __ ldr(src,              Address(sp, 4*BytesPerWord));
2271 
2272     // r0 is -1^K where K == partial copied count
2273     __ eonw(rscratch1, r0, zr);
2274     // adjust length down and src/end pos up by partial copied count
2275     __ subw(length, length, rscratch1);
2276     __ addw(src_pos, src_pos, rscratch1);
2277     __ addw(dst_pos, dst_pos, rscratch1);
2278     __ b(*stub->entry());
2279 
2280     __ bind(*stub->continuation());
2281     return;
2282   }
2283 
2284   assert(default_type != NULL && default_type->is_array_klass() && default_type->is_loaded(), "must be true at this point");
2285 
2286   int elem_size = type2aelembytes(basic_type);
2287   int shift_amount;
2288   int scale = exact_log2(elem_size);
2289 
2290   Address src_length_addr = Address(src, arrayOopDesc::length_offset_in_bytes());
2291   Address dst_length_addr = Address(dst, arrayOopDesc::length_offset_in_bytes());
2292   Address src_klass_addr = Address(src, oopDesc::klass_offset_in_bytes());
2293   Address dst_klass_addr = Address(dst, oopDesc::klass_offset_in_bytes());
2294 
2295   // test for NULL
2296   if (flags & LIR_OpArrayCopy::src_null_check) {
2297     __ cbz(src, *stub->entry());
2298   }
2299   if (flags & LIR_OpArrayCopy::dst_null_check) {
2300     __ cbz(dst, *stub->entry());
2301   }
2302 
2303   // If the compiler was not able to prove that exact type of the source or the destination
2304   // of the arraycopy is an array type, check at runtime if the source or the destination is
2305   // an instance type.
2306   if (flags & LIR_OpArrayCopy::type_check) {
2307     if (!(flags & LIR_OpArrayCopy::LIR_OpArrayCopy::dst_objarray)) {
2308       __ load_klass(tmp, dst);
2309       __ ldrw(rscratch1, Address(tmp, in_bytes(Klass::layout_helper_offset())));
2310       __ cmpw(rscratch1, Klass::_lh_neutral_value);
2311       __ br(Assembler::GE, *stub->entry());
2312     }
2313 
2314     if (!(flags & LIR_OpArrayCopy::LIR_OpArrayCopy::src_objarray)) {
2315       __ load_klass(tmp, src);
2316       __ ldrw(rscratch1, Address(tmp, in_bytes(Klass::layout_helper_offset())));
2317       __ cmpw(rscratch1, Klass::_lh_neutral_value);
2318       __ br(Assembler::GE, *stub->entry());
2319     }
2320   }
2321 
2322   // check if negative
2323   if (flags & LIR_OpArrayCopy::src_pos_positive_check) {
2324     __ cmpw(src_pos, 0);
2325     __ br(Assembler::LT, *stub->entry());
2326   }
2327   if (flags & LIR_OpArrayCopy::dst_pos_positive_check) {
2328     __ cmpw(dst_pos, 0);
2329     __ br(Assembler::LT, *stub->entry());
2330   }
2331 
2332   if (flags & LIR_OpArrayCopy::length_positive_check) {
2333     __ cmpw(length, 0);
2334     __ br(Assembler::LT, *stub->entry());
2335   }
2336 
2337   if (flags & LIR_OpArrayCopy::src_range_check) {
2338     __ addw(tmp, src_pos, length);
2339     __ ldrw(rscratch1, src_length_addr);
2340     __ cmpw(tmp, rscratch1);
2341     __ br(Assembler::HI, *stub->entry());
2342   }
2343   if (flags & LIR_OpArrayCopy::dst_range_check) {
2344     __ addw(tmp, dst_pos, length);
2345     __ ldrw(rscratch1, dst_length_addr);
2346     __ cmpw(tmp, rscratch1);
2347     __ br(Assembler::HI, *stub->entry());
2348   }
2349 
2350   if (flags & LIR_OpArrayCopy::type_check) {
2351     // We don't know the array types are compatible
2352     if (basic_type != T_OBJECT) {
2353       // Simple test for basic type arrays
2354       if (UseCompressedClassPointers) {
2355         __ ldrw(tmp, src_klass_addr);
2356         __ ldrw(rscratch1, dst_klass_addr);
2357         __ cmpw(tmp, rscratch1);
2358       } else {
2359         __ ldr(tmp, src_klass_addr);
2360         __ ldr(rscratch1, dst_klass_addr);
2361         __ cmp(tmp, rscratch1);
2362       }
2363       __ br(Assembler::NE, *stub->entry());
2364     } else {
2365       // For object arrays, if src is a sub class of dst then we can
2366       // safely do the copy.
2367       Label cont, slow;
2368 
2369 #define PUSH(r1, r2)                                    \
2370       stp(r1, r2, __ pre(sp, -2 * wordSize));
2371 
2372 #define POP(r1, r2)                                     \
2373       ldp(r1, r2, __ post(sp, 2 * wordSize));
2374 
2375       __ PUSH(src, dst);
2376 
2377       __ load_klass(src, src);
2378       __ load_klass(dst, dst);
2379 
2380       __ check_klass_subtype_fast_path(src, dst, tmp, &cont, &slow, NULL);
2381 
2382       __ PUSH(src, dst);
2383       __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
2384       __ POP(src, dst);
2385 
2386       __ cbnz(src, cont);
2387 
2388       __ bind(slow);
2389       __ POP(src, dst);
2390 
2391       address copyfunc_addr = StubRoutines::checkcast_arraycopy();
2392       if (copyfunc_addr != NULL) { // use stub if available
2393         // src is not a sub class of dst so we have to do a
2394         // per-element check.
2395 
2396         int mask = LIR_OpArrayCopy::src_objarray|LIR_OpArrayCopy::dst_objarray;
2397         if ((flags & mask) != mask) {
2398           // Check that at least both of them object arrays.
2399           assert(flags & mask, "one of the two should be known to be an object array");
2400 
2401           if (!(flags & LIR_OpArrayCopy::src_objarray)) {
2402             __ load_klass(tmp, src);
2403           } else if (!(flags & LIR_OpArrayCopy::dst_objarray)) {
2404             __ load_klass(tmp, dst);
2405           }
2406           int lh_offset = in_bytes(Klass::layout_helper_offset());
2407           Address klass_lh_addr(tmp, lh_offset);
2408           jint objArray_lh = Klass::array_layout_helper(T_OBJECT);
2409           __ ldrw(rscratch1, klass_lh_addr);
2410           __ mov(rscratch2, objArray_lh);
2411           __ eorw(rscratch1, rscratch1, rscratch2);
2412           __ cbnzw(rscratch1, *stub->entry());
2413         }
2414 
2415        // Spill because stubs can use any register they like and it's
2416        // easier to restore just those that we care about.
2417         __ stp(dst,     dst_pos, Address(sp, 0*BytesPerWord));
2418         __ stp(length,  src_pos, Address(sp, 2*BytesPerWord));
2419         __ str(src,              Address(sp, 4*BytesPerWord));
2420 
2421         __ lea(c_rarg0, Address(src, src_pos, Address::uxtw(scale)));
2422         __ add(c_rarg0, c_rarg0, arrayOopDesc::base_offset_in_bytes(basic_type));
2423         assert_different_registers(c_rarg0, dst, dst_pos, length);
2424         __ lea(c_rarg1, Address(dst, dst_pos, Address::uxtw(scale)));
2425         __ add(c_rarg1, c_rarg1, arrayOopDesc::base_offset_in_bytes(basic_type));
2426         assert_different_registers(c_rarg1, dst, length);
2427         __ uxtw(c_rarg2, length);
2428         assert_different_registers(c_rarg2, dst);
2429 
2430         __ load_klass(c_rarg4, dst);
2431         __ ldr(c_rarg4, Address(c_rarg4, ObjArrayKlass::element_klass_offset()));
2432         __ ldrw(c_rarg3, Address(c_rarg4, Klass::super_check_offset_offset()));
2433         __ far_call(RuntimeAddress(copyfunc_addr));
2434 
2435 #ifndef PRODUCT
2436         if (PrintC1Statistics) {
2437           Label failed;
2438           __ cbnz(r0, failed);
2439           __ incrementw(ExternalAddress((address)&Runtime1::_arraycopy_checkcast_cnt));
2440           __ bind(failed);
2441         }
2442 #endif
2443 
2444         __ cbz(r0, *stub->continuation());
2445 
2446 #ifndef PRODUCT
2447         if (PrintC1Statistics) {
2448           __ incrementw(ExternalAddress((address)&Runtime1::_arraycopy_checkcast_attempt_cnt));
2449         }
2450 #endif
2451         assert_different_registers(dst, dst_pos, length, src_pos, src, r0, rscratch1);
2452 
2453         // Restore previously spilled arguments
2454         __ ldp(dst,     dst_pos, Address(sp, 0*BytesPerWord));
2455         __ ldp(length,  src_pos, Address(sp, 2*BytesPerWord));
2456         __ ldr(src,              Address(sp, 4*BytesPerWord));
2457 
2458         // return value is -1^K where K is partial copied count
2459         __ eonw(rscratch1, r0, zr);
2460         // adjust length down and src/end pos up by partial copied count
2461         __ subw(length, length, rscratch1);
2462         __ addw(src_pos, src_pos, rscratch1);
2463         __ addw(dst_pos, dst_pos, rscratch1);
2464       }
2465 
2466       __ b(*stub->entry());
2467 
2468       __ bind(cont);
2469       __ POP(src, dst);
2470     }
2471   }
2472 
2473 #ifdef ASSERT
2474   if (basic_type != T_OBJECT || !(flags & LIR_OpArrayCopy::type_check)) {
2475     // Sanity check the known type with the incoming class.  For the
2476     // primitive case the types must match exactly with src.klass and
2477     // dst.klass each exactly matching the default type.  For the
2478     // object array case, if no type check is needed then either the
2479     // dst type is exactly the expected type and the src type is a
2480     // subtype which we can't check or src is the same array as dst
2481     // but not necessarily exactly of type default_type.
2482     Label known_ok, halt;
2483     __ mov_metadata(tmp, default_type->constant_encoding());
2484     if (UseCompressedClassPointers) {
2485       __ encode_klass_not_null(tmp);
2486     }
2487 
2488     if (basic_type != T_OBJECT) {
2489 
2490       if (UseCompressedClassPointers) {
2491         __ ldrw(rscratch1, dst_klass_addr);
2492         __ cmpw(tmp, rscratch1);
2493       } else {
2494         __ ldr(rscratch1, dst_klass_addr);
2495         __ cmp(tmp, rscratch1);
2496       }
2497       __ br(Assembler::NE, halt);
2498       if (UseCompressedClassPointers) {
2499         __ ldrw(rscratch1, src_klass_addr);
2500         __ cmpw(tmp, rscratch1);
2501       } else {
2502         __ ldr(rscratch1, src_klass_addr);
2503         __ cmp(tmp, rscratch1);
2504       }
2505       __ br(Assembler::EQ, known_ok);
2506     } else {
2507       if (UseCompressedClassPointers) {
2508         __ ldrw(rscratch1, dst_klass_addr);
2509         __ cmpw(tmp, rscratch1);
2510       } else {
2511         __ ldr(rscratch1, dst_klass_addr);
2512         __ cmp(tmp, rscratch1);
2513       }
2514       __ br(Assembler::EQ, known_ok);
2515       __ cmp(src, dst);
2516       __ br(Assembler::EQ, known_ok);
2517     }
2518     __ bind(halt);
2519     __ stop("incorrect type information in arraycopy");
2520     __ bind(known_ok);
2521   }
2522 #endif
2523 
2524 #ifndef PRODUCT
2525   if (PrintC1Statistics) {
2526     __ incrementw(ExternalAddress(Runtime1::arraycopy_count_address(basic_type)));
2527   }
2528 #endif
2529 
2530   __ lea(c_rarg0, Address(src, src_pos, Address::uxtw(scale)));
2531   __ add(c_rarg0, c_rarg0, arrayOopDesc::base_offset_in_bytes(basic_type));
2532   assert_different_registers(c_rarg0, dst, dst_pos, length);
2533   __ lea(c_rarg1, Address(dst, dst_pos, Address::uxtw(scale)));
2534   __ add(c_rarg1, c_rarg1, arrayOopDesc::base_offset_in_bytes(basic_type));
2535   assert_different_registers(c_rarg1, dst, length);
2536   __ uxtw(c_rarg2, length);
2537   assert_different_registers(c_rarg2, dst);
2538 
2539   bool disjoint = (flags & LIR_OpArrayCopy::overlapping) == 0;
2540   bool aligned = (flags & LIR_OpArrayCopy::unaligned) == 0;
2541   const char *name;
2542   address entry = StubRoutines::select_arraycopy_function(basic_type, aligned, disjoint, name, false);
2543 
2544  CodeBlob *cb = CodeCache::find_blob(entry);
2545  if (cb) {
2546    __ far_call(RuntimeAddress(entry));
2547  } else {
2548    __ call_VM_leaf(entry, 3);
2549  }
2550 
2551   __ bind(*stub->continuation());
2552 }
2553 
2554 
2555 
2556 
2557 void LIR_Assembler::emit_lock(LIR_OpLock* op) {
2558   Register obj = op->obj_opr()->as_register();  // may not be an oop
2559   Register hdr = op->hdr_opr()->as_register();
2560   Register lock = op->lock_opr()->as_register();
2561   if (!UseFastLocking) {
2562     __ b(*op->stub()->entry());
2563   } else if (op->code() == lir_lock) {
2564     Register scratch = noreg;
2565     if (UseBiasedLocking) {
2566       scratch = op->scratch_opr()->as_register();
2567     }
2568     assert(BasicLock::displaced_header_offset_in_bytes() == 0, "lock_reg must point to the displaced header");
2569     __ resolve(ACCESS_READ | ACCESS_WRITE, obj);
2570     // add debug info for NullPointerException only if one is possible
2571     int null_check_offset = __ lock_object(hdr, obj, lock, scratch, *op->stub()->entry());
2572     if (op->info() != NULL) {
2573       add_debug_info_for_null_check(null_check_offset, op->info());
2574     }
2575     // done
2576   } else if (op->code() == lir_unlock) {
2577     assert(BasicLock::displaced_header_offset_in_bytes() == 0, "lock_reg must point to the displaced header");
2578     __ unlock_object(hdr, obj, lock, *op->stub()->entry());
2579   } else {
2580     Unimplemented();
2581   }
2582   __ bind(*op->stub()->continuation());
2583 }
2584 
2585 
2586 void LIR_Assembler::emit_profile_call(LIR_OpProfileCall* op) {
2587   ciMethod* method = op->profiled_method();
2588   int bci          = op->profiled_bci();
2589   ciMethod* callee = op->profiled_callee();
2590 
2591   // Update counter for all call types
2592   ciMethodData* md = method->method_data_or_null();
2593   assert(md != NULL, "Sanity");
2594   ciProfileData* data = md->bci_to_data(bci);
2595   assert(data != NULL && data->is_CounterData(), "need CounterData for calls");
2596   assert(op->mdo()->is_single_cpu(),  "mdo must be allocated");
2597   Register mdo  = op->mdo()->as_register();
2598   __ mov_metadata(mdo, md->constant_encoding());
2599   Address counter_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
2600   // Perform additional virtual call profiling for invokevirtual and
2601   // invokeinterface bytecodes
2602   if (op->should_profile_receiver_type()) {
2603     assert(op->recv()->is_single_cpu(), "recv must be allocated");
2604     Register recv = op->recv()->as_register();
2605     assert_different_registers(mdo, recv);
2606     assert(data->is_VirtualCallData(), "need VirtualCallData for virtual calls");
2607     ciKlass* known_klass = op->known_holder();
2608     if (C1OptimizeVirtualCallProfiling && known_klass != NULL) {
2609       // We know the type that will be seen at this call site; we can
2610       // statically update the MethodData* rather than needing to do
2611       // dynamic tests on the receiver type
2612 
2613       // NOTE: we should probably put a lock around this search to
2614       // avoid collisions by concurrent compilations
2615       ciVirtualCallData* vc_data = (ciVirtualCallData*) data;
2616       uint i;
2617       for (i = 0; i < VirtualCallData::row_limit(); i++) {
2618         ciKlass* receiver = vc_data->receiver(i);
2619         if (known_klass->equals(receiver)) {
2620           Address data_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_count_offset(i)));
2621           __ addptr(data_addr, DataLayout::counter_increment);
2622           return;
2623         }
2624       }
2625 
2626       // Receiver type not found in profile data; select an empty slot
2627 
2628       // Note that this is less efficient than it should be because it
2629       // always does a write to the receiver part of the
2630       // VirtualCallData rather than just the first time
2631       for (i = 0; i < VirtualCallData::row_limit(); i++) {
2632         ciKlass* receiver = vc_data->receiver(i);
2633         if (receiver == NULL) {
2634           Address recv_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_offset(i)));
2635           __ mov_metadata(rscratch1, known_klass->constant_encoding());
2636           __ lea(rscratch2, recv_addr);
2637           __ str(rscratch1, Address(rscratch2));
2638           Address data_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_count_offset(i)));
2639           __ addptr(data_addr, DataLayout::counter_increment);
2640           return;
2641         }
2642       }
2643     } else {
2644       __ load_klass(recv, recv);
2645       Label update_done;
2646       type_profile_helper(mdo, md, data, recv, &update_done);
2647       // Receiver did not match any saved receiver and there is no empty row for it.
2648       // Increment total counter to indicate polymorphic case.
2649       __ addptr(counter_addr, DataLayout::counter_increment);
2650 
2651       __ bind(update_done);
2652     }
2653   } else {
2654     // Static call
2655     __ addptr(counter_addr, DataLayout::counter_increment);
2656   }
2657 }
2658 
2659 
2660 void LIR_Assembler::emit_delay(LIR_OpDelay*) {
2661   Unimplemented();
2662 }
2663 
2664 
2665 void LIR_Assembler::monitor_address(int monitor_no, LIR_Opr dst) {
2666   __ lea(dst->as_register(), frame_map()->address_for_monitor_lock(monitor_no));
2667 }
2668 
2669 void LIR_Assembler::emit_updatecrc32(LIR_OpUpdateCRC32* op) {
2670   assert(op->crc()->is_single_cpu(),  "crc must be register");
2671   assert(op->val()->is_single_cpu(),  "byte value must be register");
2672   assert(op->result_opr()->is_single_cpu(), "result must be register");
2673   Register crc = op->crc()->as_register();
2674   Register val = op->val()->as_register();
2675   Register res = op->result_opr()->as_register();
2676 
2677   assert_different_registers(val, crc, res);
2678   uint64_t offset;
2679   __ adrp(res, ExternalAddress(StubRoutines::crc_table_addr()), offset);
2680   if (offset) __ add(res, res, offset);
2681 
2682   __ mvnw(crc, crc); // ~crc
2683   __ update_byte_crc32(crc, val, res);
2684   __ mvnw(res, crc); // ~crc
2685 }
2686 
2687 void LIR_Assembler::emit_profile_type(LIR_OpProfileType* op) {
2688   COMMENT("emit_profile_type {");
2689   Register obj = op->obj()->as_register();
2690   Register tmp = op->tmp()->as_pointer_register();
2691   Address mdo_addr = as_Address(op->mdp()->as_address_ptr());
2692   ciKlass* exact_klass = op->exact_klass();
2693   intptr_t current_klass = op->current_klass();
2694   bool not_null = op->not_null();
2695   bool no_conflict = op->no_conflict();
2696 
2697   Label update, next, none;
2698 
2699   bool do_null = !not_null;
2700   bool exact_klass_set = exact_klass != NULL && ciTypeEntries::valid_ciklass(current_klass) == exact_klass;
2701   bool do_update = !TypeEntries::is_type_unknown(current_klass) && !exact_klass_set;
2702 
2703   assert(do_null || do_update, "why are we here?");
2704   assert(!TypeEntries::was_null_seen(current_klass) || do_update, "why are we here?");
2705   assert(mdo_addr.base() != rscratch1, "wrong register");
2706 
2707   __ verify_oop(obj);
2708 
2709   if (tmp != obj) {
2710     __ mov(tmp, obj);
2711   }
2712   if (do_null) {
2713     __ cbnz(tmp, update);
2714     if (!TypeEntries::was_null_seen(current_klass)) {
2715       __ ldr(rscratch2, mdo_addr);
2716       __ orr(rscratch2, rscratch2, TypeEntries::null_seen);
2717       __ str(rscratch2, mdo_addr);
2718     }
2719     if (do_update) {
2720 #ifndef ASSERT
2721       __ b(next);
2722     }
2723 #else
2724       __ b(next);
2725     }
2726   } else {
2727     __ cbnz(tmp, update);
2728     __ stop("unexpected null obj");
2729 #endif
2730   }
2731 
2732   __ bind(update);
2733 
2734   if (do_update) {
2735 #ifdef ASSERT
2736     if (exact_klass != NULL) {
2737       Label ok;
2738       __ load_klass(tmp, tmp);
2739       __ mov_metadata(rscratch1, exact_klass->constant_encoding());
2740       __ eor(rscratch1, tmp, rscratch1);
2741       __ cbz(rscratch1, ok);
2742       __ stop("exact klass and actual klass differ");
2743       __ bind(ok);
2744     }
2745 #endif
2746     if (!no_conflict) {
2747       if (exact_klass == NULL || TypeEntries::is_type_none(current_klass)) {
2748         if (exact_klass != NULL) {
2749           __ mov_metadata(tmp, exact_klass->constant_encoding());
2750         } else {
2751           __ load_klass(tmp, tmp);
2752         }
2753 
2754         __ ldr(rscratch2, mdo_addr);
2755         __ eor(tmp, tmp, rscratch2);
2756         __ andr(rscratch1, tmp, TypeEntries::type_klass_mask);
2757         // klass seen before, nothing to do. The unknown bit may have been
2758         // set already but no need to check.
2759         __ cbz(rscratch1, next);
2760 
2761         __ tbnz(tmp, exact_log2(TypeEntries::type_unknown), next); // already unknown. Nothing to do anymore.
2762 
2763         if (TypeEntries::is_type_none(current_klass)) {
2764           __ cbz(rscratch2, none);
2765           __ cmp(rscratch2, (u1)TypeEntries::null_seen);
2766           __ br(Assembler::EQ, none);
2767           // There is a chance that the checks above (re-reading profiling
2768           // data from memory) fail if another thread has just set the
2769           // profiling to this obj's klass
2770           __ dmb(Assembler::ISHLD);
2771           __ ldr(rscratch2, mdo_addr);
2772           __ eor(tmp, tmp, rscratch2);
2773           __ andr(rscratch1, tmp, TypeEntries::type_klass_mask);
2774           __ cbz(rscratch1, next);
2775         }
2776       } else {
2777         assert(ciTypeEntries::valid_ciklass(current_klass) != NULL &&
2778                ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "conflict only");
2779 
2780         __ ldr(tmp, mdo_addr);
2781         __ tbnz(tmp, exact_log2(TypeEntries::type_unknown), next); // already unknown. Nothing to do anymore.
2782       }
2783 
2784       // different than before. Cannot keep accurate profile.
2785       __ ldr(rscratch2, mdo_addr);
2786       __ orr(rscratch2, rscratch2, TypeEntries::type_unknown);
2787       __ str(rscratch2, mdo_addr);
2788 
2789       if (TypeEntries::is_type_none(current_klass)) {
2790         __ b(next);
2791 
2792         __ bind(none);
2793         // first time here. Set profile type.
2794         __ str(tmp, mdo_addr);
2795       }
2796     } else {
2797       // There's a single possible klass at this profile point
2798       assert(exact_klass != NULL, "should be");
2799       if (TypeEntries::is_type_none(current_klass)) {
2800         __ mov_metadata(tmp, exact_klass->constant_encoding());
2801         __ ldr(rscratch2, mdo_addr);
2802         __ eor(tmp, tmp, rscratch2);
2803         __ andr(rscratch1, tmp, TypeEntries::type_klass_mask);
2804         __ cbz(rscratch1, next);
2805 #ifdef ASSERT
2806         {
2807           Label ok;
2808           __ ldr(rscratch1, mdo_addr);
2809           __ cbz(rscratch1, ok);
2810           __ cmp(rscratch1, (u1)TypeEntries::null_seen);
2811           __ br(Assembler::EQ, ok);
2812           // may have been set by another thread
2813           __ dmb(Assembler::ISHLD);
2814           __ mov_metadata(rscratch1, exact_klass->constant_encoding());
2815           __ ldr(rscratch2, mdo_addr);
2816           __ eor(rscratch2, rscratch1, rscratch2);
2817           __ andr(rscratch2, rscratch2, TypeEntries::type_mask);
2818           __ cbz(rscratch2, ok);
2819 
2820           __ stop("unexpected profiling mismatch");
2821           __ bind(ok);
2822         }
2823 #endif
2824         // first time here. Set profile type.
2825         __ ldr(tmp, mdo_addr);
2826       } else {
2827         assert(ciTypeEntries::valid_ciklass(current_klass) != NULL &&
2828                ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "inconsistent");
2829 
2830         __ ldr(tmp, mdo_addr);
2831         __ tbnz(tmp, exact_log2(TypeEntries::type_unknown), next); // already unknown. Nothing to do anymore.
2832 
2833         __ orr(tmp, tmp, TypeEntries::type_unknown);
2834         __ str(tmp, mdo_addr);
2835         // FIXME: Write barrier needed here?
2836       }
2837     }
2838 
2839     __ bind(next);
2840   }
2841   COMMENT("} emit_profile_type");
2842 }
2843 
2844 
2845 void LIR_Assembler::align_backward_branch_target() {
2846 }
2847 
2848 
2849 void LIR_Assembler::negate(LIR_Opr left, LIR_Opr dest, LIR_Opr tmp) {
2850   // tmp must be unused
2851   assert(tmp->is_illegal(), "wasting a register if tmp is allocated");
2852 
2853   if (left->is_single_cpu()) {
2854     assert(dest->is_single_cpu(), "expect single result reg");
2855     __ negw(dest->as_register(), left->as_register());
2856   } else if (left->is_double_cpu()) {
2857     assert(dest->is_double_cpu(), "expect double result reg");
2858     __ neg(dest->as_register_lo(), left->as_register_lo());
2859   } else if (left->is_single_fpu()) {
2860     assert(dest->is_single_fpu(), "expect single float result reg");
2861     __ fnegs(dest->as_float_reg(), left->as_float_reg());
2862   } else {
2863     assert(left->is_double_fpu(), "expect double float operand reg");
2864     assert(dest->is_double_fpu(), "expect double float result reg");
2865     __ fnegd(dest->as_double_reg(), left->as_double_reg());
2866   }
2867 }
2868 
2869 
2870 void LIR_Assembler::leal(LIR_Opr addr, LIR_Opr dest, LIR_PatchCode patch_code, CodeEmitInfo* info) {
2871   if (patch_code != lir_patch_none) {
2872     deoptimize_trap(info);
2873     return;
2874   }
2875 
2876   __ lea(dest->as_register_lo(), as_Address(addr->as_address_ptr()));
2877 }
2878 
2879 
2880 void LIR_Assembler::rt_call(LIR_Opr result, address dest, const LIR_OprList* args, LIR_Opr tmp, CodeEmitInfo* info) {
2881   assert(!tmp->is_valid(), "don't need temporary");
2882 
2883   CodeBlob *cb = CodeCache::find_blob(dest);
2884   if (cb) {
2885     __ far_call(RuntimeAddress(dest));
2886   } else {
2887     __ mov(rscratch1, RuntimeAddress(dest));
2888     __ blr(rscratch1);
2889   }
2890 
2891   if (info != NULL) {
2892     add_call_info_here(info);
2893   }
2894   __ maybe_isb();
2895 }
2896 
2897 void LIR_Assembler::volatile_move_op(LIR_Opr src, LIR_Opr dest, BasicType type, CodeEmitInfo* info) {
2898   if (dest->is_address() || src->is_address()) {
2899     move_op(src, dest, type, lir_patch_none, info,
2900             /*pop_fpu_stack*/false, /*unaligned*/false, /*wide*/false);
2901   } else {
2902     ShouldNotReachHere();
2903   }
2904 }
2905 
2906 #ifdef ASSERT
2907 // emit run-time assertion
2908 void LIR_Assembler::emit_assert(LIR_OpAssert* op) {
2909   assert(op->code() == lir_assert, "must be");
2910 
2911   if (op->in_opr1()->is_valid()) {
2912     assert(op->in_opr2()->is_valid(), "both operands must be valid");
2913     comp_op(op->condition(), op->in_opr1(), op->in_opr2(), op);
2914   } else {
2915     assert(op->in_opr2()->is_illegal(), "both operands must be illegal");
2916     assert(op->condition() == lir_cond_always, "no other conditions allowed");
2917   }
2918 
2919   Label ok;
2920   if (op->condition() != lir_cond_always) {
2921     Assembler::Condition acond = Assembler::AL;
2922     switch (op->condition()) {
2923       case lir_cond_equal:        acond = Assembler::EQ;  break;
2924       case lir_cond_notEqual:     acond = Assembler::NE;  break;
2925       case lir_cond_less:         acond = Assembler::LT;  break;
2926       case lir_cond_lessEqual:    acond = Assembler::LE;  break;
2927       case lir_cond_greaterEqual: acond = Assembler::GE;  break;
2928       case lir_cond_greater:      acond = Assembler::GT;  break;
2929       case lir_cond_belowEqual:   acond = Assembler::LS;  break;
2930       case lir_cond_aboveEqual:   acond = Assembler::HS;  break;
2931       default:                    ShouldNotReachHere();
2932     }
2933     __ br(acond, ok);
2934   }
2935   if (op->halt()) {
2936     const char* str = __ code_string(op->msg());
2937     __ stop(str);
2938   } else {
2939     breakpoint();
2940   }
2941   __ bind(ok);
2942 }
2943 #endif
2944 
2945 #ifndef PRODUCT
2946 #define COMMENT(x)   do { __ block_comment(x); } while (0)
2947 #else
2948 #define COMMENT(x)
2949 #endif
2950 
2951 void LIR_Assembler::membar() {
2952   COMMENT("membar");
2953   __ membar(MacroAssembler::AnyAny);
2954 }
2955 
2956 void LIR_Assembler::membar_acquire() {
2957   __ membar(Assembler::LoadLoad|Assembler::LoadStore);
2958 }
2959 
2960 void LIR_Assembler::membar_release() {
2961   __ membar(Assembler::LoadStore|Assembler::StoreStore);
2962 }
2963 
2964 void LIR_Assembler::membar_loadload() {
2965   __ membar(Assembler::LoadLoad);
2966 }
2967 
2968 void LIR_Assembler::membar_storestore() {
2969   __ membar(MacroAssembler::StoreStore);
2970 }
2971 
2972 void LIR_Assembler::membar_loadstore() { __ membar(MacroAssembler::LoadStore); }
2973 
2974 void LIR_Assembler::membar_storeload() { __ membar(MacroAssembler::StoreLoad); }
2975 
2976 void LIR_Assembler::on_spin_wait() {
2977   Unimplemented();
2978 }
2979 
2980 void LIR_Assembler::get_thread(LIR_Opr result_reg) {
2981   __ mov(result_reg->as_register(), rthread);
2982 }
2983 
2984 
2985 void LIR_Assembler::peephole(LIR_List *lir) {
2986 #if 0
2987   if (tableswitch_count >= max_tableswitches)
2988     return;
2989 
2990   /*
2991     This finite-state automaton recognizes sequences of compare-and-
2992     branch instructions.  We will turn them into a tableswitch.  You
2993     could argue that C1 really shouldn't be doing this sort of
2994     optimization, but without it the code is really horrible.
2995   */
2996 
2997   enum { start_s, cmp1_s, beq_s, cmp_s } state;
2998   int first_key, last_key = -2147483648;
2999   int next_key = 0;
3000   int start_insn = -1;
3001   int last_insn = -1;
3002   Register reg = noreg;
3003   LIR_Opr reg_opr;
3004   state = start_s;
3005 
3006   LIR_OpList* inst = lir->instructions_list();
3007   for (int i = 0; i < inst->length(); i++) {
3008     LIR_Op* op = inst->at(i);
3009     switch (state) {
3010     case start_s:
3011       first_key = -1;
3012       start_insn = i;
3013       switch (op->code()) {
3014       case lir_cmp:
3015         LIR_Opr opr1 = op->as_Op2()->in_opr1();
3016         LIR_Opr opr2 = op->as_Op2()->in_opr2();
3017         if (opr1->is_cpu_register() && opr1->is_single_cpu()
3018             && opr2->is_constant()
3019             && opr2->type() == T_INT) {
3020           reg_opr = opr1;
3021           reg = opr1->as_register();
3022           first_key = opr2->as_constant_ptr()->as_jint();
3023           next_key = first_key + 1;
3024           state = cmp_s;
3025           goto next_state;
3026         }
3027         break;
3028       }
3029       break;
3030     case cmp_s:
3031       switch (op->code()) {
3032       case lir_branch:
3033         if (op->as_OpBranch()->cond() == lir_cond_equal) {
3034           state = beq_s;
3035           last_insn = i;
3036           goto next_state;
3037         }
3038       }
3039       state = start_s;
3040       break;
3041     case beq_s:
3042       switch (op->code()) {
3043       case lir_cmp: {
3044         LIR_Opr opr1 = op->as_Op2()->in_opr1();
3045         LIR_Opr opr2 = op->as_Op2()->in_opr2();
3046         if (opr1->is_cpu_register() && opr1->is_single_cpu()
3047             && opr1->as_register() == reg
3048             && opr2->is_constant()
3049             && opr2->type() == T_INT
3050             && opr2->as_constant_ptr()->as_jint() == next_key) {
3051           last_key = next_key;
3052           next_key++;
3053           state = cmp_s;
3054           goto next_state;
3055         }
3056       }
3057       }
3058       last_key = next_key;
3059       state = start_s;
3060       break;
3061     default:
3062       assert(false, "impossible state");
3063     }
3064     if (state == start_s) {
3065       if (first_key < last_key - 5L && reg != noreg) {
3066         {
3067           // printf("found run register %d starting at insn %d low value %d high value %d\n",
3068           //        reg->encoding(),
3069           //        start_insn, first_key, last_key);
3070           //   for (int i = 0; i < inst->length(); i++) {
3071           //     inst->at(i)->print();
3072           //     tty->print("\n");
3073           //   }
3074           //   tty->print("\n");
3075         }
3076 
3077         struct tableswitch *sw = &switches[tableswitch_count];
3078         sw->_insn_index = start_insn, sw->_first_key = first_key,
3079           sw->_last_key = last_key, sw->_reg = reg;
3080         inst->insert_before(last_insn + 1, new LIR_OpLabel(&sw->_after));
3081         {
3082           // Insert the new table of branches
3083           int offset = last_insn;
3084           for (int n = first_key; n < last_key; n++) {
3085             inst->insert_before
3086               (last_insn + 1,
3087                new LIR_OpBranch(lir_cond_always, T_ILLEGAL,
3088                                 inst->at(offset)->as_OpBranch()->label()));
3089             offset -= 2, i++;
3090           }
3091         }
3092         // Delete all the old compare-and-branch instructions
3093         for (int n = first_key; n < last_key; n++) {
3094           inst->remove_at(start_insn);
3095           inst->remove_at(start_insn);
3096         }
3097         // Insert the tableswitch instruction
3098         inst->insert_before(start_insn,
3099                             new LIR_Op2(lir_cmp, lir_cond_always,
3100                                         LIR_OprFact::intConst(tableswitch_count),
3101                                         reg_opr));
3102         inst->insert_before(start_insn + 1, new LIR_OpLabel(&sw->_branches));
3103         tableswitch_count++;
3104       }
3105       reg = noreg;
3106       last_key = -2147483648;
3107     }
3108   next_state:
3109     ;
3110   }
3111 #endif
3112 }
3113 
3114 void LIR_Assembler::atomic_op(LIR_Code code, LIR_Opr src, LIR_Opr data, LIR_Opr dest, LIR_Opr tmp_op) {
3115   Address addr = as_Address(src->as_address_ptr());
3116   BasicType type = src->type();
3117   bool is_oop = is_reference_type(type);
3118 
3119   void (MacroAssembler::* add)(Register prev, RegisterOrConstant incr, Register addr);
3120   void (MacroAssembler::* xchg)(Register prev, Register newv, Register addr);
3121 
3122   switch(type) {
3123   case T_INT:
3124     xchg = &MacroAssembler::atomic_xchgalw;
3125     add = &MacroAssembler::atomic_addalw;
3126     break;
3127   case T_LONG:
3128     xchg = &MacroAssembler::atomic_xchgal;
3129     add = &MacroAssembler::atomic_addal;
3130     break;
3131   case T_OBJECT:
3132   case T_ARRAY:
3133     if (UseCompressedOops) {
3134       xchg = &MacroAssembler::atomic_xchgalw;
3135       add = &MacroAssembler::atomic_addalw;
3136     } else {
3137       xchg = &MacroAssembler::atomic_xchgal;
3138       add = &MacroAssembler::atomic_addal;
3139     }
3140     break;
3141   default:
3142     ShouldNotReachHere();
3143     xchg = &MacroAssembler::atomic_xchgal;
3144     add = &MacroAssembler::atomic_addal; // unreachable
3145   }
3146 
3147   switch (code) {
3148   case lir_xadd:
3149     {
3150       RegisterOrConstant inc;
3151       Register tmp = as_reg(tmp_op);
3152       Register dst = as_reg(dest);
3153       if (data->is_constant()) {
3154         inc = RegisterOrConstant(as_long(data));
3155         assert_different_registers(dst, addr.base(), tmp,
3156                                    rscratch1, rscratch2);
3157       } else {
3158         inc = RegisterOrConstant(as_reg(data));
3159         assert_different_registers(inc.as_register(), dst, addr.base(), tmp,
3160                                    rscratch1, rscratch2);
3161       }
3162       __ lea(tmp, addr);
3163       (_masm->*add)(dst, inc, tmp);
3164       break;
3165     }
3166   case lir_xchg:
3167     {
3168       Register tmp = tmp_op->as_register();
3169       Register obj = as_reg(data);
3170       Register dst = as_reg(dest);
3171       if (is_oop && UseCompressedOops) {
3172         __ encode_heap_oop(rscratch2, obj);
3173         obj = rscratch2;
3174       }
3175       assert_different_registers(obj, addr.base(), tmp, rscratch1, dst);
3176       __ lea(tmp, addr);
3177       (_masm->*xchg)(dst, obj, tmp);
3178       if (is_oop && UseCompressedOops) {
3179         __ decode_heap_oop(dst);
3180       }
3181     }
3182     break;
3183   default:
3184     ShouldNotReachHere();
3185   }
3186   __ membar(__ AnyAny);
3187 }
3188 
3189 #undef __