1 /*
   2  * Copyright (c) 2000, 2018, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #include "precompiled.hpp"
  26 #include "c1/c1_Compilation.hpp"
  27 #include "c1/c1_LIRAssembler.hpp"
  28 #include "c1/c1_MacroAssembler.hpp"
  29 #include "c1/c1_Runtime1.hpp"
  30 #include "c1/c1_ValueStack.hpp"
  31 #include "ci/ciArrayKlass.hpp"
  32 #include "ci/ciInstance.hpp"
  33 #include "gc/shared/barrierSet.hpp"
  34 #include "gc/shared/cardTableModRefBS.hpp"
  35 #include "gc/shared/collectedHeap.hpp"
  36 #include "nativeInst_sparc.hpp"
  37 #include "oops/objArrayKlass.hpp"
  38 #include "runtime/safepointMechanism.inline.hpp"
  39 #include "runtime/sharedRuntime.hpp"
  40 
  41 #define __ _masm->
  42 
  43 
  44 //------------------------------------------------------------
  45 
  46 
  47 bool LIR_Assembler::is_small_constant(LIR_Opr opr) {
  48   if (opr->is_constant()) {
  49     LIR_Const* constant = opr->as_constant_ptr();
  50     switch (constant->type()) {
  51       case T_INT: {
  52         jint value = constant->as_jint();
  53         return Assembler::is_simm13(value);
  54       }
  55 
  56       default:
  57         return false;
  58     }
  59   }
  60   return false;
  61 }
  62 
  63 
  64 bool LIR_Assembler::is_single_instruction(LIR_Op* op) {
  65   switch (op->code()) {
  66     case lir_null_check:
  67     return true;
  68 
  69 
  70     case lir_add:
  71     case lir_ushr:
  72     case lir_shr:
  73     case lir_shl:
  74       // integer shifts and adds are always one instruction
  75       return op->result_opr()->is_single_cpu();
  76 
  77 
  78     case lir_move: {
  79       LIR_Op1* op1 = op->as_Op1();
  80       LIR_Opr src = op1->in_opr();
  81       LIR_Opr dst = op1->result_opr();
  82 
  83       if (src == dst) {
  84         NEEDS_CLEANUP;
  85         // this works around a problem where moves with the same src and dst
  86         // end up in the delay slot and then the assembler swallows the mov
  87         // since it has no effect and then it complains because the delay slot
  88         // is empty.  returning false stops the optimizer from putting this in
  89         // the delay slot
  90         return false;
  91       }
  92 
  93       // don't put moves involving oops into the delay slot since the VerifyOops code
  94       // will make it much larger than a single instruction.
  95       if (VerifyOops) {
  96         return false;
  97       }
  98 
  99       if (src->is_double_cpu() || dst->is_double_cpu() || op1->patch_code() != lir_patch_none ||
 100           ((src->is_double_fpu() || dst->is_double_fpu()) && op1->move_kind() != lir_move_normal)) {
 101         return false;
 102       }
 103 
 104       if (UseCompressedOops) {
 105         if (dst->is_address() && !dst->is_stack() && (dst->type() == T_OBJECT || dst->type() == T_ARRAY)) return false;
 106         if (src->is_address() && !src->is_stack() && (src->type() == T_OBJECT || src->type() == T_ARRAY)) return false;
 107       }
 108 
 109       if (UseCompressedClassPointers) {
 110         if (src->is_address() && !src->is_stack() && src->type() == T_ADDRESS &&
 111             src->as_address_ptr()->disp() == oopDesc::klass_offset_in_bytes()) return false;
 112       }
 113 
 114       if (dst->is_register()) {
 115         if (src->is_address() && Assembler::is_simm13(src->as_address_ptr()->disp())) {
 116           return !PatchALot;
 117         } else if (src->is_single_stack()) {
 118           return true;
 119         }
 120       }
 121 
 122       if (src->is_register()) {
 123         if (dst->is_address() && Assembler::is_simm13(dst->as_address_ptr()->disp())) {
 124           return !PatchALot;
 125         } else if (dst->is_single_stack()) {
 126           return true;
 127         }
 128       }
 129 
 130       if (dst->is_register() &&
 131           ((src->is_register() && src->is_single_word() && src->is_same_type(dst)) ||
 132            (src->is_constant() && LIR_Assembler::is_small_constant(op->as_Op1()->in_opr())))) {
 133         return true;
 134       }
 135 
 136       return false;
 137     }
 138 
 139     default:
 140       return false;
 141   }
 142   ShouldNotReachHere();
 143 }
 144 
 145 
 146 LIR_Opr LIR_Assembler::receiverOpr() {
 147   return FrameMap::O0_oop_opr;
 148 }
 149 
 150 
 151 LIR_Opr LIR_Assembler::osrBufferPointer() {
 152   return FrameMap::I0_opr;
 153 }
 154 
 155 
 156 int LIR_Assembler::initial_frame_size_in_bytes() const {
 157   return in_bytes(frame_map()->framesize_in_bytes());
 158 }
 159 
 160 
 161 // inline cache check: the inline cached class is in G5_inline_cache_reg(G5);
 162 // we fetch the class of the receiver (O0) and compare it with the cached class.
 163 // If they do not match we jump to slow case.
 164 int LIR_Assembler::check_icache() {
 165   int offset = __ offset();
 166   __ inline_cache_check(O0, G5_inline_cache_reg);
 167   return offset;
 168 }
 169 
 170 
 171 void LIR_Assembler::osr_entry() {
 172   // On-stack-replacement entry sequence (interpreter frame layout described in interpreter_sparc.cpp):
 173   //
 174   //   1. Create a new compiled activation.
 175   //   2. Initialize local variables in the compiled activation.  The expression stack must be empty
 176   //      at the osr_bci; it is not initialized.
 177   //   3. Jump to the continuation address in compiled code to resume execution.
 178 
 179   // OSR entry point
 180   offsets()->set_value(CodeOffsets::OSR_Entry, code_offset());
 181   BlockBegin* osr_entry = compilation()->hir()->osr_entry();
 182   ValueStack* entry_state = osr_entry->end()->state();
 183   int number_of_locks = entry_state->locks_size();
 184 
 185   // Create a frame for the compiled activation.
 186   __ build_frame(initial_frame_size_in_bytes(), bang_size_in_bytes());
 187 
 188   // OSR buffer is
 189   //
 190   // locals[nlocals-1..0]
 191   // monitors[number_of_locks-1..0]
 192   //
 193   // locals is a direct copy of the interpreter frame so in the osr buffer
 194   // so first slot in the local array is the last local from the interpreter
 195   // and last slot is local[0] (receiver) from the interpreter
 196   //
 197   // Similarly with locks. The first lock slot in the osr buffer is the nth lock
 198   // from the interpreter frame, the nth lock slot in the osr buffer is 0th lock
 199   // in the interpreter frame (the method lock if a sync method)
 200 
 201   // Initialize monitors in the compiled activation.
 202   //   I0: pointer to osr buffer
 203   //
 204   // All other registers are dead at this point and the locals will be
 205   // copied into place by code emitted in the IR.
 206 
 207   Register OSR_buf = osrBufferPointer()->as_register();
 208   { assert(frame::interpreter_frame_monitor_size() == BasicObjectLock::size(), "adjust code below");
 209     int monitor_offset = BytesPerWord * method()->max_locals() +
 210       (2 * BytesPerWord) * (number_of_locks - 1);
 211     // SharedRuntime::OSR_migration_begin() packs BasicObjectLocks in
 212     // the OSR buffer using 2 word entries: first the lock and then
 213     // the oop.
 214     for (int i = 0; i < number_of_locks; i++) {
 215       int slot_offset = monitor_offset - ((i * 2) * BytesPerWord);
 216 #ifdef ASSERT
 217       // verify the interpreter's monitor has a non-null object
 218       {
 219         Label L;
 220         __ ld_ptr(OSR_buf, slot_offset + 1*BytesPerWord, O7);
 221         __ cmp_and_br_short(O7, G0, Assembler::notEqual, Assembler::pt, L);
 222         __ stop("locked object is NULL");
 223         __ bind(L);
 224       }
 225 #endif // ASSERT
 226       // Copy the lock field into the compiled activation.
 227       __ ld_ptr(OSR_buf, slot_offset + 0, O7);
 228       __ st_ptr(O7, frame_map()->address_for_monitor_lock(i));
 229       __ ld_ptr(OSR_buf, slot_offset + 1*BytesPerWord, O7);
 230       __ st_ptr(O7, frame_map()->address_for_monitor_object(i));
 231     }
 232   }
 233 }
 234 
 235 
 236 // --------------------------------------------------------------------------------------------
 237 
 238 void LIR_Assembler::monitorexit(LIR_Opr obj_opr, LIR_Opr lock_opr, Register hdr, int monitor_no) {
 239   if (!GenerateSynchronizationCode) return;
 240 
 241   Register obj_reg = obj_opr->as_register();
 242   Register lock_reg = lock_opr->as_register();
 243 
 244   Address mon_addr = frame_map()->address_for_monitor_lock(monitor_no);
 245   Register reg = mon_addr.base();
 246   int offset = mon_addr.disp();
 247   // compute pointer to BasicLock
 248   if (mon_addr.is_simm13()) {
 249     __ add(reg, offset, lock_reg);
 250   }
 251   else {
 252     __ set(offset, lock_reg);
 253     __ add(reg, lock_reg, lock_reg);
 254   }
 255   // unlock object
 256   MonitorAccessStub* slow_case = new MonitorExitStub(lock_opr, UseFastLocking, monitor_no);
 257   // _slow_case_stubs->append(slow_case);
 258   // temporary fix: must be created after exceptionhandler, therefore as call stub
 259   _slow_case_stubs->append(slow_case);
 260   if (UseFastLocking) {
 261     // try inlined fast unlocking first, revert to slow locking if it fails
 262     // note: lock_reg points to the displaced header since the displaced header offset is 0!
 263     assert(BasicLock::displaced_header_offset_in_bytes() == 0, "lock_reg must point to the displaced header");
 264     __ unlock_object(hdr, obj_reg, lock_reg, *slow_case->entry());
 265   } else {
 266     // always do slow unlocking
 267     // note: the slow unlocking code could be inlined here, however if we use
 268     //       slow unlocking, speed doesn't matter anyway and this solution is
 269     //       simpler and requires less duplicated code - additionally, the
 270     //       slow unlocking code is the same in either case which simplifies
 271     //       debugging
 272     __ br(Assembler::always, false, Assembler::pt, *slow_case->entry());
 273     __ delayed()->nop();
 274   }
 275   // done
 276   __ bind(*slow_case->continuation());
 277 }
 278 
 279 
 280 int LIR_Assembler::emit_exception_handler() {
 281   // if the last instruction is a call (typically to do a throw which
 282   // is coming at the end after block reordering) the return address
 283   // must still point into the code area in order to avoid assertion
 284   // failures when searching for the corresponding bci => add a nop
 285   // (was bug 5/14/1999 - gri)
 286   __ nop();
 287 
 288   // generate code for exception handler
 289   ciMethod* method = compilation()->method();
 290 
 291   address handler_base = __ start_a_stub(exception_handler_size());
 292 
 293   if (handler_base == NULL) {
 294     // not enough space left for the handler
 295     bailout("exception handler overflow");
 296     return -1;
 297   }
 298 
 299   int offset = code_offset();
 300 
 301   __ call(Runtime1::entry_for(Runtime1::handle_exception_from_callee_id), relocInfo::runtime_call_type);
 302   __ delayed()->nop();
 303   __ should_not_reach_here();
 304   guarantee(code_offset() - offset <= exception_handler_size(), "overflow");
 305   __ end_a_stub();
 306 
 307   return offset;
 308 }
 309 
 310 
 311 // Emit the code to remove the frame from the stack in the exception
 312 // unwind path.
 313 int LIR_Assembler::emit_unwind_handler() {
 314 #ifndef PRODUCT
 315   if (CommentedAssembly) {
 316     _masm->block_comment("Unwind handler");
 317   }
 318 #endif
 319 
 320   int offset = code_offset();
 321 
 322   // Fetch the exception from TLS and clear out exception related thread state
 323   __ ld_ptr(G2_thread, in_bytes(JavaThread::exception_oop_offset()), O0);
 324   __ st_ptr(G0, G2_thread, in_bytes(JavaThread::exception_oop_offset()));
 325   __ st_ptr(G0, G2_thread, in_bytes(JavaThread::exception_pc_offset()));
 326 
 327   __ bind(_unwind_handler_entry);
 328   __ verify_not_null_oop(O0);
 329   if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
 330     __ mov(O0, I0);  // Preserve the exception
 331   }
 332 
 333   // Preform needed unlocking
 334   MonitorExitStub* stub = NULL;
 335   if (method()->is_synchronized()) {
 336     monitor_address(0, FrameMap::I1_opr);
 337     stub = new MonitorExitStub(FrameMap::I1_opr, true, 0);
 338     __ unlock_object(I3, I2, I1, *stub->entry());
 339     __ bind(*stub->continuation());
 340   }
 341 
 342   if (compilation()->env()->dtrace_method_probes()) {
 343     __ mov(G2_thread, O0);
 344     __ save_thread(I1); // need to preserve thread in G2 across
 345                         // runtime call
 346     metadata2reg(method()->constant_encoding(), O1);
 347     __ call(CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_exit), relocInfo::runtime_call_type);
 348     __ delayed()->nop();
 349     __ restore_thread(I1);
 350   }
 351 
 352   if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
 353     __ mov(I0, O0);  // Restore the exception
 354   }
 355 
 356   // dispatch to the unwind logic
 357   __ call(Runtime1::entry_for(Runtime1::unwind_exception_id), relocInfo::runtime_call_type);
 358   __ delayed()->nop();
 359 
 360   // Emit the slow path assembly
 361   if (stub != NULL) {
 362     stub->emit_code(this);
 363   }
 364 
 365   return offset;
 366 }
 367 
 368 
 369 int LIR_Assembler::emit_deopt_handler() {
 370   // if the last instruction is a call (typically to do a throw which
 371   // is coming at the end after block reordering) the return address
 372   // must still point into the code area in order to avoid assertion
 373   // failures when searching for the corresponding bci => add a nop
 374   // (was bug 5/14/1999 - gri)
 375   __ nop();
 376 
 377   // generate code for deopt handler
 378   ciMethod* method = compilation()->method();
 379   address handler_base = __ start_a_stub(deopt_handler_size());
 380   if (handler_base == NULL) {
 381     // not enough space left for the handler
 382     bailout("deopt handler overflow");
 383     return -1;
 384   }
 385 
 386   int offset = code_offset();
 387   AddressLiteral deopt_blob(SharedRuntime::deopt_blob()->unpack());
 388   __ JUMP(deopt_blob, G3_scratch, 0); // sethi;jmp
 389   __ delayed()->nop();
 390   guarantee(code_offset() - offset <= deopt_handler_size(), "overflow");
 391   __ end_a_stub();
 392 
 393   return offset;
 394 }
 395 
 396 
 397 void LIR_Assembler::jobject2reg(jobject o, Register reg) {
 398   if (o == NULL) {
 399     __ set(NULL_WORD, reg);
 400   } else {
 401 #ifdef ASSERT
 402     {
 403       ThreadInVMfromNative tiv(JavaThread::current());
 404       assert(Universe::heap()->is_in_reserved(JNIHandles::resolve(o)), "should be real oop");
 405     }
 406 #endif
 407     int oop_index = __ oop_recorder()->find_index(o);
 408     RelocationHolder rspec = oop_Relocation::spec(oop_index);
 409     __ set(NULL_WORD, reg, rspec); // Will be set when the nmethod is created
 410   }
 411 }
 412 
 413 
 414 void LIR_Assembler::jobject2reg_with_patching(Register reg, CodeEmitInfo *info) {
 415   // Allocate a new index in table to hold the object once it's been patched
 416   int oop_index = __ oop_recorder()->allocate_oop_index(NULL);
 417   PatchingStub* patch = new PatchingStub(_masm, patching_id(info), oop_index);
 418 
 419   AddressLiteral addrlit(NULL, oop_Relocation::spec(oop_index));
 420   assert(addrlit.rspec().type() == relocInfo::oop_type, "must be an oop reloc");
 421   // It may not seem necessary to use a sethi/add pair to load a NULL into dest, but the
 422   // NULL will be dynamically patched later and the patched value may be large.  We must
 423   // therefore generate the sethi/add as a placeholders
 424   __ patchable_set(addrlit, reg);
 425 
 426   patching_epilog(patch, lir_patch_normal, reg, info);
 427 }
 428 
 429 
 430 void LIR_Assembler::metadata2reg(Metadata* o, Register reg) {
 431   __ set_metadata_constant(o, reg);
 432 }
 433 
 434 void LIR_Assembler::klass2reg_with_patching(Register reg, CodeEmitInfo *info) {
 435   // Allocate a new index in table to hold the klass once it's been patched
 436   int index = __ oop_recorder()->allocate_metadata_index(NULL);
 437   PatchingStub* patch = new PatchingStub(_masm, PatchingStub::load_klass_id, index);
 438   AddressLiteral addrlit(NULL, metadata_Relocation::spec(index));
 439   assert(addrlit.rspec().type() == relocInfo::metadata_type, "must be an metadata reloc");
 440   // It may not seem necessary to use a sethi/add pair to load a NULL into dest, but the
 441   // NULL will be dynamically patched later and the patched value may be large.  We must
 442   // therefore generate the sethi/add as a placeholders
 443   __ patchable_set(addrlit, reg);
 444 
 445   patching_epilog(patch, lir_patch_normal, reg, info);
 446 }
 447 
 448 void LIR_Assembler::emit_op3(LIR_Op3* op) {
 449   switch (op->code()) {
 450     case lir_idiv:
 451     case lir_irem:  // Both idiv & irem are handled after the switch (below).
 452       break;
 453     case lir_fmaf:
 454       __ fmadd(FloatRegisterImpl::S,
 455                op->in_opr1()->as_float_reg(),
 456                op->in_opr2()->as_float_reg(),
 457                op->in_opr3()->as_float_reg(),
 458                op->result_opr()->as_float_reg());
 459       return;
 460     case lir_fmad:
 461       __ fmadd(FloatRegisterImpl::D,
 462                op->in_opr1()->as_double_reg(),
 463                op->in_opr2()->as_double_reg(),
 464                op->in_opr3()->as_double_reg(),
 465                op->result_opr()->as_double_reg());
 466       return;
 467     default:
 468       ShouldNotReachHere();
 469       break;
 470   }
 471 
 472   // Handle idiv & irem:
 473 
 474   Register Rdividend = op->in_opr1()->as_register();
 475   Register Rdivisor  = noreg;
 476   Register Rscratch  = op->in_opr3()->as_register();
 477   Register Rresult   = op->result_opr()->as_register();
 478   int divisor = -1;
 479 
 480   if (op->in_opr2()->is_register()) {
 481     Rdivisor = op->in_opr2()->as_register();
 482   } else {
 483     divisor = op->in_opr2()->as_constant_ptr()->as_jint();
 484     assert(Assembler::is_simm13(divisor), "can only handle simm13");
 485   }
 486 
 487   assert(Rdividend != Rscratch, "");
 488   assert(Rdivisor  != Rscratch, "");
 489   assert(op->code() == lir_idiv || op->code() == lir_irem, "Must be irem or idiv");
 490 
 491   if (Rdivisor == noreg && is_power_of_2(divisor)) {
 492     // convert division by a power of two into some shifts and logical operations
 493     if (op->code() == lir_idiv) {
 494       if (divisor == 2) {
 495         __ srl(Rdividend, 31, Rscratch);
 496       } else {
 497         __ sra(Rdividend, 31, Rscratch);
 498         __ and3(Rscratch, divisor - 1, Rscratch);
 499       }
 500       __ add(Rdividend, Rscratch, Rscratch);
 501       __ sra(Rscratch, log2_intptr(divisor), Rresult);
 502       return;
 503     } else {
 504       if (divisor == 2) {
 505         __ srl(Rdividend, 31, Rscratch);
 506       } else {
 507         __ sra(Rdividend, 31, Rscratch);
 508         __ and3(Rscratch, divisor - 1,Rscratch);
 509       }
 510       __ add(Rdividend, Rscratch, Rscratch);
 511       __ andn(Rscratch, divisor - 1,Rscratch);
 512       __ sub(Rdividend, Rscratch, Rresult);
 513       return;
 514     }
 515   }
 516 
 517   __ sra(Rdividend, 31, Rscratch);
 518   __ wry(Rscratch);
 519 
 520   add_debug_info_for_div0_here(op->info());
 521 
 522   if (Rdivisor != noreg) {
 523     __ sdivcc(Rdividend, Rdivisor, (op->code() == lir_idiv ? Rresult : Rscratch));
 524   } else {
 525     assert(Assembler::is_simm13(divisor), "can only handle simm13");
 526     __ sdivcc(Rdividend, divisor, (op->code() == lir_idiv ? Rresult : Rscratch));
 527   }
 528 
 529   Label skip;
 530   __ br(Assembler::overflowSet, true, Assembler::pn, skip);
 531   __ delayed()->Assembler::sethi(0x80000000, (op->code() == lir_idiv ? Rresult : Rscratch));
 532   __ bind(skip);
 533 
 534   if (op->code() == lir_irem) {
 535     if (Rdivisor != noreg) {
 536       __ smul(Rscratch, Rdivisor, Rscratch);
 537     } else {
 538       __ smul(Rscratch, divisor, Rscratch);
 539     }
 540     __ sub(Rdividend, Rscratch, Rresult);
 541   }
 542 }
 543 
 544 
 545 void LIR_Assembler::emit_opBranch(LIR_OpBranch* op) {
 546 #ifdef ASSERT
 547   assert(op->block() == NULL || op->block()->label() == op->label(), "wrong label");
 548   if (op->block() != NULL)  _branch_target_blocks.append(op->block());
 549   if (op->ublock() != NULL) _branch_target_blocks.append(op->ublock());
 550 #endif
 551   assert(op->info() == NULL, "shouldn't have CodeEmitInfo");
 552 
 553   if (op->cond() == lir_cond_always) {
 554     __ br(Assembler::always, false, Assembler::pt, *(op->label()));
 555   } else if (op->code() == lir_cond_float_branch) {
 556     assert(op->ublock() != NULL, "must have unordered successor");
 557     bool is_unordered = (op->ublock() == op->block());
 558     Assembler::Condition acond;
 559     switch (op->cond()) {
 560       case lir_cond_equal:         acond = Assembler::f_equal;    break;
 561       case lir_cond_notEqual:      acond = Assembler::f_notEqual; break;
 562       case lir_cond_less:          acond = (is_unordered ? Assembler::f_unorderedOrLess          : Assembler::f_less);           break;
 563       case lir_cond_greater:       acond = (is_unordered ? Assembler::f_unorderedOrGreater       : Assembler::f_greater);        break;
 564       case lir_cond_lessEqual:     acond = (is_unordered ? Assembler::f_unorderedOrLessOrEqual   : Assembler::f_lessOrEqual);    break;
 565       case lir_cond_greaterEqual:  acond = (is_unordered ? Assembler::f_unorderedOrGreaterOrEqual: Assembler::f_greaterOrEqual); break;
 566       default :                         ShouldNotReachHere();
 567     }
 568     __ fb( acond, false, Assembler::pn, *(op->label()));
 569   } else {
 570     assert (op->code() == lir_branch, "just checking");
 571 
 572     Assembler::Condition acond;
 573     switch (op->cond()) {
 574       case lir_cond_equal:        acond = Assembler::equal;                break;
 575       case lir_cond_notEqual:     acond = Assembler::notEqual;             break;
 576       case lir_cond_less:         acond = Assembler::less;                 break;
 577       case lir_cond_lessEqual:    acond = Assembler::lessEqual;            break;
 578       case lir_cond_greaterEqual: acond = Assembler::greaterEqual;         break;
 579       case lir_cond_greater:      acond = Assembler::greater;              break;
 580       case lir_cond_aboveEqual:   acond = Assembler::greaterEqualUnsigned; break;
 581       case lir_cond_belowEqual:   acond = Assembler::lessEqualUnsigned;    break;
 582       default:                         ShouldNotReachHere();
 583     };
 584 
 585     // sparc has different condition codes for testing 32-bit
 586     // vs. 64-bit values.  We could always test xcc is we could
 587     // guarantee that 32-bit loads always sign extended but that isn't
 588     // true and since sign extension isn't free, it would impose a
 589     // slight cost.
 590     if  (op->type() == T_INT) {
 591       __ br(acond, false, Assembler::pn, *(op->label()));
 592     } else
 593       __ brx(acond, false, Assembler::pn, *(op->label()));
 594   }
 595   // The peephole pass fills the delay slot
 596 }
 597 
 598 
 599 void LIR_Assembler::emit_opConvert(LIR_OpConvert* op) {
 600   Bytecodes::Code code = op->bytecode();
 601   LIR_Opr dst = op->result_opr();
 602 
 603   switch(code) {
 604     case Bytecodes::_i2l: {
 605       Register rlo  = dst->as_register_lo();
 606       Register rhi  = dst->as_register_hi();
 607       Register rval = op->in_opr()->as_register();
 608       __ sra(rval, 0, rlo);
 609       break;
 610     }
 611     case Bytecodes::_i2d:
 612     case Bytecodes::_i2f: {
 613       bool is_double = (code == Bytecodes::_i2d);
 614       FloatRegister rdst = is_double ? dst->as_double_reg() : dst->as_float_reg();
 615       FloatRegisterImpl::Width w = is_double ? FloatRegisterImpl::D : FloatRegisterImpl::S;
 616       FloatRegister rsrc = op->in_opr()->as_float_reg();
 617       if (rsrc != rdst) {
 618         __ fmov(FloatRegisterImpl::S, rsrc, rdst);
 619       }
 620       __ fitof(w, rdst, rdst);
 621       break;
 622     }
 623     case Bytecodes::_f2i:{
 624       FloatRegister rsrc = op->in_opr()->as_float_reg();
 625       Address       addr = frame_map()->address_for_slot(dst->single_stack_ix());
 626       Label L;
 627       // result must be 0 if value is NaN; test by comparing value to itself
 628       __ fcmp(FloatRegisterImpl::S, Assembler::fcc0, rsrc, rsrc);
 629       __ fb(Assembler::f_unordered, true, Assembler::pn, L);
 630       __ delayed()->st(G0, addr); // annuled if contents of rsrc is not NaN
 631       __ ftoi(FloatRegisterImpl::S, rsrc, rsrc);
 632       // move integer result from float register to int register
 633       __ stf(FloatRegisterImpl::S, rsrc, addr.base(), addr.disp());
 634       __ bind (L);
 635       break;
 636     }
 637     case Bytecodes::_l2i: {
 638       Register rlo  = op->in_opr()->as_register_lo();
 639       Register rhi  = op->in_opr()->as_register_hi();
 640       Register rdst = dst->as_register();
 641       __ sra(rlo, 0, rdst);
 642       break;
 643     }
 644     case Bytecodes::_d2f:
 645     case Bytecodes::_f2d: {
 646       bool is_double = (code == Bytecodes::_f2d);
 647       assert((!is_double && dst->is_single_fpu()) || (is_double && dst->is_double_fpu()), "check");
 648       LIR_Opr val = op->in_opr();
 649       FloatRegister rval = (code == Bytecodes::_d2f) ? val->as_double_reg() : val->as_float_reg();
 650       FloatRegister rdst = is_double ? dst->as_double_reg() : dst->as_float_reg();
 651       FloatRegisterImpl::Width vw = is_double ? FloatRegisterImpl::S : FloatRegisterImpl::D;
 652       FloatRegisterImpl::Width dw = is_double ? FloatRegisterImpl::D : FloatRegisterImpl::S;
 653       __ ftof(vw, dw, rval, rdst);
 654       break;
 655     }
 656     case Bytecodes::_i2s:
 657     case Bytecodes::_i2b: {
 658       Register rval = op->in_opr()->as_register();
 659       Register rdst = dst->as_register();
 660       int shift = (code == Bytecodes::_i2b) ? (BitsPerInt - T_BYTE_aelem_bytes * BitsPerByte) : (BitsPerInt - BitsPerShort);
 661       __ sll (rval, shift, rdst);
 662       __ sra (rdst, shift, rdst);
 663       break;
 664     }
 665     case Bytecodes::_i2c: {
 666       Register rval = op->in_opr()->as_register();
 667       Register rdst = dst->as_register();
 668       int shift = BitsPerInt - T_CHAR_aelem_bytes * BitsPerByte;
 669       __ sll (rval, shift, rdst);
 670       __ srl (rdst, shift, rdst);
 671       break;
 672     }
 673 
 674     default: ShouldNotReachHere();
 675   }
 676 }
 677 
 678 
 679 void LIR_Assembler::align_call(LIR_Code) {
 680   // do nothing since all instructions are word aligned on sparc
 681 }
 682 
 683 
 684 void LIR_Assembler::call(LIR_OpJavaCall* op, relocInfo::relocType rtype) {
 685   __ call(op->addr(), rtype);
 686   // The peephole pass fills the delay slot, add_call_info is done in
 687   // LIR_Assembler::emit_delay.
 688 }
 689 
 690 
 691 void LIR_Assembler::ic_call(LIR_OpJavaCall* op) {
 692   __ ic_call(op->addr(), false);
 693   // The peephole pass fills the delay slot, add_call_info is done in
 694   // LIR_Assembler::emit_delay.
 695 }
 696 
 697 
 698 void LIR_Assembler::vtable_call(LIR_OpJavaCall* op) {
 699   add_debug_info_for_null_check_here(op->info());
 700   __ load_klass(O0, G3_scratch);
 701   if (Assembler::is_simm13(op->vtable_offset())) {
 702     __ ld_ptr(G3_scratch, op->vtable_offset(), G5_method);
 703   } else {
 704     // This will generate 2 instructions
 705     __ set(op->vtable_offset(), G5_method);
 706     // ld_ptr, set_hi, set
 707     __ ld_ptr(G3_scratch, G5_method, G5_method);
 708   }
 709   __ ld_ptr(G5_method, Method::from_compiled_offset(), G3_scratch);
 710   __ callr(G3_scratch, G0);
 711   // the peephole pass fills the delay slot
 712 }
 713 
 714 int LIR_Assembler::store(LIR_Opr from_reg, Register base, int offset, BasicType type, bool wide, bool unaligned) {
 715   int store_offset;
 716   if (!Assembler::is_simm13(offset + (type == T_LONG) ? wordSize : 0)) {
 717     assert(base != O7, "destroying register");
 718     assert(!unaligned, "can't handle this");
 719     // for offsets larger than a simm13 we setup the offset in O7
 720     __ set(offset, O7);
 721     store_offset = store(from_reg, base, O7, type, wide);
 722   } else {
 723     if (type == T_ARRAY || type == T_OBJECT) {
 724       __ verify_oop(from_reg->as_register());
 725     }
 726     store_offset = code_offset();
 727     switch (type) {
 728       case T_BOOLEAN: // fall through
 729       case T_BYTE  : __ stb(from_reg->as_register(), base, offset); break;
 730       case T_CHAR  : __ sth(from_reg->as_register(), base, offset); break;
 731       case T_SHORT : __ sth(from_reg->as_register(), base, offset); break;
 732       case T_INT   : __ stw(from_reg->as_register(), base, offset); break;
 733       case T_LONG  :
 734         if (unaligned || PatchALot) {
 735           // Don't use O7 here because it may be equal to 'base' (see LIR_Assembler::reg2mem)
 736           assert(G3_scratch != base, "can't handle this");
 737           assert(G3_scratch != from_reg->as_register_lo(), "can't handle this");
 738           __ srax(from_reg->as_register_lo(), 32, G3_scratch);
 739           __ stw(from_reg->as_register_lo(), base, offset + lo_word_offset_in_bytes);
 740           __ stw(G3_scratch,                 base, offset + hi_word_offset_in_bytes);
 741         } else {
 742           __ stx(from_reg->as_register_lo(), base, offset);
 743         }
 744         break;
 745       case T_ADDRESS:
 746       case T_METADATA:
 747         __ st_ptr(from_reg->as_register(), base, offset);
 748         break;
 749       case T_ARRAY : // fall through
 750       case T_OBJECT:
 751         {
 752           if (UseCompressedOops && !wide) {
 753             __ encode_heap_oop(from_reg->as_register(), G3_scratch);
 754             store_offset = code_offset();
 755             __ stw(G3_scratch, base, offset);
 756           } else {
 757             __ st_ptr(from_reg->as_register(), base, offset);
 758           }
 759           break;
 760         }
 761 
 762       case T_FLOAT : __ stf(FloatRegisterImpl::S, from_reg->as_float_reg(), base, offset); break;
 763       case T_DOUBLE:
 764         {
 765           FloatRegister reg = from_reg->as_double_reg();
 766           // split unaligned stores
 767           if (unaligned || PatchALot) {
 768             assert(Assembler::is_simm13(offset + 4), "must be");
 769             __ stf(FloatRegisterImpl::S, reg->successor(), base, offset + 4);
 770             __ stf(FloatRegisterImpl::S, reg,              base, offset);
 771           } else {
 772             __ stf(FloatRegisterImpl::D, reg, base, offset);
 773           }
 774           break;
 775         }
 776       default      : ShouldNotReachHere();
 777     }
 778   }
 779   return store_offset;
 780 }
 781 
 782 
 783 int LIR_Assembler::store(LIR_Opr from_reg, Register base, Register disp, BasicType type, bool wide) {
 784   if (type == T_ARRAY || type == T_OBJECT) {
 785     __ verify_oop(from_reg->as_register());
 786   }
 787   int store_offset = code_offset();
 788   switch (type) {
 789     case T_BOOLEAN: // fall through
 790     case T_BYTE  : __ stb(from_reg->as_register(), base, disp); break;
 791     case T_CHAR  : __ sth(from_reg->as_register(), base, disp); break;
 792     case T_SHORT : __ sth(from_reg->as_register(), base, disp); break;
 793     case T_INT   : __ stw(from_reg->as_register(), base, disp); break;
 794     case T_LONG  :
 795       __ stx(from_reg->as_register_lo(), base, disp);
 796       break;
 797     case T_ADDRESS:
 798       __ st_ptr(from_reg->as_register(), base, disp);
 799       break;
 800     case T_ARRAY : // fall through
 801     case T_OBJECT:
 802       {
 803         if (UseCompressedOops && !wide) {
 804           __ encode_heap_oop(from_reg->as_register(), G3_scratch);
 805           store_offset = code_offset();
 806           __ stw(G3_scratch, base, disp);
 807         } else {
 808           __ st_ptr(from_reg->as_register(), base, disp);
 809         }
 810         break;
 811       }
 812     case T_FLOAT : __ stf(FloatRegisterImpl::S, from_reg->as_float_reg(), base, disp); break;
 813     case T_DOUBLE: __ stf(FloatRegisterImpl::D, from_reg->as_double_reg(), base, disp); break;
 814     default      : ShouldNotReachHere();
 815   }
 816   return store_offset;
 817 }
 818 
 819 
 820 int LIR_Assembler::load(Register base, int offset, LIR_Opr to_reg, BasicType type, bool wide, bool unaligned) {
 821   int load_offset;
 822   if (!Assembler::is_simm13(offset + (type == T_LONG) ? wordSize : 0)) {
 823     assert(base != O7, "destroying register");
 824     assert(!unaligned, "can't handle this");
 825     // for offsets larger than a simm13 we setup the offset in O7
 826     __ set(offset, O7);
 827     load_offset = load(base, O7, to_reg, type, wide);
 828   } else {
 829     load_offset = code_offset();
 830     switch(type) {
 831       case T_BOOLEAN: // fall through
 832       case T_BYTE  : __ ldsb(base, offset, to_reg->as_register()); break;
 833       case T_CHAR  : __ lduh(base, offset, to_reg->as_register()); break;
 834       case T_SHORT : __ ldsh(base, offset, to_reg->as_register()); break;
 835       case T_INT   : __ ld(base, offset, to_reg->as_register()); break;
 836       case T_LONG  :
 837         if (!unaligned && !PatchALot) {
 838           __ ldx(base, offset, to_reg->as_register_lo());
 839         } else {
 840           assert(base != to_reg->as_register_lo(), "can't handle this");
 841           assert(O7 != to_reg->as_register_lo(), "can't handle this");
 842           __ ld(base, offset + hi_word_offset_in_bytes, to_reg->as_register_lo());
 843           __ lduw(base, offset + lo_word_offset_in_bytes, O7); // in case O7 is base or offset, use it last
 844           __ sllx(to_reg->as_register_lo(), 32, to_reg->as_register_lo());
 845           __ or3(to_reg->as_register_lo(), O7, to_reg->as_register_lo());
 846         }
 847         break;
 848       case T_METADATA:  __ ld_ptr(base, offset, to_reg->as_register()); break;
 849       case T_ADDRESS:
 850         if (offset == oopDesc::klass_offset_in_bytes() && UseCompressedClassPointers) {
 851           __ lduw(base, offset, to_reg->as_register());
 852           __ decode_klass_not_null(to_reg->as_register());
 853         } else
 854         {
 855           __ ld_ptr(base, offset, to_reg->as_register());
 856         }
 857         break;
 858       case T_ARRAY : // fall through
 859       case T_OBJECT:
 860         {
 861           if (UseCompressedOops && !wide) {
 862             __ lduw(base, offset, to_reg->as_register());
 863             __ decode_heap_oop(to_reg->as_register());
 864           } else {
 865             __ ld_ptr(base, offset, to_reg->as_register());
 866           }
 867           break;
 868         }
 869       case T_FLOAT:  __ ldf(FloatRegisterImpl::S, base, offset, to_reg->as_float_reg()); break;
 870       case T_DOUBLE:
 871         {
 872           FloatRegister reg = to_reg->as_double_reg();
 873           // split unaligned loads
 874           if (unaligned || PatchALot) {
 875             __ ldf(FloatRegisterImpl::S, base, offset + 4, reg->successor());
 876             __ ldf(FloatRegisterImpl::S, base, offset,     reg);
 877           } else {
 878             __ ldf(FloatRegisterImpl::D, base, offset, to_reg->as_double_reg());
 879           }
 880           break;
 881         }
 882       default      : ShouldNotReachHere();
 883     }
 884     if (type == T_ARRAY || type == T_OBJECT) {
 885       __ verify_oop(to_reg->as_register());
 886     }
 887   }
 888   return load_offset;
 889 }
 890 
 891 
 892 int LIR_Assembler::load(Register base, Register disp, LIR_Opr to_reg, BasicType type, bool wide) {
 893   int load_offset = code_offset();
 894   switch(type) {
 895     case T_BOOLEAN: // fall through
 896     case T_BYTE  :  __ ldsb(base, disp, to_reg->as_register()); break;
 897     case T_CHAR  :  __ lduh(base, disp, to_reg->as_register()); break;
 898     case T_SHORT :  __ ldsh(base, disp, to_reg->as_register()); break;
 899     case T_INT   :  __ ld(base, disp, to_reg->as_register()); break;
 900     case T_ADDRESS: __ ld_ptr(base, disp, to_reg->as_register()); break;
 901     case T_ARRAY : // fall through
 902     case T_OBJECT:
 903       {
 904           if (UseCompressedOops && !wide) {
 905             __ lduw(base, disp, to_reg->as_register());
 906             __ decode_heap_oop(to_reg->as_register());
 907           } else {
 908             __ ld_ptr(base, disp, to_reg->as_register());
 909           }
 910           break;
 911       }
 912     case T_FLOAT:  __ ldf(FloatRegisterImpl::S, base, disp, to_reg->as_float_reg()); break;
 913     case T_DOUBLE: __ ldf(FloatRegisterImpl::D, base, disp, to_reg->as_double_reg()); break;
 914     case T_LONG  :
 915       __ ldx(base, disp, to_reg->as_register_lo());
 916       break;
 917     default      : ShouldNotReachHere();
 918   }
 919   if (type == T_ARRAY || type == T_OBJECT) {
 920     __ verify_oop(to_reg->as_register());
 921   }
 922   return load_offset;
 923 }
 924 
 925 void LIR_Assembler::const2stack(LIR_Opr src, LIR_Opr dest) {
 926   LIR_Const* c = src->as_constant_ptr();
 927   switch (c->type()) {
 928     case T_INT:
 929     case T_FLOAT: {
 930       Register src_reg = O7;
 931       int value = c->as_jint_bits();
 932       if (value == 0) {
 933         src_reg = G0;
 934       } else {
 935         __ set(value, O7);
 936       }
 937       Address addr = frame_map()->address_for_slot(dest->single_stack_ix());
 938       __ stw(src_reg, addr.base(), addr.disp());
 939       break;
 940     }
 941     case T_ADDRESS: {
 942       Register src_reg = O7;
 943       int value = c->as_jint_bits();
 944       if (value == 0) {
 945         src_reg = G0;
 946       } else {
 947         __ set(value, O7);
 948       }
 949       Address addr = frame_map()->address_for_slot(dest->single_stack_ix());
 950       __ st_ptr(src_reg, addr.base(), addr.disp());
 951       break;
 952     }
 953     case T_OBJECT: {
 954       Register src_reg = O7;
 955       jobject2reg(c->as_jobject(), src_reg);
 956       Address addr = frame_map()->address_for_slot(dest->single_stack_ix());
 957       __ st_ptr(src_reg, addr.base(), addr.disp());
 958       break;
 959     }
 960     case T_LONG:
 961     case T_DOUBLE: {
 962       Address addr = frame_map()->address_for_double_slot(dest->double_stack_ix());
 963 
 964       Register tmp = O7;
 965       int value_lo = c->as_jint_lo_bits();
 966       if (value_lo == 0) {
 967         tmp = G0;
 968       } else {
 969         __ set(value_lo, O7);
 970       }
 971       __ stw(tmp, addr.base(), addr.disp() + lo_word_offset_in_bytes);
 972       int value_hi = c->as_jint_hi_bits();
 973       if (value_hi == 0) {
 974         tmp = G0;
 975       } else {
 976         __ set(value_hi, O7);
 977       }
 978       __ stw(tmp, addr.base(), addr.disp() + hi_word_offset_in_bytes);
 979       break;
 980     }
 981     default:
 982       Unimplemented();
 983   }
 984 }
 985 
 986 
 987 void LIR_Assembler::const2mem(LIR_Opr src, LIR_Opr dest, BasicType type, CodeEmitInfo* info, bool wide) {
 988   LIR_Const* c = src->as_constant_ptr();
 989   LIR_Address* addr     = dest->as_address_ptr();
 990   Register base = addr->base()->as_pointer_register();
 991   int offset = -1;
 992 
 993   switch (c->type()) {
 994     case T_INT:
 995     case T_FLOAT:
 996     case T_ADDRESS: {
 997       LIR_Opr tmp = FrameMap::O7_opr;
 998       int value = c->as_jint_bits();
 999       if (value == 0) {
1000         tmp = FrameMap::G0_opr;
1001       } else if (Assembler::is_simm13(value)) {
1002         __ set(value, O7);
1003       }
1004       if (addr->index()->is_valid()) {
1005         assert(addr->disp() == 0, "must be zero");
1006         offset = store(tmp, base, addr->index()->as_pointer_register(), type, wide);
1007       } else {
1008         assert(Assembler::is_simm13(addr->disp()), "can't handle larger addresses");
1009         offset = store(tmp, base, addr->disp(), type, wide, false);
1010       }
1011       break;
1012     }
1013     case T_LONG:
1014     case T_DOUBLE: {
1015       assert(!addr->index()->is_valid(), "can't handle reg reg address here");
1016       assert(Assembler::is_simm13(addr->disp()) &&
1017              Assembler::is_simm13(addr->disp() + 4), "can't handle larger addresses");
1018 
1019       LIR_Opr tmp = FrameMap::O7_opr;
1020       int value_lo = c->as_jint_lo_bits();
1021       if (value_lo == 0) {
1022         tmp = FrameMap::G0_opr;
1023       } else {
1024         __ set(value_lo, O7);
1025       }
1026       offset = store(tmp, base, addr->disp() + lo_word_offset_in_bytes, T_INT, wide, false);
1027       int value_hi = c->as_jint_hi_bits();
1028       if (value_hi == 0) {
1029         tmp = FrameMap::G0_opr;
1030       } else {
1031         __ set(value_hi, O7);
1032       }
1033       store(tmp, base, addr->disp() + hi_word_offset_in_bytes, T_INT, wide, false);
1034       break;
1035     }
1036     case T_OBJECT: {
1037       jobject obj = c->as_jobject();
1038       LIR_Opr tmp;
1039       if (obj == NULL) {
1040         tmp = FrameMap::G0_opr;
1041       } else {
1042         tmp = FrameMap::O7_opr;
1043         jobject2reg(c->as_jobject(), O7);
1044       }
1045       // handle either reg+reg or reg+disp address
1046       if (addr->index()->is_valid()) {
1047         assert(addr->disp() == 0, "must be zero");
1048         offset = store(tmp, base, addr->index()->as_pointer_register(), type, wide);
1049       } else {
1050         assert(Assembler::is_simm13(addr->disp()), "can't handle larger addresses");
1051         offset = store(tmp, base, addr->disp(), type, wide, false);
1052       }
1053 
1054       break;
1055     }
1056     default:
1057       Unimplemented();
1058   }
1059   if (info != NULL) {
1060     assert(offset != -1, "offset should've been set");
1061     add_debug_info_for_null_check(offset, info);
1062   }
1063 }
1064 
1065 
1066 void LIR_Assembler::const2reg(LIR_Opr src, LIR_Opr dest, LIR_PatchCode patch_code, CodeEmitInfo* info) {
1067   LIR_Const* c = src->as_constant_ptr();
1068   LIR_Opr to_reg = dest;
1069 
1070   switch (c->type()) {
1071     case T_INT:
1072     case T_ADDRESS:
1073       {
1074         jint con = c->as_jint();
1075         if (to_reg->is_single_cpu()) {
1076           assert(patch_code == lir_patch_none, "no patching handled here");
1077           __ set(con, to_reg->as_register());
1078         } else {
1079           ShouldNotReachHere();
1080           assert(to_reg->is_single_fpu(), "wrong register kind");
1081 
1082           __ set(con, O7);
1083           Address temp_slot(SP, (frame::register_save_words * wordSize) + STACK_BIAS);
1084           __ st(O7, temp_slot);
1085           __ ldf(FloatRegisterImpl::S, temp_slot, to_reg->as_float_reg());
1086         }
1087       }
1088       break;
1089 
1090     case T_LONG:
1091       {
1092         jlong con = c->as_jlong();
1093 
1094         if (to_reg->is_double_cpu()) {
1095           __ set(con,  to_reg->as_register_lo());
1096         } else if (to_reg->is_single_cpu()) {
1097           __ set(con, to_reg->as_register());
1098         } else {
1099           ShouldNotReachHere();
1100           assert(to_reg->is_double_fpu(), "wrong register kind");
1101           Address temp_slot_lo(SP, ((frame::register_save_words  ) * wordSize) + STACK_BIAS);
1102           Address temp_slot_hi(SP, ((frame::register_save_words) * wordSize) + (longSize/2) + STACK_BIAS);
1103           __ set(low(con),  O7);
1104           __ st(O7, temp_slot_lo);
1105           __ set(high(con), O7);
1106           __ st(O7, temp_slot_hi);
1107           __ ldf(FloatRegisterImpl::D, temp_slot_lo, to_reg->as_double_reg());
1108         }
1109       }
1110       break;
1111 
1112     case T_OBJECT:
1113       {
1114         if (patch_code == lir_patch_none) {
1115           jobject2reg(c->as_jobject(), to_reg->as_register());
1116         } else {
1117           jobject2reg_with_patching(to_reg->as_register(), info);
1118         }
1119       }
1120       break;
1121 
1122     case T_METADATA:
1123       {
1124         if (patch_code == lir_patch_none) {
1125           metadata2reg(c->as_metadata(), to_reg->as_register());
1126         } else {
1127           klass2reg_with_patching(to_reg->as_register(), info);
1128         }
1129       }
1130       break;
1131 
1132     case T_FLOAT:
1133       {
1134         address const_addr = __ float_constant(c->as_jfloat());
1135         if (const_addr == NULL) {
1136           bailout("const section overflow");
1137           break;
1138         }
1139         RelocationHolder rspec = internal_word_Relocation::spec(const_addr);
1140         AddressLiteral const_addrlit(const_addr, rspec);
1141         if (to_reg->is_single_fpu()) {
1142           __ patchable_sethi(const_addrlit, O7);
1143           __ relocate(rspec);
1144           __ ldf(FloatRegisterImpl::S, O7, const_addrlit.low10(), to_reg->as_float_reg());
1145 
1146         } else {
1147           assert(to_reg->is_single_cpu(), "Must be a cpu register.");
1148 
1149           __ set(const_addrlit, O7);
1150           __ ld(O7, 0, to_reg->as_register());
1151         }
1152       }
1153       break;
1154 
1155     case T_DOUBLE:
1156       {
1157         address const_addr = __ double_constant(c->as_jdouble());
1158         if (const_addr == NULL) {
1159           bailout("const section overflow");
1160           break;
1161         }
1162         RelocationHolder rspec = internal_word_Relocation::spec(const_addr);
1163 
1164         if (to_reg->is_double_fpu()) {
1165           AddressLiteral const_addrlit(const_addr, rspec);
1166           __ patchable_sethi(const_addrlit, O7);
1167           __ relocate(rspec);
1168           __ ldf (FloatRegisterImpl::D, O7, const_addrlit.low10(), to_reg->as_double_reg());
1169         } else {
1170           assert(to_reg->is_double_cpu(), "Must be a long register.");
1171           __ set(jlong_cast(c->as_jdouble()), to_reg->as_register_lo());
1172         }
1173 
1174       }
1175       break;
1176 
1177     default:
1178       ShouldNotReachHere();
1179   }
1180 }
1181 
1182 Address LIR_Assembler::as_Address(LIR_Address* addr) {
1183   Register reg = addr->base()->as_pointer_register();
1184   LIR_Opr index = addr->index();
1185   if (index->is_illegal()) {
1186     return Address(reg, addr->disp());
1187   } else {
1188     assert (addr->disp() == 0, "unsupported address mode");
1189     return Address(reg, index->as_pointer_register());
1190   }
1191 }
1192 
1193 
1194 void LIR_Assembler::stack2stack(LIR_Opr src, LIR_Opr dest, BasicType type) {
1195   switch (type) {
1196     case T_INT:
1197     case T_FLOAT: {
1198       Register tmp = O7;
1199       Address from = frame_map()->address_for_slot(src->single_stack_ix());
1200       Address to   = frame_map()->address_for_slot(dest->single_stack_ix());
1201       __ lduw(from.base(), from.disp(), tmp);
1202       __ stw(tmp, to.base(), to.disp());
1203       break;
1204     }
1205     case T_OBJECT: {
1206       Register tmp = O7;
1207       Address from = frame_map()->address_for_slot(src->single_stack_ix());
1208       Address to   = frame_map()->address_for_slot(dest->single_stack_ix());
1209       __ ld_ptr(from.base(), from.disp(), tmp);
1210       __ st_ptr(tmp, to.base(), to.disp());
1211       break;
1212     }
1213     case T_LONG:
1214     case T_DOUBLE: {
1215       Register tmp = O7;
1216       Address from = frame_map()->address_for_double_slot(src->double_stack_ix());
1217       Address to   = frame_map()->address_for_double_slot(dest->double_stack_ix());
1218       __ lduw(from.base(), from.disp(), tmp);
1219       __ stw(tmp, to.base(), to.disp());
1220       __ lduw(from.base(), from.disp() + 4, tmp);
1221       __ stw(tmp, to.base(), to.disp() + 4);
1222       break;
1223     }
1224 
1225     default:
1226       ShouldNotReachHere();
1227   }
1228 }
1229 
1230 
1231 Address LIR_Assembler::as_Address_hi(LIR_Address* addr) {
1232   Address base = as_Address(addr);
1233   return Address(base.base(), base.disp() + hi_word_offset_in_bytes);
1234 }
1235 
1236 
1237 Address LIR_Assembler::as_Address_lo(LIR_Address* addr) {
1238   Address base = as_Address(addr);
1239   return Address(base.base(), base.disp() + lo_word_offset_in_bytes);
1240 }
1241 
1242 
1243 void LIR_Assembler::mem2reg(LIR_Opr src_opr, LIR_Opr dest, BasicType type,
1244                             LIR_PatchCode patch_code, CodeEmitInfo* info, bool wide, bool unaligned) {
1245 
1246   assert(type != T_METADATA, "load of metadata ptr not supported");
1247   LIR_Address* addr = src_opr->as_address_ptr();
1248   LIR_Opr to_reg = dest;
1249 
1250   Register src = addr->base()->as_pointer_register();
1251   Register disp_reg = noreg;
1252   int disp_value = addr->disp();
1253   bool needs_patching = (patch_code != lir_patch_none);
1254 
1255   if (addr->base()->type() == T_OBJECT) {
1256     __ verify_oop(src);
1257   }
1258 
1259   PatchingStub* patch = NULL;
1260   if (needs_patching) {
1261     patch = new PatchingStub(_masm, PatchingStub::access_field_id);
1262     assert(!to_reg->is_double_cpu() ||
1263            patch_code == lir_patch_none ||
1264            patch_code == lir_patch_normal, "patching doesn't match register");
1265   }
1266 
1267   if (addr->index()->is_illegal()) {
1268     if (!Assembler::is_simm13(disp_value) && (!unaligned || Assembler::is_simm13(disp_value + 4))) {
1269       if (needs_patching) {
1270         __ patchable_set(0, O7);
1271       } else {
1272         __ set(disp_value, O7);
1273       }
1274       disp_reg = O7;
1275     }
1276   } else if (unaligned || PatchALot) {
1277     __ add(src, addr->index()->as_pointer_register(), O7);
1278     src = O7;
1279   } else {
1280     disp_reg = addr->index()->as_pointer_register();
1281     assert(disp_value == 0, "can't handle 3 operand addresses");
1282   }
1283 
1284   // remember the offset of the load.  The patching_epilog must be done
1285   // before the call to add_debug_info, otherwise the PcDescs don't get
1286   // entered in increasing order.
1287   int offset = code_offset();
1288 
1289   assert(disp_reg != noreg || Assembler::is_simm13(disp_value), "should have set this up");
1290   if (disp_reg == noreg) {
1291     offset = load(src, disp_value, to_reg, type, wide, unaligned);
1292   } else {
1293     assert(!unaligned, "can't handle this");
1294     offset = load(src, disp_reg, to_reg, type, wide);
1295   }
1296 
1297   if (patch != NULL) {
1298     patching_epilog(patch, patch_code, src, info);
1299   }
1300   if (info != NULL) add_debug_info_for_null_check(offset, info);
1301 }
1302 
1303 
1304 void LIR_Assembler::stack2reg(LIR_Opr src, LIR_Opr dest, BasicType type) {
1305   Address addr;
1306   if (src->is_single_word()) {
1307     addr = frame_map()->address_for_slot(src->single_stack_ix());
1308   } else if (src->is_double_word())  {
1309     addr = frame_map()->address_for_double_slot(src->double_stack_ix());
1310   }
1311 
1312   bool unaligned = (addr.disp() - STACK_BIAS) % 8 != 0;
1313   load(addr.base(), addr.disp(), dest, dest->type(), true /*wide*/, unaligned);
1314 }
1315 
1316 
1317 void LIR_Assembler::reg2stack(LIR_Opr from_reg, LIR_Opr dest, BasicType type, bool pop_fpu_stack) {
1318   Address addr;
1319   if (dest->is_single_word()) {
1320     addr = frame_map()->address_for_slot(dest->single_stack_ix());
1321   } else if (dest->is_double_word())  {
1322     addr = frame_map()->address_for_slot(dest->double_stack_ix());
1323   }
1324   bool unaligned = (addr.disp() - STACK_BIAS) % 8 != 0;
1325   store(from_reg, addr.base(), addr.disp(), from_reg->type(), true /*wide*/, unaligned);
1326 }
1327 
1328 
1329 void LIR_Assembler::reg2reg(LIR_Opr from_reg, LIR_Opr to_reg) {
1330   if (from_reg->is_float_kind() && to_reg->is_float_kind()) {
1331     if (from_reg->is_double_fpu()) {
1332       // double to double moves
1333       assert(to_reg->is_double_fpu(), "should match");
1334       __ fmov(FloatRegisterImpl::D, from_reg->as_double_reg(), to_reg->as_double_reg());
1335     } else {
1336       // float to float moves
1337       assert(to_reg->is_single_fpu(), "should match");
1338       __ fmov(FloatRegisterImpl::S, from_reg->as_float_reg(), to_reg->as_float_reg());
1339     }
1340   } else if (!from_reg->is_float_kind() && !to_reg->is_float_kind()) {
1341     if (from_reg->is_double_cpu()) {
1342       __ mov(from_reg->as_pointer_register(), to_reg->as_pointer_register());
1343     } else if (to_reg->is_double_cpu()) {
1344       // int to int moves
1345       __ mov(from_reg->as_register(), to_reg->as_register_lo());
1346     } else {
1347       // int to int moves
1348       __ mov(from_reg->as_register(), to_reg->as_register());
1349     }
1350   } else {
1351     ShouldNotReachHere();
1352   }
1353   if (to_reg->type() == T_OBJECT || to_reg->type() == T_ARRAY) {
1354     __ verify_oop(to_reg->as_register());
1355   }
1356 }
1357 
1358 
1359 void LIR_Assembler::reg2mem(LIR_Opr from_reg, LIR_Opr dest, BasicType type,
1360                             LIR_PatchCode patch_code, CodeEmitInfo* info, bool pop_fpu_stack,
1361                             bool wide, bool unaligned) {
1362   assert(type != T_METADATA, "store of metadata ptr not supported");
1363   LIR_Address* addr = dest->as_address_ptr();
1364 
1365   Register src = addr->base()->as_pointer_register();
1366   Register disp_reg = noreg;
1367   int disp_value = addr->disp();
1368   bool needs_patching = (patch_code != lir_patch_none);
1369 
1370   if (addr->base()->is_oop_register()) {
1371     __ verify_oop(src);
1372   }
1373 
1374   PatchingStub* patch = NULL;
1375   if (needs_patching) {
1376     patch = new PatchingStub(_masm, PatchingStub::access_field_id);
1377     assert(!from_reg->is_double_cpu() ||
1378            patch_code == lir_patch_none ||
1379            patch_code == lir_patch_normal, "patching doesn't match register");
1380   }
1381 
1382   if (addr->index()->is_illegal()) {
1383     if (!Assembler::is_simm13(disp_value) && (!unaligned || Assembler::is_simm13(disp_value + 4))) {
1384       if (needs_patching) {
1385         __ patchable_set(0, O7);
1386       } else {
1387         __ set(disp_value, O7);
1388       }
1389       disp_reg = O7;
1390     }
1391   } else if (unaligned || PatchALot) {
1392     __ add(src, addr->index()->as_pointer_register(), O7);
1393     src = O7;
1394   } else {
1395     disp_reg = addr->index()->as_pointer_register();
1396     assert(disp_value == 0, "can't handle 3 operand addresses");
1397   }
1398 
1399   // remember the offset of the store.  The patching_epilog must be done
1400   // before the call to add_debug_info_for_null_check, otherwise the PcDescs don't get
1401   // entered in increasing order.
1402   int offset;
1403 
1404   assert(disp_reg != noreg || Assembler::is_simm13(disp_value), "should have set this up");
1405   if (disp_reg == noreg) {
1406     offset = store(from_reg, src, disp_value, type, wide, unaligned);
1407   } else {
1408     assert(!unaligned, "can't handle this");
1409     offset = store(from_reg, src, disp_reg, type, wide);
1410   }
1411 
1412   if (patch != NULL) {
1413     patching_epilog(patch, patch_code, src, info);
1414   }
1415 
1416   if (info != NULL) add_debug_info_for_null_check(offset, info);
1417 }
1418 
1419 
1420 void LIR_Assembler::return_op(LIR_Opr result) {
1421   if (StackReservedPages > 0 && compilation()->has_reserved_stack_access()) {
1422     __ reserved_stack_check();
1423   }
1424   if (SafepointMechanism::uses_thread_local_poll()) {
1425     __ ld_ptr(Address(G2_thread, Thread::polling_page_offset()), L0);
1426   } else {
1427     __ set((intptr_t)os::get_polling_page(), L0);
1428   }
1429   __ relocate(relocInfo::poll_return_type);
1430   __ ld_ptr(L0, 0, G0);
1431   __ ret();
1432   __ delayed()->restore();
1433 }
1434 
1435 
1436 int LIR_Assembler::safepoint_poll(LIR_Opr tmp, CodeEmitInfo* info) {
1437   if (SafepointMechanism::uses_thread_local_poll()) {
1438     __ ld_ptr(Address(G2_thread, Thread::polling_page_offset()), tmp->as_register());
1439   } else {
1440     __ set((intptr_t)os::get_polling_page(), tmp->as_register());
1441   }
1442   if (info != NULL) {
1443     add_debug_info_for_branch(info);
1444   }
1445   int offset = __ offset();
1446 
1447   __ relocate(relocInfo::poll_type);
1448   __ ld_ptr(tmp->as_register(), 0, G0);
1449   return offset;
1450 }
1451 
1452 
1453 void LIR_Assembler::emit_static_call_stub() {
1454   address call_pc = __ pc();
1455   address stub = __ start_a_stub(call_stub_size());
1456   if (stub == NULL) {
1457     bailout("static call stub overflow");
1458     return;
1459   }
1460 
1461   int start = __ offset();
1462   __ relocate(static_stub_Relocation::spec(call_pc));
1463 
1464   __ set_metadata(NULL, G5);
1465   // must be set to -1 at code generation time
1466   AddressLiteral addrlit(-1);
1467   __ jump_to(addrlit, G3);
1468   __ delayed()->nop();
1469 
1470   assert(__ offset() - start <= call_stub_size(), "stub too big");
1471   __ end_a_stub();
1472 }
1473 
1474 
1475 void LIR_Assembler::comp_op(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Op2* op) {
1476   if (opr1->is_single_fpu()) {
1477     __ fcmp(FloatRegisterImpl::S, Assembler::fcc0, opr1->as_float_reg(), opr2->as_float_reg());
1478   } else if (opr1->is_double_fpu()) {
1479     __ fcmp(FloatRegisterImpl::D, Assembler::fcc0, opr1->as_double_reg(), opr2->as_double_reg());
1480   } else if (opr1->is_single_cpu()) {
1481     if (opr2->is_constant()) {
1482       switch (opr2->as_constant_ptr()->type()) {
1483         case T_INT:
1484           { jint con = opr2->as_constant_ptr()->as_jint();
1485             if (Assembler::is_simm13(con)) {
1486               __ cmp(opr1->as_register(), con);
1487             } else {
1488               __ set(con, O7);
1489               __ cmp(opr1->as_register(), O7);
1490             }
1491           }
1492           break;
1493 
1494         case T_OBJECT:
1495           // there are only equal/notequal comparisions on objects
1496           { jobject con = opr2->as_constant_ptr()->as_jobject();
1497             if (con == NULL) {
1498               __ cmp(opr1->as_register(), 0);
1499             } else {
1500               jobject2reg(con, O7);
1501               __ cmp(opr1->as_register(), O7);
1502             }
1503           }
1504           break;
1505 
1506         default:
1507           ShouldNotReachHere();
1508           break;
1509       }
1510     } else {
1511       if (opr2->is_address()) {
1512         LIR_Address * addr = opr2->as_address_ptr();
1513         BasicType type = addr->type();
1514         if ( type == T_OBJECT ) __ ld_ptr(as_Address(addr), O7);
1515         else                    __ ld(as_Address(addr), O7);
1516         __ cmp(opr1->as_register(), O7);
1517       } else {
1518         __ cmp(opr1->as_register(), opr2->as_register());
1519       }
1520     }
1521   } else if (opr1->is_double_cpu()) {
1522     Register xlo = opr1->as_register_lo();
1523     Register xhi = opr1->as_register_hi();
1524     if (opr2->is_constant() && opr2->as_jlong() == 0) {
1525       assert(condition == lir_cond_equal || condition == lir_cond_notEqual, "only handles these cases");
1526       __ orcc(xhi, G0, G0);
1527     } else if (opr2->is_register()) {
1528       Register ylo = opr2->as_register_lo();
1529       Register yhi = opr2->as_register_hi();
1530       __ cmp(xlo, ylo);
1531     } else {
1532       ShouldNotReachHere();
1533     }
1534   } else if (opr1->is_address()) {
1535     LIR_Address * addr = opr1->as_address_ptr();
1536     BasicType type = addr->type();
1537     assert (opr2->is_constant(), "Checking");
1538     if ( type == T_OBJECT ) __ ld_ptr(as_Address(addr), O7);
1539     else                    __ ld(as_Address(addr), O7);
1540     __ cmp(O7, opr2->as_constant_ptr()->as_jint());
1541   } else {
1542     ShouldNotReachHere();
1543   }
1544 }
1545 
1546 
1547 void LIR_Assembler::comp_fl2i(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dst, LIR_Op2* op){
1548   if (code == lir_cmp_fd2i || code == lir_ucmp_fd2i) {
1549     bool is_unordered_less = (code == lir_ucmp_fd2i);
1550     if (left->is_single_fpu()) {
1551       __ float_cmp(true, is_unordered_less ? -1 : 1, left->as_float_reg(), right->as_float_reg(), dst->as_register());
1552     } else if (left->is_double_fpu()) {
1553       __ float_cmp(false, is_unordered_less ? -1 : 1, left->as_double_reg(), right->as_double_reg(), dst->as_register());
1554     } else {
1555       ShouldNotReachHere();
1556     }
1557   } else if (code == lir_cmp_l2i) {
1558     __ lcmp(left->as_register_lo(), right->as_register_lo(), dst->as_register());
1559   } else {
1560     ShouldNotReachHere();
1561   }
1562 }
1563 
1564 
1565 void LIR_Assembler::cmove(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Opr result, BasicType type) {
1566   Assembler::Condition acond;
1567   switch (condition) {
1568     case lir_cond_equal:        acond = Assembler::equal;        break;
1569     case lir_cond_notEqual:     acond = Assembler::notEqual;     break;
1570     case lir_cond_less:         acond = Assembler::less;         break;
1571     case lir_cond_lessEqual:    acond = Assembler::lessEqual;    break;
1572     case lir_cond_greaterEqual: acond = Assembler::greaterEqual; break;
1573     case lir_cond_greater:      acond = Assembler::greater;      break;
1574     case lir_cond_aboveEqual:   acond = Assembler::greaterEqualUnsigned;      break;
1575     case lir_cond_belowEqual:   acond = Assembler::lessEqualUnsigned;      break;
1576     default:                         ShouldNotReachHere();
1577   };
1578 
1579   if (opr1->is_constant() && opr1->type() == T_INT) {
1580     Register dest = result->as_register();
1581     // load up first part of constant before branch
1582     // and do the rest in the delay slot.
1583     if (!Assembler::is_simm13(opr1->as_jint())) {
1584       __ sethi(opr1->as_jint(), dest);
1585     }
1586   } else if (opr1->is_constant()) {
1587     const2reg(opr1, result, lir_patch_none, NULL);
1588   } else if (opr1->is_register()) {
1589     reg2reg(opr1, result);
1590   } else if (opr1->is_stack()) {
1591     stack2reg(opr1, result, result->type());
1592   } else {
1593     ShouldNotReachHere();
1594   }
1595   Label skip;
1596     if  (type == T_INT) {
1597       __ br(acond, false, Assembler::pt, skip);
1598     } else {
1599       __ brx(acond, false, Assembler::pt, skip); // checks icc on 32bit and xcc on 64bit
1600     }
1601   if (opr1->is_constant() && opr1->type() == T_INT) {
1602     Register dest = result->as_register();
1603     if (Assembler::is_simm13(opr1->as_jint())) {
1604       __ delayed()->or3(G0, opr1->as_jint(), dest);
1605     } else {
1606       // the sethi has been done above, so just put in the low 10 bits
1607       __ delayed()->or3(dest, opr1->as_jint() & 0x3ff, dest);
1608     }
1609   } else {
1610     // can't do anything useful in the delay slot
1611     __ delayed()->nop();
1612   }
1613   if (opr2->is_constant()) {
1614     const2reg(opr2, result, lir_patch_none, NULL);
1615   } else if (opr2->is_register()) {
1616     reg2reg(opr2, result);
1617   } else if (opr2->is_stack()) {
1618     stack2reg(opr2, result, result->type());
1619   } else {
1620     ShouldNotReachHere();
1621   }
1622   __ bind(skip);
1623 }
1624 
1625 
1626 void LIR_Assembler::arith_op(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dest, CodeEmitInfo* info, bool pop_fpu_stack) {
1627   assert(info == NULL, "unused on this code path");
1628   assert(left->is_register(), "wrong items state");
1629   assert(dest->is_register(), "wrong items state");
1630 
1631   if (right->is_register()) {
1632     if (dest->is_float_kind()) {
1633 
1634       FloatRegister lreg, rreg, res;
1635       FloatRegisterImpl::Width w;
1636       if (right->is_single_fpu()) {
1637         w = FloatRegisterImpl::S;
1638         lreg = left->as_float_reg();
1639         rreg = right->as_float_reg();
1640         res  = dest->as_float_reg();
1641       } else {
1642         w = FloatRegisterImpl::D;
1643         lreg = left->as_double_reg();
1644         rreg = right->as_double_reg();
1645         res  = dest->as_double_reg();
1646       }
1647 
1648       switch (code) {
1649         case lir_add: __ fadd(w, lreg, rreg, res); break;
1650         case lir_sub: __ fsub(w, lreg, rreg, res); break;
1651         case lir_mul: // fall through
1652         case lir_mul_strictfp: __ fmul(w, lreg, rreg, res); break;
1653         case lir_div: // fall through
1654         case lir_div_strictfp: __ fdiv(w, lreg, rreg, res); break;
1655         default: ShouldNotReachHere();
1656       }
1657 
1658     } else if (dest->is_double_cpu()) {
1659       Register dst_lo = dest->as_register_lo();
1660       Register op1_lo = left->as_pointer_register();
1661       Register op2_lo = right->as_pointer_register();
1662 
1663       switch (code) {
1664         case lir_add:
1665           __ add(op1_lo, op2_lo, dst_lo);
1666           break;
1667 
1668         case lir_sub:
1669           __ sub(op1_lo, op2_lo, dst_lo);
1670           break;
1671 
1672         default: ShouldNotReachHere();
1673       }
1674     } else {
1675       assert (right->is_single_cpu(), "Just Checking");
1676 
1677       Register lreg = left->as_register();
1678       Register res  = dest->as_register();
1679       Register rreg = right->as_register();
1680       switch (code) {
1681         case lir_add:  __ add  (lreg, rreg, res); break;
1682         case lir_sub:  __ sub  (lreg, rreg, res); break;
1683         case lir_mul:  __ mulx (lreg, rreg, res); break;
1684         default: ShouldNotReachHere();
1685       }
1686     }
1687   } else {
1688     assert (right->is_constant(), "must be constant");
1689 
1690     if (dest->is_single_cpu()) {
1691       Register lreg = left->as_register();
1692       Register res  = dest->as_register();
1693       int    simm13 = right->as_constant_ptr()->as_jint();
1694 
1695       switch (code) {
1696         case lir_add:  __ add  (lreg, simm13, res); break;
1697         case lir_sub:  __ sub  (lreg, simm13, res); break;
1698         case lir_mul:  __ mulx (lreg, simm13, res); break;
1699         default: ShouldNotReachHere();
1700       }
1701     } else {
1702       Register lreg = left->as_pointer_register();
1703       Register res  = dest->as_register_lo();
1704       long con = right->as_constant_ptr()->as_jlong();
1705       assert(Assembler::is_simm13(con), "must be simm13");
1706 
1707       switch (code) {
1708         case lir_add:  __ add  (lreg, (int)con, res); break;
1709         case lir_sub:  __ sub  (lreg, (int)con, res); break;
1710         case lir_mul:  __ mulx (lreg, (int)con, res); break;
1711         default: ShouldNotReachHere();
1712       }
1713     }
1714   }
1715 }
1716 
1717 
1718 void LIR_Assembler::fpop() {
1719   // do nothing
1720 }
1721 
1722 
1723 void LIR_Assembler::intrinsic_op(LIR_Code code, LIR_Opr value, LIR_Opr thread, LIR_Opr dest, LIR_Op* op) {
1724   switch (code) {
1725     case lir_tan: {
1726       assert(thread->is_valid(), "preserve the thread object for performance reasons");
1727       assert(dest->as_double_reg() == F0, "the result will be in f0/f1");
1728       break;
1729     }
1730     case lir_sqrt: {
1731       assert(!thread->is_valid(), "there is no need for a thread_reg for dsqrt");
1732       FloatRegister src_reg = value->as_double_reg();
1733       FloatRegister dst_reg = dest->as_double_reg();
1734       __ fsqrt(FloatRegisterImpl::D, src_reg, dst_reg);
1735       break;
1736     }
1737     case lir_abs: {
1738       assert(!thread->is_valid(), "there is no need for a thread_reg for fabs");
1739       FloatRegister src_reg = value->as_double_reg();
1740       FloatRegister dst_reg = dest->as_double_reg();
1741       __ fabs(FloatRegisterImpl::D, src_reg, dst_reg);
1742       break;
1743     }
1744     default: {
1745       ShouldNotReachHere();
1746       break;
1747     }
1748   }
1749 }
1750 
1751 
1752 void LIR_Assembler::logic_op(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dest) {
1753   if (right->is_constant()) {
1754     if (dest->is_single_cpu()) {
1755       int simm13 = right->as_constant_ptr()->as_jint();
1756       switch (code) {
1757         case lir_logic_and:   __ and3 (left->as_register(), simm13, dest->as_register()); break;
1758         case lir_logic_or:    __ or3  (left->as_register(), simm13, dest->as_register()); break;
1759         case lir_logic_xor:   __ xor3 (left->as_register(), simm13, dest->as_register()); break;
1760         default: ShouldNotReachHere();
1761       }
1762     } else {
1763       long c = right->as_constant_ptr()->as_jlong();
1764       assert(c == (int)c && Assembler::is_simm13(c), "out of range");
1765       int simm13 = (int)c;
1766       switch (code) {
1767         case lir_logic_and:
1768           __ and3 (left->as_register_lo(), simm13, dest->as_register_lo());
1769           break;
1770 
1771         case lir_logic_or:
1772           __ or3 (left->as_register_lo(), simm13, dest->as_register_lo());
1773           break;
1774 
1775         case lir_logic_xor:
1776           __ xor3 (left->as_register_lo(), simm13, dest->as_register_lo());
1777           break;
1778 
1779         default: ShouldNotReachHere();
1780       }
1781     }
1782   } else {
1783     assert(right->is_register(), "right should be in register");
1784 
1785     if (dest->is_single_cpu()) {
1786       switch (code) {
1787         case lir_logic_and:   __ and3 (left->as_register(), right->as_register(), dest->as_register()); break;
1788         case lir_logic_or:    __ or3  (left->as_register(), right->as_register(), dest->as_register()); break;
1789         case lir_logic_xor:   __ xor3 (left->as_register(), right->as_register(), dest->as_register()); break;
1790         default: ShouldNotReachHere();
1791       }
1792     } else {
1793       Register l = (left->is_single_cpu() && left->is_oop_register()) ? left->as_register() :
1794                                                                         left->as_register_lo();
1795       Register r = (right->is_single_cpu() && right->is_oop_register()) ? right->as_register() :
1796                                                                           right->as_register_lo();
1797 
1798       switch (code) {
1799         case lir_logic_and: __ and3 (l, r, dest->as_register_lo()); break;
1800         case lir_logic_or:  __ or3  (l, r, dest->as_register_lo()); break;
1801         case lir_logic_xor: __ xor3 (l, r, dest->as_register_lo()); break;
1802         default: ShouldNotReachHere();
1803       }
1804     }
1805   }
1806 }
1807 
1808 
1809 int LIR_Assembler::shift_amount(BasicType t) {
1810   int elem_size = type2aelembytes(t);
1811   switch (elem_size) {
1812     case 1 : return 0;
1813     case 2 : return 1;
1814     case 4 : return 2;
1815     case 8 : return 3;
1816   }
1817   ShouldNotReachHere();
1818   return -1;
1819 }
1820 
1821 
1822 void LIR_Assembler::throw_op(LIR_Opr exceptionPC, LIR_Opr exceptionOop, CodeEmitInfo* info) {
1823   assert(exceptionOop->as_register() == Oexception, "should match");
1824   assert(exceptionPC->as_register() == Oissuing_pc, "should match");
1825 
1826   info->add_register_oop(exceptionOop);
1827 
1828   // reuse the debug info from the safepoint poll for the throw op itself
1829   address pc_for_athrow  = __ pc();
1830   int pc_for_athrow_offset = __ offset();
1831   RelocationHolder rspec = internal_word_Relocation::spec(pc_for_athrow);
1832   __ set(pc_for_athrow, Oissuing_pc, rspec);
1833   add_call_info(pc_for_athrow_offset, info); // for exception handler
1834 
1835   __ call(Runtime1::entry_for(Runtime1::handle_exception_id), relocInfo::runtime_call_type);
1836   __ delayed()->nop();
1837 }
1838 
1839 
1840 void LIR_Assembler::unwind_op(LIR_Opr exceptionOop) {
1841   assert(exceptionOop->as_register() == Oexception, "should match");
1842 
1843   __ br(Assembler::always, false, Assembler::pt, _unwind_handler_entry);
1844   __ delayed()->nop();
1845 }
1846 
1847 void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) {
1848   Register src = op->src()->as_register();
1849   Register dst = op->dst()->as_register();
1850   Register src_pos = op->src_pos()->as_register();
1851   Register dst_pos = op->dst_pos()->as_register();
1852   Register length  = op->length()->as_register();
1853   Register tmp = op->tmp()->as_register();
1854   Register tmp2 = O7;
1855 
1856   int flags = op->flags();
1857   ciArrayKlass* default_type = op->expected_type();
1858   BasicType basic_type = default_type != NULL ? default_type->element_type()->basic_type() : T_ILLEGAL;
1859   if (basic_type == T_ARRAY) basic_type = T_OBJECT;
1860 
1861   // higher 32bits must be null
1862   __ sra(dst_pos, 0, dst_pos);
1863   __ sra(src_pos, 0, src_pos);
1864   __ sra(length, 0, length);
1865 
1866   // set up the arraycopy stub information
1867   ArrayCopyStub* stub = op->stub();
1868 
1869   // always do stub if no type information is available.  it's ok if
1870   // the known type isn't loaded since the code sanity checks
1871   // in debug mode and the type isn't required when we know the exact type
1872   // also check that the type is an array type.
1873   if (op->expected_type() == NULL) {
1874     __ mov(src,     O0);
1875     __ mov(src_pos, O1);
1876     __ mov(dst,     O2);
1877     __ mov(dst_pos, O3);
1878     __ mov(length,  O4);
1879     address copyfunc_addr = StubRoutines::generic_arraycopy();
1880 
1881     if (copyfunc_addr == NULL) { // Use C version if stub was not generated
1882       __ call_VM_leaf(tmp, CAST_FROM_FN_PTR(address, Runtime1::arraycopy));
1883     } else {
1884 #ifndef PRODUCT
1885       if (PrintC1Statistics) {
1886         address counter = (address)&Runtime1::_generic_arraycopystub_cnt;
1887         __ inc_counter(counter, G1, G3);
1888       }
1889 #endif
1890       __ call_VM_leaf(tmp, copyfunc_addr);
1891     }
1892 
1893     if (copyfunc_addr != NULL) {
1894       __ xor3(O0, -1, tmp);
1895       __ sub(length, tmp, length);
1896       __ add(src_pos, tmp, src_pos);
1897       __ cmp_zero_and_br(Assembler::less, O0, *stub->entry());
1898       __ delayed()->add(dst_pos, tmp, dst_pos);
1899     } else {
1900       __ cmp_zero_and_br(Assembler::less, O0, *stub->entry());
1901       __ delayed()->nop();
1902     }
1903     __ bind(*stub->continuation());
1904     return;
1905   }
1906 
1907   assert(default_type != NULL && default_type->is_array_klass(), "must be true at this point");
1908 
1909   // make sure src and dst are non-null and load array length
1910   if (flags & LIR_OpArrayCopy::src_null_check) {
1911     __ tst(src);
1912     __ brx(Assembler::equal, false, Assembler::pn, *stub->entry());
1913     __ delayed()->nop();
1914   }
1915 
1916   if (flags & LIR_OpArrayCopy::dst_null_check) {
1917     __ tst(dst);
1918     __ brx(Assembler::equal, false, Assembler::pn, *stub->entry());
1919     __ delayed()->nop();
1920   }
1921 
1922   // If the compiler was not able to prove that exact type of the source or the destination
1923   // of the arraycopy is an array type, check at runtime if the source or the destination is
1924   // an instance type.
1925   if (flags & LIR_OpArrayCopy::type_check) {
1926     if (!(flags & LIR_OpArrayCopy::LIR_OpArrayCopy::dst_objarray)) {
1927       __ load_klass(dst, tmp);
1928       __ lduw(tmp, in_bytes(Klass::layout_helper_offset()), tmp2);
1929       __ cmp(tmp2, Klass::_lh_neutral_value);
1930       __ br(Assembler::greaterEqual, false, Assembler::pn, *stub->entry());
1931       __ delayed()->nop();
1932     }
1933 
1934     if (!(flags & LIR_OpArrayCopy::LIR_OpArrayCopy::src_objarray)) {
1935       __ load_klass(src, tmp);
1936       __ lduw(tmp, in_bytes(Klass::layout_helper_offset()), tmp2);
1937       __ cmp(tmp2, Klass::_lh_neutral_value);
1938       __ br(Assembler::greaterEqual, false, Assembler::pn, *stub->entry());
1939       __ delayed()->nop();
1940     }
1941   }
1942 
1943   if (flags & LIR_OpArrayCopy::src_pos_positive_check) {
1944     // test src_pos register
1945     __ cmp_zero_and_br(Assembler::less, src_pos, *stub->entry());
1946     __ delayed()->nop();
1947   }
1948 
1949   if (flags & LIR_OpArrayCopy::dst_pos_positive_check) {
1950     // test dst_pos register
1951     __ cmp_zero_and_br(Assembler::less, dst_pos, *stub->entry());
1952     __ delayed()->nop();
1953   }
1954 
1955   if (flags & LIR_OpArrayCopy::length_positive_check) {
1956     // make sure length isn't negative
1957     __ cmp_zero_and_br(Assembler::less, length, *stub->entry());
1958     __ delayed()->nop();
1959   }
1960 
1961   if (flags & LIR_OpArrayCopy::src_range_check) {
1962     __ ld(src, arrayOopDesc::length_offset_in_bytes(), tmp2);
1963     __ add(length, src_pos, tmp);
1964     __ cmp(tmp2, tmp);
1965     __ br(Assembler::carrySet, false, Assembler::pn, *stub->entry());
1966     __ delayed()->nop();
1967   }
1968 
1969   if (flags & LIR_OpArrayCopy::dst_range_check) {
1970     __ ld(dst, arrayOopDesc::length_offset_in_bytes(), tmp2);
1971     __ add(length, dst_pos, tmp);
1972     __ cmp(tmp2, tmp);
1973     __ br(Assembler::carrySet, false, Assembler::pn, *stub->entry());
1974     __ delayed()->nop();
1975   }
1976 
1977   int shift = shift_amount(basic_type);
1978 
1979   if (flags & LIR_OpArrayCopy::type_check) {
1980     // We don't know the array types are compatible
1981     if (basic_type != T_OBJECT) {
1982       // Simple test for basic type arrays
1983       if (UseCompressedClassPointers) {
1984         // We don't need decode because we just need to compare
1985         __ lduw(src, oopDesc::klass_offset_in_bytes(), tmp);
1986         __ lduw(dst, oopDesc::klass_offset_in_bytes(), tmp2);
1987         __ cmp(tmp, tmp2);
1988         __ br(Assembler::notEqual, false, Assembler::pt, *stub->entry());
1989       } else {
1990         __ ld_ptr(src, oopDesc::klass_offset_in_bytes(), tmp);
1991         __ ld_ptr(dst, oopDesc::klass_offset_in_bytes(), tmp2);
1992         __ cmp(tmp, tmp2);
1993         __ brx(Assembler::notEqual, false, Assembler::pt, *stub->entry());
1994       }
1995       __ delayed()->nop();
1996     } else {
1997       // For object arrays, if src is a sub class of dst then we can
1998       // safely do the copy.
1999       address copyfunc_addr = StubRoutines::checkcast_arraycopy();
2000 
2001       Label cont, slow;
2002       assert_different_registers(tmp, tmp2, G3, G1);
2003 
2004       __ load_klass(src, G3);
2005       __ load_klass(dst, G1);
2006 
2007       __ check_klass_subtype_fast_path(G3, G1, tmp, tmp2, &cont, copyfunc_addr == NULL ? stub->entry() : &slow, NULL);
2008 
2009       __ call(Runtime1::entry_for(Runtime1::slow_subtype_check_id), relocInfo::runtime_call_type);
2010       __ delayed()->nop();
2011 
2012       __ cmp(G3, 0);
2013       if (copyfunc_addr != NULL) { // use stub if available
2014         // src is not a sub class of dst so we have to do a
2015         // per-element check.
2016         __ br(Assembler::notEqual, false, Assembler::pt, cont);
2017         __ delayed()->nop();
2018 
2019         __ bind(slow);
2020 
2021         int mask = LIR_OpArrayCopy::src_objarray|LIR_OpArrayCopy::dst_objarray;
2022         if ((flags & mask) != mask) {
2023           // Check that at least both of them object arrays.
2024           assert(flags & mask, "one of the two should be known to be an object array");
2025 
2026           if (!(flags & LIR_OpArrayCopy::src_objarray)) {
2027             __ load_klass(src, tmp);
2028           } else if (!(flags & LIR_OpArrayCopy::dst_objarray)) {
2029             __ load_klass(dst, tmp);
2030           }
2031           int lh_offset = in_bytes(Klass::layout_helper_offset());
2032 
2033           __ lduw(tmp, lh_offset, tmp2);
2034 
2035           jint objArray_lh = Klass::array_layout_helper(T_OBJECT);
2036           __ set(objArray_lh, tmp);
2037           __ cmp(tmp, tmp2);
2038           __ br(Assembler::notEqual, false, Assembler::pt,  *stub->entry());
2039           __ delayed()->nop();
2040         }
2041 
2042         Register src_ptr = O0;
2043         Register dst_ptr = O1;
2044         Register len     = O2;
2045         Register chk_off = O3;
2046         Register super_k = O4;
2047 
2048         __ add(src, arrayOopDesc::base_offset_in_bytes(basic_type), src_ptr);
2049         if (shift == 0) {
2050           __ add(src_ptr, src_pos, src_ptr);
2051         } else {
2052           __ sll(src_pos, shift, tmp);
2053           __ add(src_ptr, tmp, src_ptr);
2054         }
2055 
2056         __ add(dst, arrayOopDesc::base_offset_in_bytes(basic_type), dst_ptr);
2057         if (shift == 0) {
2058           __ add(dst_ptr, dst_pos, dst_ptr);
2059         } else {
2060           __ sll(dst_pos, shift, tmp);
2061           __ add(dst_ptr, tmp, dst_ptr);
2062         }
2063         __ mov(length, len);
2064         __ load_klass(dst, tmp);
2065 
2066         int ek_offset = in_bytes(ObjArrayKlass::element_klass_offset());
2067         __ ld_ptr(tmp, ek_offset, super_k);
2068 
2069         int sco_offset = in_bytes(Klass::super_check_offset_offset());
2070         __ lduw(super_k, sco_offset, chk_off);
2071 
2072         __ call_VM_leaf(tmp, copyfunc_addr);
2073 
2074 #ifndef PRODUCT
2075         if (PrintC1Statistics) {
2076           Label failed;
2077           __ br_notnull_short(O0, Assembler::pn, failed);
2078           __ inc_counter((address)&Runtime1::_arraycopy_checkcast_cnt, G1, G3);
2079           __ bind(failed);
2080         }
2081 #endif
2082 
2083         __ br_null(O0, false, Assembler::pt,  *stub->continuation());
2084         __ delayed()->xor3(O0, -1, tmp);
2085 
2086 #ifndef PRODUCT
2087         if (PrintC1Statistics) {
2088           __ inc_counter((address)&Runtime1::_arraycopy_checkcast_attempt_cnt, G1, G3);
2089         }
2090 #endif
2091 
2092         __ sub(length, tmp, length);
2093         __ add(src_pos, tmp, src_pos);
2094         __ br(Assembler::always, false, Assembler::pt, *stub->entry());
2095         __ delayed()->add(dst_pos, tmp, dst_pos);
2096 
2097         __ bind(cont);
2098       } else {
2099         __ br(Assembler::equal, false, Assembler::pn, *stub->entry());
2100         __ delayed()->nop();
2101         __ bind(cont);
2102       }
2103     }
2104   }
2105 
2106 #ifdef ASSERT
2107   if (basic_type != T_OBJECT || !(flags & LIR_OpArrayCopy::type_check)) {
2108     // Sanity check the known type with the incoming class.  For the
2109     // primitive case the types must match exactly with src.klass and
2110     // dst.klass each exactly matching the default type.  For the
2111     // object array case, if no type check is needed then either the
2112     // dst type is exactly the expected type and the src type is a
2113     // subtype which we can't check or src is the same array as dst
2114     // but not necessarily exactly of type default_type.
2115     Label known_ok, halt;
2116     metadata2reg(op->expected_type()->constant_encoding(), tmp);
2117     if (UseCompressedClassPointers) {
2118       // tmp holds the default type. It currently comes uncompressed after the
2119       // load of a constant, so encode it.
2120       __ encode_klass_not_null(tmp);
2121       // load the raw value of the dst klass, since we will be comparing
2122       // uncompressed values directly.
2123       __ lduw(dst, oopDesc::klass_offset_in_bytes(), tmp2);
2124       if (basic_type != T_OBJECT) {
2125         __ cmp(tmp, tmp2);
2126         __ br(Assembler::notEqual, false, Assembler::pn, halt);
2127         // load the raw value of the src klass.
2128         __ delayed()->lduw(src, oopDesc::klass_offset_in_bytes(), tmp2);
2129         __ cmp_and_br_short(tmp, tmp2, Assembler::equal, Assembler::pn, known_ok);
2130       } else {
2131         __ cmp(tmp, tmp2);
2132         __ br(Assembler::equal, false, Assembler::pn, known_ok);
2133         __ delayed()->cmp(src, dst);
2134         __ brx(Assembler::equal, false, Assembler::pn, known_ok);
2135         __ delayed()->nop();
2136       }
2137     } else {
2138       __ ld_ptr(dst, oopDesc::klass_offset_in_bytes(), tmp2);
2139       if (basic_type != T_OBJECT) {
2140         __ cmp(tmp, tmp2);
2141         __ brx(Assembler::notEqual, false, Assembler::pn, halt);
2142         __ delayed()->ld_ptr(src, oopDesc::klass_offset_in_bytes(), tmp2);
2143         __ cmp_and_brx_short(tmp, tmp2, Assembler::equal, Assembler::pn, known_ok);
2144       } else {
2145         __ cmp(tmp, tmp2);
2146         __ brx(Assembler::equal, false, Assembler::pn, known_ok);
2147         __ delayed()->cmp(src, dst);
2148         __ brx(Assembler::equal, false, Assembler::pn, known_ok);
2149         __ delayed()->nop();
2150       }
2151     }
2152     __ bind(halt);
2153     __ stop("incorrect type information in arraycopy");
2154     __ bind(known_ok);
2155   }
2156 #endif
2157 
2158 #ifndef PRODUCT
2159   if (PrintC1Statistics) {
2160     address counter = Runtime1::arraycopy_count_address(basic_type);
2161     __ inc_counter(counter, G1, G3);
2162   }
2163 #endif
2164 
2165   Register src_ptr = O0;
2166   Register dst_ptr = O1;
2167   Register len     = O2;
2168 
2169   __ add(src, arrayOopDesc::base_offset_in_bytes(basic_type), src_ptr);
2170   if (shift == 0) {
2171     __ add(src_ptr, src_pos, src_ptr);
2172   } else {
2173     __ sll(src_pos, shift, tmp);
2174     __ add(src_ptr, tmp, src_ptr);
2175   }
2176 
2177   __ add(dst, arrayOopDesc::base_offset_in_bytes(basic_type), dst_ptr);
2178   if (shift == 0) {
2179     __ add(dst_ptr, dst_pos, dst_ptr);
2180   } else {
2181     __ sll(dst_pos, shift, tmp);
2182     __ add(dst_ptr, tmp, dst_ptr);
2183   }
2184 
2185   bool disjoint = (flags & LIR_OpArrayCopy::overlapping) == 0;
2186   bool aligned = (flags & LIR_OpArrayCopy::unaligned) == 0;
2187   const char *name;
2188   address entry = StubRoutines::select_arraycopy_function(basic_type, aligned, disjoint, name, false);
2189 
2190   // arraycopy stubs takes a length in number of elements, so don't scale it.
2191   __ mov(length, len);
2192   __ call_VM_leaf(tmp, entry);
2193 
2194   __ bind(*stub->continuation());
2195 }
2196 
2197 
2198 void LIR_Assembler::shift_op(LIR_Code code, LIR_Opr left, LIR_Opr count, LIR_Opr dest, LIR_Opr tmp) {
2199   if (dest->is_single_cpu()) {
2200     if (left->type() == T_OBJECT) {
2201       switch (code) {
2202         case lir_shl:  __ sllx  (left->as_register(), count->as_register(), dest->as_register()); break;
2203         case lir_shr:  __ srax  (left->as_register(), count->as_register(), dest->as_register()); break;
2204         case lir_ushr: __ srl   (left->as_register(), count->as_register(), dest->as_register()); break;
2205         default: ShouldNotReachHere();
2206       }
2207     } else
2208       switch (code) {
2209         case lir_shl:  __ sll   (left->as_register(), count->as_register(), dest->as_register()); break;
2210         case lir_shr:  __ sra   (left->as_register(), count->as_register(), dest->as_register()); break;
2211         case lir_ushr: __ srl   (left->as_register(), count->as_register(), dest->as_register()); break;
2212         default: ShouldNotReachHere();
2213       }
2214   } else {
2215     switch (code) {
2216       case lir_shl:  __ sllx  (left->as_register_lo(), count->as_register(), dest->as_register_lo()); break;
2217       case lir_shr:  __ srax  (left->as_register_lo(), count->as_register(), dest->as_register_lo()); break;
2218       case lir_ushr: __ srlx  (left->as_register_lo(), count->as_register(), dest->as_register_lo()); break;
2219       default: ShouldNotReachHere();
2220     }
2221   }
2222 }
2223 
2224 
2225 void LIR_Assembler::shift_op(LIR_Code code, LIR_Opr left, jint count, LIR_Opr dest) {
2226   if (left->type() == T_OBJECT) {
2227     count = count & 63;  // shouldn't shift by more than sizeof(intptr_t)
2228     Register l = left->as_register();
2229     Register d = dest->as_register_lo();
2230     switch (code) {
2231       case lir_shl:  __ sllx  (l, count, d); break;
2232       case lir_shr:  __ srax  (l, count, d); break;
2233       case lir_ushr: __ srlx  (l, count, d); break;
2234       default: ShouldNotReachHere();
2235     }
2236     return;
2237   }
2238 
2239   if (dest->is_single_cpu()) {
2240     count = count & 0x1F; // Java spec
2241     switch (code) {
2242       case lir_shl:  __ sll   (left->as_register(), count, dest->as_register()); break;
2243       case lir_shr:  __ sra   (left->as_register(), count, dest->as_register()); break;
2244       case lir_ushr: __ srl   (left->as_register(), count, dest->as_register()); break;
2245       default: ShouldNotReachHere();
2246     }
2247   } else if (dest->is_double_cpu()) {
2248     count = count & 63; // Java spec
2249     switch (code) {
2250       case lir_shl:  __ sllx  (left->as_pointer_register(), count, dest->as_pointer_register()); break;
2251       case lir_shr:  __ srax  (left->as_pointer_register(), count, dest->as_pointer_register()); break;
2252       case lir_ushr: __ srlx  (left->as_pointer_register(), count, dest->as_pointer_register()); break;
2253       default: ShouldNotReachHere();
2254     }
2255   } else {
2256     ShouldNotReachHere();
2257   }
2258 }
2259 
2260 
2261 void LIR_Assembler::emit_alloc_obj(LIR_OpAllocObj* op) {
2262   assert(op->tmp1()->as_register()  == G1 &&
2263          op->tmp2()->as_register()  == G3 &&
2264          op->tmp3()->as_register()  == G4 &&
2265          op->obj()->as_register()   == O0 &&
2266          op->klass()->as_register() == G5, "must be");
2267   if (op->init_check()) {
2268     __ ldub(op->klass()->as_register(),
2269           in_bytes(InstanceKlass::init_state_offset()),
2270           op->tmp1()->as_register());
2271     add_debug_info_for_null_check_here(op->stub()->info());
2272     __ cmp(op->tmp1()->as_register(), InstanceKlass::fully_initialized);
2273     __ br(Assembler::notEqual, false, Assembler::pn, *op->stub()->entry());
2274     __ delayed()->nop();
2275   }
2276   __ allocate_object(op->obj()->as_register(),
2277                      op->tmp1()->as_register(),
2278                      op->tmp2()->as_register(),
2279                      op->tmp3()->as_register(),
2280                      op->header_size(),
2281                      op->object_size(),
2282                      op->klass()->as_register(),
2283                      *op->stub()->entry());
2284   __ bind(*op->stub()->continuation());
2285   __ verify_oop(op->obj()->as_register());
2286 }
2287 
2288 
2289 void LIR_Assembler::emit_alloc_array(LIR_OpAllocArray* op) {
2290   assert(op->tmp1()->as_register()  == G1 &&
2291          op->tmp2()->as_register()  == G3 &&
2292          op->tmp3()->as_register()  == G4 &&
2293          op->tmp4()->as_register()  == O1 &&
2294          op->klass()->as_register() == G5, "must be");
2295 
2296   __ signx(op->len()->as_register());
2297   if (UseSlowPath ||
2298       (!UseFastNewObjectArray && (op->type() == T_OBJECT || op->type() == T_ARRAY)) ||
2299       (!UseFastNewTypeArray   && (op->type() != T_OBJECT && op->type() != T_ARRAY))) {
2300     __ br(Assembler::always, false, Assembler::pt, *op->stub()->entry());
2301     __ delayed()->nop();
2302   } else {
2303     __ allocate_array(op->obj()->as_register(),
2304                       op->len()->as_register(),
2305                       op->tmp1()->as_register(),
2306                       op->tmp2()->as_register(),
2307                       op->tmp3()->as_register(),
2308                       arrayOopDesc::header_size(op->type()),
2309                       type2aelembytes(op->type()),
2310                       op->klass()->as_register(),
2311                       *op->stub()->entry());
2312   }
2313   __ bind(*op->stub()->continuation());
2314 }
2315 
2316 
2317 void LIR_Assembler::type_profile_helper(Register mdo, int mdo_offset_bias,
2318                                         ciMethodData *md, ciProfileData *data,
2319                                         Register recv, Register tmp1, Label* update_done) {
2320   uint i;
2321   for (i = 0; i < VirtualCallData::row_limit(); i++) {
2322     Label next_test;
2323     // See if the receiver is receiver[n].
2324     Address receiver_addr(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_offset(i)) -
2325                           mdo_offset_bias);
2326     __ ld_ptr(receiver_addr, tmp1);
2327     __ verify_klass_ptr(tmp1);
2328     __ cmp_and_brx_short(recv, tmp1, Assembler::notEqual, Assembler::pt, next_test);
2329     Address data_addr(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_count_offset(i)) -
2330                       mdo_offset_bias);
2331     __ ld_ptr(data_addr, tmp1);
2332     __ add(tmp1, DataLayout::counter_increment, tmp1);
2333     __ st_ptr(tmp1, data_addr);
2334     __ ba(*update_done);
2335     __ delayed()->nop();
2336     __ bind(next_test);
2337   }
2338 
2339   // Didn't find receiver; find next empty slot and fill it in
2340   for (i = 0; i < VirtualCallData::row_limit(); i++) {
2341     Label next_test;
2342     Address recv_addr(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_offset(i)) -
2343                       mdo_offset_bias);
2344     __ ld_ptr(recv_addr, tmp1);
2345     __ br_notnull_short(tmp1, Assembler::pt, next_test);
2346     __ st_ptr(recv, recv_addr);
2347     __ set(DataLayout::counter_increment, tmp1);
2348     __ st_ptr(tmp1, mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_count_offset(i)) -
2349               mdo_offset_bias);
2350     __ ba(*update_done);
2351     __ delayed()->nop();
2352     __ bind(next_test);
2353   }
2354 }
2355 
2356 
2357 void LIR_Assembler::setup_md_access(ciMethod* method, int bci,
2358                                     ciMethodData*& md, ciProfileData*& data, int& mdo_offset_bias) {
2359   md = method->method_data_or_null();
2360   assert(md != NULL, "Sanity");
2361   data = md->bci_to_data(bci);
2362   assert(data != NULL,       "need data for checkcast");
2363   assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
2364   if (!Assembler::is_simm13(md->byte_offset_of_slot(data, DataLayout::header_offset()) + data->size_in_bytes())) {
2365     // The offset is large so bias the mdo by the base of the slot so
2366     // that the ld can use simm13s to reference the slots of the data
2367     mdo_offset_bias = md->byte_offset_of_slot(data, DataLayout::header_offset());
2368   }
2369 }
2370 
2371 void LIR_Assembler::emit_typecheck_helper(LIR_OpTypeCheck *op, Label* success, Label* failure, Label* obj_is_null) {
2372   // we always need a stub for the failure case.
2373   CodeStub* stub = op->stub();
2374   Register obj = op->object()->as_register();
2375   Register k_RInfo = op->tmp1()->as_register();
2376   Register klass_RInfo = op->tmp2()->as_register();
2377   Register dst = op->result_opr()->as_register();
2378   Register Rtmp1 = op->tmp3()->as_register();
2379   ciKlass* k = op->klass();
2380 
2381 
2382   if (obj == k_RInfo) {
2383     k_RInfo = klass_RInfo;
2384     klass_RInfo = obj;
2385   }
2386 
2387   ciMethodData* md;
2388   ciProfileData* data;
2389   int mdo_offset_bias = 0;
2390   if (op->should_profile()) {
2391     ciMethod* method = op->profiled_method();
2392     assert(method != NULL, "Should have method");
2393     setup_md_access(method, op->profiled_bci(), md, data, mdo_offset_bias);
2394 
2395     Label not_null;
2396     __ br_notnull_short(obj, Assembler::pn, not_null);
2397     Register mdo      = k_RInfo;
2398     Register data_val = Rtmp1;
2399     metadata2reg(md->constant_encoding(), mdo);
2400     if (mdo_offset_bias > 0) {
2401       __ set(mdo_offset_bias, data_val);
2402       __ add(mdo, data_val, mdo);
2403     }
2404     Address flags_addr(mdo, md->byte_offset_of_slot(data, DataLayout::flags_offset()) - mdo_offset_bias);
2405     __ ldub(flags_addr, data_val);
2406     __ or3(data_val, BitData::null_seen_byte_constant(), data_val);
2407     __ stb(data_val, flags_addr);
2408     __ ba(*obj_is_null);
2409     __ delayed()->nop();
2410     __ bind(not_null);
2411   } else {
2412     __ br_null(obj, false, Assembler::pn, *obj_is_null);
2413     __ delayed()->nop();
2414   }
2415 
2416   Label profile_cast_failure, profile_cast_success;
2417   Label *failure_target = op->should_profile() ? &profile_cast_failure : failure;
2418   Label *success_target = op->should_profile() ? &profile_cast_success : success;
2419 
2420   // patching may screw with our temporaries on sparc,
2421   // so let's do it before loading the class
2422   if (k->is_loaded()) {
2423     metadata2reg(k->constant_encoding(), k_RInfo);
2424   } else {
2425     klass2reg_with_patching(k_RInfo, op->info_for_patch());
2426   }
2427   assert(obj != k_RInfo, "must be different");
2428 
2429   // get object class
2430   // not a safepoint as obj null check happens earlier
2431   __ load_klass(obj, klass_RInfo);
2432   if (op->fast_check()) {
2433     assert_different_registers(klass_RInfo, k_RInfo);
2434     __ cmp(k_RInfo, klass_RInfo);
2435     __ brx(Assembler::notEqual, false, Assembler::pt, *failure_target);
2436     __ delayed()->nop();
2437   } else {
2438     bool need_slow_path = true;
2439     if (k->is_loaded()) {
2440       if ((int) k->super_check_offset() != in_bytes(Klass::secondary_super_cache_offset()))
2441         need_slow_path = false;
2442       // perform the fast part of the checking logic
2443       __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, noreg,
2444                                        (need_slow_path ? success_target : NULL),
2445                                        failure_target, NULL,
2446                                        RegisterOrConstant(k->super_check_offset()));
2447     } else {
2448       // perform the fast part of the checking logic
2449       __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, O7, success_target,
2450                                        failure_target, NULL);
2451     }
2452     if (need_slow_path) {
2453       // call out-of-line instance of __ check_klass_subtype_slow_path(...):
2454       assert(klass_RInfo == G3 && k_RInfo == G1, "incorrect call setup");
2455       __ call(Runtime1::entry_for(Runtime1::slow_subtype_check_id), relocInfo::runtime_call_type);
2456       __ delayed()->nop();
2457       __ cmp(G3, 0);
2458       __ br(Assembler::equal, false, Assembler::pn, *failure_target);
2459       __ delayed()->nop();
2460       // Fall through to success case
2461     }
2462   }
2463 
2464   if (op->should_profile()) {
2465     Register mdo  = klass_RInfo, recv = k_RInfo, tmp1 = Rtmp1;
2466     assert_different_registers(obj, mdo, recv, tmp1);
2467     __ bind(profile_cast_success);
2468     metadata2reg(md->constant_encoding(), mdo);
2469     if (mdo_offset_bias > 0) {
2470       __ set(mdo_offset_bias, tmp1);
2471       __ add(mdo, tmp1, mdo);
2472     }
2473     __ load_klass(obj, recv);
2474     type_profile_helper(mdo, mdo_offset_bias, md, data, recv, tmp1, success);
2475     // Jump over the failure case
2476     __ ba(*success);
2477     __ delayed()->nop();
2478     // Cast failure case
2479     __ bind(profile_cast_failure);
2480     metadata2reg(md->constant_encoding(), mdo);
2481     if (mdo_offset_bias > 0) {
2482       __ set(mdo_offset_bias, tmp1);
2483       __ add(mdo, tmp1, mdo);
2484     }
2485     Address data_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()) - mdo_offset_bias);
2486     __ ld_ptr(data_addr, tmp1);
2487     __ sub(tmp1, DataLayout::counter_increment, tmp1);
2488     __ st_ptr(tmp1, data_addr);
2489     __ ba(*failure);
2490     __ delayed()->nop();
2491   }
2492   __ ba(*success);
2493   __ delayed()->nop();
2494 }
2495 
2496 void LIR_Assembler::emit_opTypeCheck(LIR_OpTypeCheck* op) {
2497   LIR_Code code = op->code();
2498   if (code == lir_store_check) {
2499     Register value = op->object()->as_register();
2500     Register array = op->array()->as_register();
2501     Register k_RInfo = op->tmp1()->as_register();
2502     Register klass_RInfo = op->tmp2()->as_register();
2503     Register Rtmp1 = op->tmp3()->as_register();
2504 
2505     __ verify_oop(value);
2506     CodeStub* stub = op->stub();
2507     // check if it needs to be profiled
2508     ciMethodData* md;
2509     ciProfileData* data;
2510     int mdo_offset_bias = 0;
2511     if (op->should_profile()) {
2512       ciMethod* method = op->profiled_method();
2513       assert(method != NULL, "Should have method");
2514       setup_md_access(method, op->profiled_bci(), md, data, mdo_offset_bias);
2515     }
2516     Label profile_cast_success, profile_cast_failure, done;
2517     Label *success_target = op->should_profile() ? &profile_cast_success : &done;
2518     Label *failure_target = op->should_profile() ? &profile_cast_failure : stub->entry();
2519 
2520     if (op->should_profile()) {
2521       Label not_null;
2522       __ br_notnull_short(value, Assembler::pn, not_null);
2523       Register mdo      = k_RInfo;
2524       Register data_val = Rtmp1;
2525       metadata2reg(md->constant_encoding(), mdo);
2526       if (mdo_offset_bias > 0) {
2527         __ set(mdo_offset_bias, data_val);
2528         __ add(mdo, data_val, mdo);
2529       }
2530       Address flags_addr(mdo, md->byte_offset_of_slot(data, DataLayout::flags_offset()) - mdo_offset_bias);
2531       __ ldub(flags_addr, data_val);
2532       __ or3(data_val, BitData::null_seen_byte_constant(), data_val);
2533       __ stb(data_val, flags_addr);
2534       __ ba_short(done);
2535       __ bind(not_null);
2536     } else {
2537       __ br_null_short(value, Assembler::pn, done);
2538     }
2539     add_debug_info_for_null_check_here(op->info_for_exception());
2540     __ load_klass(array, k_RInfo);
2541     __ load_klass(value, klass_RInfo);
2542 
2543     // get instance klass
2544     __ ld_ptr(Address(k_RInfo, ObjArrayKlass::element_klass_offset()), k_RInfo);
2545     // perform the fast part of the checking logic
2546     __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, O7, success_target, failure_target, NULL);
2547 
2548     // call out-of-line instance of __ check_klass_subtype_slow_path(...):
2549     assert(klass_RInfo == G3 && k_RInfo == G1, "incorrect call setup");
2550     __ call(Runtime1::entry_for(Runtime1::slow_subtype_check_id), relocInfo::runtime_call_type);
2551     __ delayed()->nop();
2552     __ cmp(G3, 0);
2553     __ br(Assembler::equal, false, Assembler::pn, *failure_target);
2554     __ delayed()->nop();
2555     // fall through to the success case
2556 
2557     if (op->should_profile()) {
2558       Register mdo  = klass_RInfo, recv = k_RInfo, tmp1 = Rtmp1;
2559       assert_different_registers(value, mdo, recv, tmp1);
2560       __ bind(profile_cast_success);
2561       metadata2reg(md->constant_encoding(), mdo);
2562       if (mdo_offset_bias > 0) {
2563         __ set(mdo_offset_bias, tmp1);
2564         __ add(mdo, tmp1, mdo);
2565       }
2566       __ load_klass(value, recv);
2567       type_profile_helper(mdo, mdo_offset_bias, md, data, recv, tmp1, &done);
2568       __ ba_short(done);
2569       // Cast failure case
2570       __ bind(profile_cast_failure);
2571       metadata2reg(md->constant_encoding(), mdo);
2572       if (mdo_offset_bias > 0) {
2573         __ set(mdo_offset_bias, tmp1);
2574         __ add(mdo, tmp1, mdo);
2575       }
2576       Address data_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()) - mdo_offset_bias);
2577       __ ld_ptr(data_addr, tmp1);
2578       __ sub(tmp1, DataLayout::counter_increment, tmp1);
2579       __ st_ptr(tmp1, data_addr);
2580       __ ba(*stub->entry());
2581       __ delayed()->nop();
2582     }
2583     __ bind(done);
2584   } else if (code == lir_checkcast) {
2585     Register obj = op->object()->as_register();
2586     Register dst = op->result_opr()->as_register();
2587     Label success;
2588     emit_typecheck_helper(op, &success, op->stub()->entry(), &success);
2589     __ bind(success);
2590     __ mov(obj, dst);
2591   } else if (code == lir_instanceof) {
2592     Register obj = op->object()->as_register();
2593     Register dst = op->result_opr()->as_register();
2594     Label success, failure, done;
2595     emit_typecheck_helper(op, &success, &failure, &failure);
2596     __ bind(failure);
2597     __ set(0, dst);
2598     __ ba_short(done);
2599     __ bind(success);
2600     __ set(1, dst);
2601     __ bind(done);
2602   } else {
2603     ShouldNotReachHere();
2604   }
2605 
2606 }
2607 
2608 
2609 void LIR_Assembler::emit_compare_and_swap(LIR_OpCompareAndSwap* op) {
2610   if (op->code() == lir_cas_long) {
2611     assert(VM_Version::supports_cx8(), "wrong machine");
2612     Register addr = op->addr()->as_pointer_register();
2613     Register cmp_value_lo = op->cmp_value()->as_register_lo();
2614     Register cmp_value_hi = op->cmp_value()->as_register_hi();
2615     Register new_value_lo = op->new_value()->as_register_lo();
2616     Register new_value_hi = op->new_value()->as_register_hi();
2617     Register t1 = op->tmp1()->as_register();
2618     Register t2 = op->tmp2()->as_register();
2619     __ mov(cmp_value_lo, t1);
2620     __ mov(new_value_lo, t2);
2621     // perform the compare and swap operation
2622     __ casx(addr, t1, t2);
2623     // generate condition code - if the swap succeeded, t2 ("new value" reg) was
2624     // overwritten with the original value in "addr" and will be equal to t1.
2625     __ cmp(t1, t2);
2626   } else if (op->code() == lir_cas_int || op->code() == lir_cas_obj) {
2627     Register addr = op->addr()->as_pointer_register();
2628     Register cmp_value = op->cmp_value()->as_register();
2629     Register new_value = op->new_value()->as_register();
2630     Register t1 = op->tmp1()->as_register();
2631     Register t2 = op->tmp2()->as_register();
2632     __ mov(cmp_value, t1);
2633     __ mov(new_value, t2);
2634     if (op->code() == lir_cas_obj) {
2635       if (UseCompressedOops) {
2636         __ encode_heap_oop(t1);
2637         __ encode_heap_oop(t2);
2638         __ cas(addr, t1, t2);
2639       } else {
2640         __ cas_ptr(addr, t1, t2);
2641       }
2642     } else {
2643       __ cas(addr, t1, t2);
2644     }
2645     __ cmp(t1, t2);
2646   } else {
2647     Unimplemented();
2648   }
2649 }
2650 
2651 void LIR_Assembler::set_24bit_FPU() {
2652   Unimplemented();
2653 }
2654 
2655 
2656 void LIR_Assembler::reset_FPU() {
2657   Unimplemented();
2658 }
2659 
2660 
2661 void LIR_Assembler::breakpoint() {
2662   __ breakpoint_trap();
2663 }
2664 
2665 
2666 void LIR_Assembler::push(LIR_Opr opr) {
2667   Unimplemented();
2668 }
2669 
2670 
2671 void LIR_Assembler::pop(LIR_Opr opr) {
2672   Unimplemented();
2673 }
2674 
2675 
2676 void LIR_Assembler::monitor_address(int monitor_no, LIR_Opr dst_opr) {
2677   Address mon_addr = frame_map()->address_for_monitor_lock(monitor_no);
2678   Register dst = dst_opr->as_register();
2679   Register reg = mon_addr.base();
2680   int offset = mon_addr.disp();
2681   // compute pointer to BasicLock
2682   if (mon_addr.is_simm13()) {
2683     __ add(reg, offset, dst);
2684   } else {
2685     __ set(offset, dst);
2686     __ add(dst, reg, dst);
2687   }
2688 }
2689 
2690 void LIR_Assembler::emit_updatecrc32(LIR_OpUpdateCRC32* op) {
2691   assert(op->crc()->is_single_cpu(),  "crc must be register");
2692   assert(op->val()->is_single_cpu(),  "byte value must be register");
2693   assert(op->result_opr()->is_single_cpu(), "result must be register");
2694   Register crc = op->crc()->as_register();
2695   Register val = op->val()->as_register();
2696   Register table = op->result_opr()->as_register();
2697   Register res   = op->result_opr()->as_register();
2698 
2699   assert_different_registers(val, crc, table);
2700 
2701   __ set(ExternalAddress(StubRoutines::crc_table_addr()), table);
2702   __ not1(crc);
2703   __ clruwu(crc);
2704   __ update_byte_crc32(crc, val, table);
2705   __ not1(crc);
2706 
2707   __ mov(crc, res);
2708 }
2709 
2710 void LIR_Assembler::emit_lock(LIR_OpLock* op) {
2711   Register obj = op->obj_opr()->as_register();
2712   Register hdr = op->hdr_opr()->as_register();
2713   Register lock = op->lock_opr()->as_register();
2714 
2715   // obj may not be an oop
2716   if (op->code() == lir_lock) {
2717     MonitorEnterStub* stub = (MonitorEnterStub*)op->stub();
2718     if (UseFastLocking) {
2719       assert(BasicLock::displaced_header_offset_in_bytes() == 0, "lock_reg must point to the displaced header");
2720       // add debug info for NullPointerException only if one is possible
2721       if (op->info() != NULL) {
2722         add_debug_info_for_null_check_here(op->info());
2723       }
2724       __ lock_object(hdr, obj, lock, op->scratch_opr()->as_register(), *op->stub()->entry());
2725     } else {
2726       // always do slow locking
2727       // note: the slow locking code could be inlined here, however if we use
2728       //       slow locking, speed doesn't matter anyway and this solution is
2729       //       simpler and requires less duplicated code - additionally, the
2730       //       slow locking code is the same in either case which simplifies
2731       //       debugging
2732       __ br(Assembler::always, false, Assembler::pt, *op->stub()->entry());
2733       __ delayed()->nop();
2734     }
2735   } else {
2736     assert (op->code() == lir_unlock, "Invalid code, expected lir_unlock");
2737     if (UseFastLocking) {
2738       assert(BasicLock::displaced_header_offset_in_bytes() == 0, "lock_reg must point to the displaced header");
2739       __ unlock_object(hdr, obj, lock, *op->stub()->entry());
2740     } else {
2741       // always do slow unlocking
2742       // note: the slow unlocking code could be inlined here, however if we use
2743       //       slow unlocking, speed doesn't matter anyway and this solution is
2744       //       simpler and requires less duplicated code - additionally, the
2745       //       slow unlocking code is the same in either case which simplifies
2746       //       debugging
2747       __ br(Assembler::always, false, Assembler::pt, *op->stub()->entry());
2748       __ delayed()->nop();
2749     }
2750   }
2751   __ bind(*op->stub()->continuation());
2752 }
2753 
2754 
2755 void LIR_Assembler::emit_profile_call(LIR_OpProfileCall* op) {
2756   ciMethod* method = op->profiled_method();
2757   int bci          = op->profiled_bci();
2758   ciMethod* callee = op->profiled_callee();
2759 
2760   // Update counter for all call types
2761   ciMethodData* md = method->method_data_or_null();
2762   assert(md != NULL, "Sanity");
2763   ciProfileData* data = md->bci_to_data(bci);
2764   assert(data != NULL && data->is_CounterData(), "need CounterData for calls");
2765   assert(op->mdo()->is_single_cpu(),  "mdo must be allocated");
2766   Register mdo  = op->mdo()->as_register();
2767   assert(op->tmp1()->is_double_cpu(), "tmp1 must be allocated");
2768   Register tmp1 = op->tmp1()->as_register_lo();
2769   metadata2reg(md->constant_encoding(), mdo);
2770   int mdo_offset_bias = 0;
2771   if (!Assembler::is_simm13(md->byte_offset_of_slot(data, CounterData::count_offset()) +
2772                             data->size_in_bytes())) {
2773     // The offset is large so bias the mdo by the base of the slot so
2774     // that the ld can use simm13s to reference the slots of the data
2775     mdo_offset_bias = md->byte_offset_of_slot(data, CounterData::count_offset());
2776     __ set(mdo_offset_bias, O7);
2777     __ add(mdo, O7, mdo);
2778   }
2779 
2780   Address counter_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()) - mdo_offset_bias);
2781   // Perform additional virtual call profiling for invokevirtual and
2782   // invokeinterface bytecodes
2783   if (op->should_profile_receiver_type()) {
2784     assert(op->recv()->is_single_cpu(), "recv must be allocated");
2785     Register recv = op->recv()->as_register();
2786     assert_different_registers(mdo, tmp1, recv);
2787     assert(data->is_VirtualCallData(), "need VirtualCallData for virtual calls");
2788     ciKlass* known_klass = op->known_holder();
2789     if (C1OptimizeVirtualCallProfiling && known_klass != NULL) {
2790       // We know the type that will be seen at this call site; we can
2791       // statically update the MethodData* rather than needing to do
2792       // dynamic tests on the receiver type
2793 
2794       // NOTE: we should probably put a lock around this search to
2795       // avoid collisions by concurrent compilations
2796       ciVirtualCallData* vc_data = (ciVirtualCallData*) data;
2797       uint i;
2798       for (i = 0; i < VirtualCallData::row_limit(); i++) {
2799         ciKlass* receiver = vc_data->receiver(i);
2800         if (known_klass->equals(receiver)) {
2801           Address data_addr(mdo, md->byte_offset_of_slot(data,
2802                                                          VirtualCallData::receiver_count_offset(i)) -
2803                             mdo_offset_bias);
2804           __ ld_ptr(data_addr, tmp1);
2805           __ add(tmp1, DataLayout::counter_increment, tmp1);
2806           __ st_ptr(tmp1, data_addr);
2807           return;
2808         }
2809       }
2810 
2811       // Receiver type not found in profile data; select an empty slot
2812 
2813       // Note that this is less efficient than it should be because it
2814       // always does a write to the receiver part of the
2815       // VirtualCallData rather than just the first time
2816       for (i = 0; i < VirtualCallData::row_limit(); i++) {
2817         ciKlass* receiver = vc_data->receiver(i);
2818         if (receiver == NULL) {
2819           Address recv_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_offset(i)) -
2820                             mdo_offset_bias);
2821           metadata2reg(known_klass->constant_encoding(), tmp1);
2822           __ st_ptr(tmp1, recv_addr);
2823           Address data_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_count_offset(i)) -
2824                             mdo_offset_bias);
2825           __ ld_ptr(data_addr, tmp1);
2826           __ add(tmp1, DataLayout::counter_increment, tmp1);
2827           __ st_ptr(tmp1, data_addr);
2828           return;
2829         }
2830       }
2831     } else {
2832       __ load_klass(recv, recv);
2833       Label update_done;
2834       type_profile_helper(mdo, mdo_offset_bias, md, data, recv, tmp1, &update_done);
2835       // Receiver did not match any saved receiver and there is no empty row for it.
2836       // Increment total counter to indicate polymorphic case.
2837       __ ld_ptr(counter_addr, tmp1);
2838       __ add(tmp1, DataLayout::counter_increment, tmp1);
2839       __ st_ptr(tmp1, counter_addr);
2840 
2841       __ bind(update_done);
2842     }
2843   } else {
2844     // Static call
2845     __ ld_ptr(counter_addr, tmp1);
2846     __ add(tmp1, DataLayout::counter_increment, tmp1);
2847     __ st_ptr(tmp1, counter_addr);
2848   }
2849 }
2850 
2851 void LIR_Assembler::emit_profile_type(LIR_OpProfileType* op) {
2852   Register obj = op->obj()->as_register();
2853   Register tmp1 = op->tmp()->as_pointer_register();
2854   Register tmp2 = G1;
2855   Address mdo_addr = as_Address(op->mdp()->as_address_ptr());
2856   ciKlass* exact_klass = op->exact_klass();
2857   intptr_t current_klass = op->current_klass();
2858   bool not_null = op->not_null();
2859   bool no_conflict = op->no_conflict();
2860 
2861   Label update, next, none;
2862 
2863   bool do_null = !not_null;
2864   bool exact_klass_set = exact_klass != NULL && ciTypeEntries::valid_ciklass(current_klass) == exact_klass;
2865   bool do_update = !TypeEntries::is_type_unknown(current_klass) && !exact_klass_set;
2866 
2867   assert(do_null || do_update, "why are we here?");
2868   assert(!TypeEntries::was_null_seen(current_klass) || do_update, "why are we here?");
2869 
2870   __ verify_oop(obj);
2871 
2872   if (tmp1 != obj) {
2873     __ mov(obj, tmp1);
2874   }
2875   if (do_null) {
2876     __ br_notnull_short(tmp1, Assembler::pt, update);
2877     if (!TypeEntries::was_null_seen(current_klass)) {
2878       __ ld_ptr(mdo_addr, tmp1);
2879       __ or3(tmp1, TypeEntries::null_seen, tmp1);
2880       __ st_ptr(tmp1, mdo_addr);
2881     }
2882     if (do_update) {
2883       __ ba(next);
2884       __ delayed()->nop();
2885     }
2886 #ifdef ASSERT
2887   } else {
2888     __ br_notnull_short(tmp1, Assembler::pt, update);
2889     __ stop("unexpect null obj");
2890 #endif
2891   }
2892 
2893   __ bind(update);
2894 
2895   if (do_update) {
2896 #ifdef ASSERT
2897     if (exact_klass != NULL) {
2898       Label ok;
2899       __ load_klass(tmp1, tmp1);
2900       metadata2reg(exact_klass->constant_encoding(), tmp2);
2901       __ cmp_and_br_short(tmp1, tmp2, Assembler::equal, Assembler::pt, ok);
2902       __ stop("exact klass and actual klass differ");
2903       __ bind(ok);
2904     }
2905 #endif
2906 
2907     Label do_update;
2908     __ ld_ptr(mdo_addr, tmp2);
2909 
2910     if (!no_conflict) {
2911       if (exact_klass == NULL || TypeEntries::is_type_none(current_klass)) {
2912         if (exact_klass != NULL) {
2913           metadata2reg(exact_klass->constant_encoding(), tmp1);
2914         } else {
2915           __ load_klass(tmp1, tmp1);
2916         }
2917 
2918         __ xor3(tmp1, tmp2, tmp1);
2919         __ btst(TypeEntries::type_klass_mask, tmp1);
2920         // klass seen before, nothing to do. The unknown bit may have been
2921         // set already but no need to check.
2922         __ brx(Assembler::zero, false, Assembler::pt, next);
2923         __ delayed()->
2924 
2925            btst(TypeEntries::type_unknown, tmp1);
2926         // already unknown. Nothing to do anymore.
2927         __ brx(Assembler::notZero, false, Assembler::pt, next);
2928 
2929         if (TypeEntries::is_type_none(current_klass)) {
2930           __ delayed()->btst(TypeEntries::type_mask, tmp2);
2931           __ brx(Assembler::zero, true, Assembler::pt, do_update);
2932           // first time here. Set profile type.
2933           __ delayed()->or3(tmp2, tmp1, tmp2);
2934         } else {
2935           __ delayed()->nop();
2936         }
2937       } else {
2938         assert(ciTypeEntries::valid_ciklass(current_klass) != NULL &&
2939                ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "conflict only");
2940 
2941         __ btst(TypeEntries::type_unknown, tmp2);
2942         // already unknown. Nothing to do anymore.
2943         __ brx(Assembler::notZero, false, Assembler::pt, next);
2944         __ delayed()->nop();
2945       }
2946 
2947       // different than before. Cannot keep accurate profile.
2948       __ or3(tmp2, TypeEntries::type_unknown, tmp2);
2949     } else {
2950       // There's a single possible klass at this profile point
2951       assert(exact_klass != NULL, "should be");
2952       if (TypeEntries::is_type_none(current_klass)) {
2953         metadata2reg(exact_klass->constant_encoding(), tmp1);
2954         __ xor3(tmp1, tmp2, tmp1);
2955         __ btst(TypeEntries::type_klass_mask, tmp1);
2956         __ brx(Assembler::zero, false, Assembler::pt, next);
2957 #ifdef ASSERT
2958 
2959         {
2960           Label ok;
2961           __ delayed()->btst(TypeEntries::type_mask, tmp2);
2962           __ brx(Assembler::zero, true, Assembler::pt, ok);
2963           __ delayed()->nop();
2964 
2965           __ stop("unexpected profiling mismatch");
2966           __ bind(ok);
2967         }
2968         // first time here. Set profile type.
2969         __ or3(tmp2, tmp1, tmp2);
2970 #else
2971         // first time here. Set profile type.
2972         __ delayed()->or3(tmp2, tmp1, tmp2);
2973 #endif
2974 
2975       } else {
2976         assert(ciTypeEntries::valid_ciklass(current_klass) != NULL &&
2977                ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "inconsistent");
2978 
2979         // already unknown. Nothing to do anymore.
2980         __ btst(TypeEntries::type_unknown, tmp2);
2981         __ brx(Assembler::notZero, false, Assembler::pt, next);
2982         __ delayed()->or3(tmp2, TypeEntries::type_unknown, tmp2);
2983       }
2984     }
2985 
2986     __ bind(do_update);
2987     __ st_ptr(tmp2, mdo_addr);
2988 
2989     __ bind(next);
2990   }
2991 }
2992 
2993 void LIR_Assembler::align_backward_branch_target() {
2994   __ align(OptoLoopAlignment);
2995 }
2996 
2997 
2998 void LIR_Assembler::emit_delay(LIR_OpDelay* op) {
2999   // make sure we are expecting a delay
3000   // this has the side effect of clearing the delay state
3001   // so we can use _masm instead of _masm->delayed() to do the
3002   // code generation.
3003   __ delayed();
3004 
3005   // make sure we only emit one instruction
3006   int offset = code_offset();
3007   op->delay_op()->emit_code(this);
3008 #ifdef ASSERT
3009   if (code_offset() - offset != NativeInstruction::nop_instruction_size) {
3010     op->delay_op()->print();
3011   }
3012   assert(code_offset() - offset == NativeInstruction::nop_instruction_size,
3013          "only one instruction can go in a delay slot");
3014 #endif
3015 
3016   // we may also be emitting the call info for the instruction
3017   // which we are the delay slot of.
3018   CodeEmitInfo* call_info = op->call_info();
3019   if (call_info) {
3020     add_call_info(code_offset(), call_info);
3021   }
3022 
3023   if (VerifyStackAtCalls) {
3024     _masm->sub(FP, SP, O7);
3025     _masm->cmp(O7, initial_frame_size_in_bytes());
3026     _masm->trap(Assembler::notEqual, Assembler::ptr_cc, G0, ST_RESERVED_FOR_USER_0+2 );
3027   }
3028 }
3029 
3030 
3031 void LIR_Assembler::negate(LIR_Opr left, LIR_Opr dest) {
3032   assert(left->is_register(), "can only handle registers");
3033 
3034   if (left->is_single_cpu()) {
3035     __ neg(left->as_register(), dest->as_register());
3036   } else if (left->is_single_fpu()) {
3037     __ fneg(FloatRegisterImpl::S, left->as_float_reg(), dest->as_float_reg());
3038   } else if (left->is_double_fpu()) {
3039     __ fneg(FloatRegisterImpl::D, left->as_double_reg(), dest->as_double_reg());
3040   } else {
3041     assert (left->is_double_cpu(), "Must be a long");
3042     Register Rlow = left->as_register_lo();
3043     Register Rhi = left->as_register_hi();
3044     __ sub(G0, Rlow, dest->as_register_lo());
3045   }
3046 }
3047 
3048 
3049 void LIR_Assembler::fxch(int i) {
3050   Unimplemented();
3051 }
3052 
3053 void LIR_Assembler::fld(int i) {
3054   Unimplemented();
3055 }
3056 
3057 void LIR_Assembler::ffree(int i) {
3058   Unimplemented();
3059 }
3060 
3061 void LIR_Assembler::rt_call(LIR_Opr result, address dest,
3062                             const LIR_OprList* args, LIR_Opr tmp, CodeEmitInfo* info) {
3063 
3064   // if tmp is invalid, then the function being called doesn't destroy the thread
3065   if (tmp->is_valid()) {
3066     __ save_thread(tmp->as_pointer_register());
3067   }
3068   __ call(dest, relocInfo::runtime_call_type);
3069   __ delayed()->nop();
3070   if (info != NULL) {
3071     add_call_info_here(info);
3072   }
3073   if (tmp->is_valid()) {
3074     __ restore_thread(tmp->as_pointer_register());
3075   }
3076 
3077 #ifdef ASSERT
3078   __ verify_thread();
3079 #endif // ASSERT
3080 }
3081 
3082 
3083 void LIR_Assembler::volatile_move_op(LIR_Opr src, LIR_Opr dest, BasicType type, CodeEmitInfo* info) {
3084   ShouldNotReachHere();
3085 
3086   NEEDS_CLEANUP;
3087   if (type == T_LONG) {
3088     LIR_Address* mem_addr = dest->is_address() ? dest->as_address_ptr() : src->as_address_ptr();
3089 
3090     // (extended to allow indexed as well as constant displaced for JSR-166)
3091     Register idx = noreg; // contains either constant offset or index
3092 
3093     int disp = mem_addr->disp();
3094     if (mem_addr->index() == LIR_OprFact::illegalOpr) {
3095       if (!Assembler::is_simm13(disp)) {
3096         idx = O7;
3097         __ set(disp, idx);
3098       }
3099     } else {
3100       assert(disp == 0, "not both indexed and disp");
3101       idx = mem_addr->index()->as_register();
3102     }
3103 
3104     int null_check_offset = -1;
3105 
3106     Register base = mem_addr->base()->as_register();
3107     if (src->is_register() && dest->is_address()) {
3108       // G4 is high half, G5 is low half
3109       // clear the top bits of G5, and scale up G4
3110       __ srl (src->as_register_lo(),  0, G5);
3111       __ sllx(src->as_register_hi(), 32, G4);
3112       // combine the two halves into the 64 bits of G4
3113       __ or3(G4, G5, G4);
3114       null_check_offset = __ offset();
3115       if (idx == noreg) {
3116         __ stx(G4, base, disp);
3117       } else {
3118         __ stx(G4, base, idx);
3119       }
3120     } else if (src->is_address() && dest->is_register()) {
3121       null_check_offset = __ offset();
3122       if (idx == noreg) {
3123         __ ldx(base, disp, G5);
3124       } else {
3125         __ ldx(base, idx, G5);
3126       }
3127       __ srax(G5, 32, dest->as_register_hi()); // fetch the high half into hi
3128       __ mov (G5, dest->as_register_lo());     // copy low half into lo
3129     } else {
3130       Unimplemented();
3131     }
3132     if (info != NULL) {
3133       add_debug_info_for_null_check(null_check_offset, info);
3134     }
3135 
3136   } else {
3137     // use normal move for all other volatiles since they don't need
3138     // special handling to remain atomic.
3139     move_op(src, dest, type, lir_patch_none, info, false, false, false);
3140   }
3141 }
3142 
3143 void LIR_Assembler::membar() {
3144   // only StoreLoad membars are ever explicitly needed on sparcs in TSO mode
3145   __ membar( Assembler::Membar_mask_bits(Assembler::StoreLoad) );
3146 }
3147 
3148 void LIR_Assembler::membar_acquire() {
3149   // no-op on TSO
3150 }
3151 
3152 void LIR_Assembler::membar_release() {
3153   // no-op on TSO
3154 }
3155 
3156 void LIR_Assembler::membar_loadload() {
3157   // no-op
3158   //__ membar(Assembler::Membar_mask_bits(Assembler::loadload));
3159 }
3160 
3161 void LIR_Assembler::membar_storestore() {
3162   // no-op
3163   //__ membar(Assembler::Membar_mask_bits(Assembler::storestore));
3164 }
3165 
3166 void LIR_Assembler::membar_loadstore() {
3167   // no-op
3168   //__ membar(Assembler::Membar_mask_bits(Assembler::loadstore));
3169 }
3170 
3171 void LIR_Assembler::membar_storeload() {
3172   __ membar(Assembler::Membar_mask_bits(Assembler::StoreLoad));
3173 }
3174 
3175 void LIR_Assembler::on_spin_wait() {
3176   Unimplemented();
3177 }
3178 
3179 // Pack two sequential registers containing 32 bit values
3180 // into a single 64 bit register.
3181 // src and src->successor() are packed into dst
3182 // src and dst may be the same register.
3183 // Note: src is destroyed
3184 void LIR_Assembler::pack64(LIR_Opr src, LIR_Opr dst) {
3185   Register rs = src->as_register();
3186   Register rd = dst->as_register_lo();
3187   __ sllx(rs, 32, rs);
3188   __ srl(rs->successor(), 0, rs->successor());
3189   __ or3(rs, rs->successor(), rd);
3190 }
3191 
3192 // Unpack a 64 bit value in a register into
3193 // two sequential registers.
3194 // src is unpacked into dst and dst->successor()
3195 void LIR_Assembler::unpack64(LIR_Opr src, LIR_Opr dst) {
3196   Register rs = src->as_register_lo();
3197   Register rd = dst->as_register_hi();
3198   assert_different_registers(rs, rd, rd->successor());
3199   __ srlx(rs, 32, rd);
3200   __ srl (rs,  0, rd->successor());
3201 }
3202 
3203 void LIR_Assembler::leal(LIR_Opr addr_opr, LIR_Opr dest) {
3204   const LIR_Address* addr = addr_opr->as_address_ptr();
3205   assert(addr->scale() == LIR_Address::times_1, "can't handle complex addresses yet");
3206   const Register dest_reg = dest->as_pointer_register();
3207   const Register base_reg = addr->base()->as_pointer_register();
3208 
3209   if (Assembler::is_simm13(addr->disp())) {
3210     if (addr->index()->is_valid()) {
3211       const Register index_reg = addr->index()->as_pointer_register();
3212       assert(index_reg != G3_scratch, "invariant");
3213       __ add(base_reg, addr->disp(), G3_scratch);
3214       __ add(index_reg, G3_scratch, dest_reg);
3215     } else {
3216       __ add(base_reg, addr->disp(), dest_reg);
3217     }
3218   } else {
3219     __ set(addr->disp(), G3_scratch);
3220     if (addr->index()->is_valid()) {
3221       const Register index_reg = addr->index()->as_pointer_register();
3222       assert(index_reg != G3_scratch, "invariant");
3223       __ add(index_reg, G3_scratch, G3_scratch);
3224     }
3225     __ add(base_reg, G3_scratch, dest_reg);
3226   }
3227 }
3228 
3229 
3230 void LIR_Assembler::get_thread(LIR_Opr result_reg) {
3231   assert(result_reg->is_register(), "check");
3232   __ mov(G2_thread, result_reg->as_register());
3233 }
3234 
3235 #ifdef ASSERT
3236 // emit run-time assertion
3237 void LIR_Assembler::emit_assert(LIR_OpAssert* op) {
3238   assert(op->code() == lir_assert, "must be");
3239 
3240   if (op->in_opr1()->is_valid()) {
3241     assert(op->in_opr2()->is_valid(), "both operands must be valid");
3242     comp_op(op->condition(), op->in_opr1(), op->in_opr2(), op);
3243   } else {
3244     assert(op->in_opr2()->is_illegal(), "both operands must be illegal");
3245     assert(op->condition() == lir_cond_always, "no other conditions allowed");
3246   }
3247 
3248   Label ok;
3249   if (op->condition() != lir_cond_always) {
3250     Assembler::Condition acond;
3251     switch (op->condition()) {
3252       case lir_cond_equal:        acond = Assembler::equal;                break;
3253       case lir_cond_notEqual:     acond = Assembler::notEqual;             break;
3254       case lir_cond_less:         acond = Assembler::less;                 break;
3255       case lir_cond_lessEqual:    acond = Assembler::lessEqual;            break;
3256       case lir_cond_greaterEqual: acond = Assembler::greaterEqual;         break;
3257       case lir_cond_greater:      acond = Assembler::greater;              break;
3258       case lir_cond_aboveEqual:   acond = Assembler::greaterEqualUnsigned; break;
3259       case lir_cond_belowEqual:   acond = Assembler::lessEqualUnsigned;    break;
3260       default:                         ShouldNotReachHere();
3261     };
3262     __ br(acond, false, Assembler::pt, ok);
3263     __ delayed()->nop();
3264   }
3265   if (op->halt()) {
3266     const char* str = __ code_string(op->msg());
3267     __ stop(str);
3268   } else {
3269     breakpoint();
3270   }
3271   __ bind(ok);
3272 }
3273 #endif
3274 
3275 void LIR_Assembler::peephole(LIR_List* lir) {
3276   LIR_OpList* inst = lir->instructions_list();
3277   for (int i = 0; i < inst->length(); i++) {
3278     LIR_Op* op = inst->at(i);
3279     switch (op->code()) {
3280       case lir_cond_float_branch:
3281       case lir_branch: {
3282         LIR_OpBranch* branch = op->as_OpBranch();
3283         assert(branch->info() == NULL, "shouldn't be state on branches anymore");
3284         LIR_Op* delay_op = NULL;
3285         // we'd like to be able to pull following instructions into
3286         // this slot but we don't know enough to do it safely yet so
3287         // only optimize block to block control flow.
3288         if (LIRFillDelaySlots && branch->block()) {
3289           LIR_Op* prev = inst->at(i - 1);
3290           if (prev && LIR_Assembler::is_single_instruction(prev) && prev->info() == NULL) {
3291             // swap previous instruction into delay slot
3292             inst->at_put(i - 1, op);
3293             inst->at_put(i, new LIR_OpDelay(prev, op->info()));
3294 #ifndef PRODUCT
3295             if (LIRTracePeephole) {
3296               tty->print_cr("delayed");
3297               inst->at(i - 1)->print();
3298               inst->at(i)->print();
3299               tty->cr();
3300             }
3301 #endif
3302             continue;
3303           }
3304         }
3305 
3306         if (!delay_op) {
3307           delay_op = new LIR_OpDelay(new LIR_Op0(lir_nop), NULL);
3308         }
3309         inst->insert_before(i + 1, delay_op);
3310         break;
3311       }
3312       case lir_static_call:
3313       case lir_virtual_call:
3314       case lir_icvirtual_call:
3315       case lir_optvirtual_call:
3316       case lir_dynamic_call: {
3317         LIR_Op* prev = inst->at(i - 1);
3318         if (LIRFillDelaySlots && prev && prev->code() == lir_move && prev->info() == NULL &&
3319             (op->code() != lir_virtual_call ||
3320              !prev->result_opr()->is_single_cpu() ||
3321              prev->result_opr()->as_register() != O0) &&
3322             LIR_Assembler::is_single_instruction(prev)) {
3323           // Only moves without info can be put into the delay slot.
3324           // Also don't allow the setup of the receiver in the delay
3325           // slot for vtable calls.
3326           inst->at_put(i - 1, op);
3327           inst->at_put(i, new LIR_OpDelay(prev, op->info()));
3328 #ifndef PRODUCT
3329           if (LIRTracePeephole) {
3330             tty->print_cr("delayed");
3331             inst->at(i - 1)->print();
3332             inst->at(i)->print();
3333             tty->cr();
3334           }
3335 #endif
3336         } else {
3337           LIR_Op* delay_op = new LIR_OpDelay(new LIR_Op0(lir_nop), op->as_OpJavaCall()->info());
3338           inst->insert_before(i + 1, delay_op);
3339           i++;
3340         }
3341         break;
3342       }
3343     }
3344   }
3345 }
3346 
3347 void LIR_Assembler::atomic_op(LIR_Code code, LIR_Opr src, LIR_Opr data, LIR_Opr dest, LIR_Opr tmp) {
3348   LIR_Address* addr = src->as_address_ptr();
3349 
3350   assert(data == dest, "swap uses only 2 operands");
3351   assert (code == lir_xchg, "no xadd on sparc");
3352 
3353   if (data->type() == T_INT) {
3354     __ swap(as_Address(addr), data->as_register());
3355   } else if (data->is_oop()) {
3356     Register obj = data->as_register();
3357     Register narrow = tmp->as_register();
3358     assert(UseCompressedOops, "swap is 32bit only");
3359     __ encode_heap_oop(obj, narrow);
3360     __ swap(as_Address(addr), narrow);
3361     __ decode_heap_oop(narrow, obj);
3362   } else {
3363     ShouldNotReachHere();
3364   }
3365 }
3366 
3367 #undef __