1 /*
   2  * Copyright (c) 2016, Oracle and/or its affiliates. All rights reserved.
   3  * Copyright (c) 2016 SAP SE. All rights reserved.
   4  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   5  *
   6  * This code is free software; you can redistribute it and/or modify it
   7  * under the terms of the GNU General Public License version 2 only, as
   8  * published by the Free Software Foundation.
   9  *
  10  * This code is distributed in the hope that it will be useful, but WITHOUT
  11  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  12  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  13  * version 2 for more details (a copy is included in the LICENSE file that
  14  * accompanied this code).
  15  *
  16  * You should have received a copy of the GNU General Public License version
  17  * 2 along with this work; if not, write to the Free Software Foundation,
  18  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  19  *
  20  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  21  * or visit www.oracle.com if you need additional information or have any
  22  * questions.
  23  *
  24  */
  25 
  26 #include "precompiled.hpp"
  27 #include "c1/c1_Compilation.hpp"
  28 #include "c1/c1_LIRAssembler.hpp"
  29 #include "c1/c1_MacroAssembler.hpp"
  30 #include "c1/c1_Runtime1.hpp"
  31 #include "c1/c1_ValueStack.hpp"
  32 #include "ci/ciArrayKlass.hpp"
  33 #include "ci/ciInstance.hpp"
  34 #include "gc/shared/collectedHeap.hpp"
  35 #include "gc/shared/barrierSet.hpp"
  36 #include "gc/shared/cardTableModRefBS.hpp"
  37 #include "nativeInst_s390.hpp"
  38 #include "oops/objArrayKlass.hpp"
  39 #include "runtime/sharedRuntime.hpp"
  40 #include "vmreg_s390.inline.hpp"
  41 
  42 #define __ _masm->
  43 
  44 #ifndef PRODUCT
  45 #undef __
  46 #define __ (Verbose ? (_masm->block_comment(FILE_AND_LINE),_masm) : _masm)->
  47 #endif
  48 
  49 //------------------------------------------------------------
  50 
  51 bool LIR_Assembler::is_small_constant(LIR_Opr opr) {
  52   // Not used on ZARCH_64
  53   ShouldNotCallThis();
  54   return false;
  55 }
  56 
  57 LIR_Opr LIR_Assembler::receiverOpr() {
  58   return FrameMap::Z_R2_oop_opr;
  59 }
  60 
  61 LIR_Opr LIR_Assembler::osrBufferPointer() {
  62   return FrameMap::Z_R2_opr;
  63 }
  64 
  65 int LIR_Assembler::initial_frame_size_in_bytes() const {
  66   return in_bytes(frame_map()->framesize_in_bytes());
  67 }
  68 
  69 // Inline cache check: done before the frame is built.
  70 // The inline cached class is in Z_inline_cache(Z_R9).
  71 // We fetch the class of the receiver and compare it with the cached class.
  72 // If they do not match we jump to the slow case.
  73 int LIR_Assembler::check_icache() {
  74   Register receiver = receiverOpr()->as_register();
  75   int offset = __ offset();
  76   __ inline_cache_check(receiver, Z_inline_cache);
  77   return offset;
  78 }
  79 
  80 void LIR_Assembler::osr_entry() {
  81   // On-stack-replacement entry sequence (interpreter frame layout described in interpreter_sparc.cpp):
  82   //
  83   //   1. Create a new compiled activation.
  84   //   2. Initialize local variables in the compiled activation. The expression stack must be empty
  85   //      at the osr_bci; it is not initialized.
  86   //   3. Jump to the continuation address in compiled code to resume execution.
  87 
  88   // OSR entry point
  89   offsets()->set_value(CodeOffsets::OSR_Entry, code_offset());
  90   BlockBegin* osr_entry = compilation()->hir()->osr_entry();
  91   ValueStack* entry_state = osr_entry->end()->state();
  92   int number_of_locks = entry_state->locks_size();
  93 
  94   // Create a frame for the compiled activation.
  95   __ build_frame(initial_frame_size_in_bytes(), bang_size_in_bytes());
  96 
  97   // OSR buffer is
  98   //
  99   // locals[nlocals-1..0]
 100   // monitors[number_of_locks-1..0]
 101   //
 102   // Locals is a direct copy of the interpreter frame so in the osr buffer
 103   // the first slot in the local array is the last local from the interpreter
 104   // and the last slot is local[0] (receiver) from the interpreter
 105   //
 106   // Similarly with locks. The first lock slot in the osr buffer is the nth lock
 107   // from the interpreter frame, the nth lock slot in the osr buffer is 0th lock
 108   // in the interpreter frame (the method lock if a sync method)
 109 
 110   // Initialize monitors in the compiled activation.
 111   //   I0: pointer to osr buffer
 112   //
 113   // All other registers are dead at this point and the locals will be
 114   // copied into place by code emitted in the IR.
 115 
 116   Register OSR_buf = osrBufferPointer()->as_register();
 117   { assert(frame::interpreter_frame_monitor_size() == BasicObjectLock::size(), "adjust code below");
 118     int monitor_offset = BytesPerWord * method()->max_locals() +
 119       (2 * BytesPerWord) * (number_of_locks - 1);
 120     // SharedRuntime::OSR_migration_begin() packs BasicObjectLocks in
 121     // the OSR buffer using 2 word entries: first the lock and then
 122     // the oop.
 123     for (int i = 0; i < number_of_locks; i++) {
 124       int slot_offset = monitor_offset - ((i * 2) * BytesPerWord);
 125       // Verify the interpreter's monitor has a non-null object.
 126       __ asm_assert_mem8_isnot_zero(slot_offset + 1*BytesPerWord, OSR_buf, "locked object is NULL", __LINE__);
 127       // Copy the lock field into the compiled activation.
 128       __ z_lg(Z_R1_scratch, slot_offset + 0, OSR_buf);
 129       __ z_stg(Z_R1_scratch, frame_map()->address_for_monitor_lock(i));
 130       __ z_lg(Z_R1_scratch, slot_offset + 1*BytesPerWord, OSR_buf);
 131       __ z_stg(Z_R1_scratch, frame_map()->address_for_monitor_object(i));
 132     }
 133   }
 134 }
 135 
 136 // --------------------------------------------------------------------------------------------
 137 
 138 address LIR_Assembler::emit_call_c(address a) {
 139   __ align_call_far_patchable(__ pc());
 140   address call_addr = __ call_c_opt(a);
 141   if (call_addr == NULL) {
 142     bailout("const section overflow");
 143   }
 144   return call_addr;
 145 }
 146 
 147 int LIR_Assembler::emit_exception_handler() {
 148   // If the last instruction is a call (typically to do a throw which
 149   // is coming at the end after block reordering) the return address
 150   // must still point into the code area in order to avoid assertion
 151   // failures when searching for the corresponding bci. => Add a nop.
 152   // (was bug 5/14/1999 - gri)
 153   __ nop();
 154 
 155   // Generate code for exception handler.
 156   address handler_base = __ start_a_stub(exception_handler_size);
 157   if (handler_base == NULL) {
 158     // Not enough space left for the handler.
 159     bailout("exception handler overflow");
 160     return -1;
 161   }
 162 
 163   int offset = code_offset();
 164 
 165   address a = Runtime1::entry_for (Runtime1::handle_exception_from_callee_id);
 166   address call_addr = emit_call_c(a);
 167   CHECK_BAILOUT_(-1);
 168   __ should_not_reach_here();
 169   guarantee(code_offset() - offset <= exception_handler_size, "overflow");
 170   __ end_a_stub();
 171 
 172   return offset;
 173 }
 174 
 175 // Emit the code to remove the frame from the stack in the exception
 176 // unwind path.
 177 int LIR_Assembler::emit_unwind_handler() {
 178 #ifndef PRODUCT
 179   if (CommentedAssembly) {
 180     _masm->block_comment("Unwind handler");
 181   }
 182 #endif
 183 
 184   int offset = code_offset();
 185   Register exception_oop_callee_saved = Z_R10; // Z_R10 is callee-saved.
 186   Register Rtmp1                      = Z_R11;
 187   Register Rtmp2                      = Z_R12;
 188 
 189   // Fetch the exception from TLS and clear out exception related thread state.
 190   Address exc_oop_addr = Address(Z_thread, JavaThread::exception_oop_offset());
 191   Address exc_pc_addr  = Address(Z_thread, JavaThread::exception_pc_offset());
 192   __ z_lg(Z_EXC_OOP, exc_oop_addr);
 193   __ clear_mem(exc_oop_addr, sizeof(oop));
 194   __ clear_mem(exc_pc_addr, sizeof(intptr_t));
 195 
 196   __ bind(_unwind_handler_entry);
 197   __ verify_not_null_oop(Z_EXC_OOP);
 198   if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
 199     __ lgr_if_needed(exception_oop_callee_saved, Z_EXC_OOP); // Preserve the exception.
 200   }
 201 
 202   // Preform needed unlocking.
 203   MonitorExitStub* stub = NULL;
 204   if (method()->is_synchronized()) {
 205     // Runtime1::monitorexit_id expects lock address in Z_R1_scratch.
 206     LIR_Opr lock = FrameMap::as_opr(Z_R1_scratch);
 207     monitor_address(0, lock);
 208     stub = new MonitorExitStub(lock, true, 0);
 209     __ unlock_object(Rtmp1, Rtmp2, lock->as_register(), *stub->entry());
 210     __ bind(*stub->continuation());
 211   }
 212 
 213   if (compilation()->env()->dtrace_method_probes()) {
 214     ShouldNotReachHere(); // Not supported.
 215 #if 0
 216     __ mov(rdi, r15_thread);
 217     __ mov_metadata(rsi, method()->constant_encoding());
 218     __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_exit)));
 219 #endif
 220   }
 221 
 222   if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
 223     __ lgr_if_needed(Z_EXC_OOP, exception_oop_callee_saved);  // Restore the exception.
 224   }
 225 
 226   // Remove the activation and dispatch to the unwind handler.
 227   __ pop_frame();
 228   __ z_lg(Z_EXC_PC, _z_abi16(return_pc), Z_SP);
 229 
 230   // Z_EXC_OOP: exception oop
 231   // Z_EXC_PC: exception pc
 232 
 233   // Dispatch to the unwind logic.
 234   __ load_const_optimized(Z_R5, Runtime1::entry_for (Runtime1::unwind_exception_id));
 235   __ z_br(Z_R5);
 236 
 237   // Emit the slow path assembly.
 238   if (stub != NULL) {
 239     stub->emit_code(this);
 240   }
 241 
 242   return offset;
 243 }
 244 
 245 int LIR_Assembler::emit_deopt_handler() {
 246   // If the last instruction is a call (typically to do a throw which
 247   // is coming at the end after block reordering) the return address
 248   // must still point into the code area in order to avoid assertion
 249   // failures when searching for the corresponding bci. => Add a nop.
 250   // (was bug 5/14/1999 - gri)
 251   __ nop();
 252 
 253   // Generate code for exception handler.
 254   address handler_base = __ start_a_stub(deopt_handler_size);
 255   if (handler_base == NULL) {
 256     // Not enough space left for the handler.
 257     bailout("deopt handler overflow");
 258     return -1;
 259   }  int offset = code_offset();
 260   // Size must be constant (see HandlerImpl::emit_deopt_handler).
 261   __ load_const(Z_R1_scratch, SharedRuntime::deopt_blob()->unpack());
 262   __ call(Z_R1_scratch);
 263   guarantee(code_offset() - offset <= deopt_handler_size, "overflow");
 264   __ end_a_stub();
 265 
 266   return offset;
 267 }
 268 
 269 void LIR_Assembler::jobject2reg(jobject o, Register reg) {
 270   if (o == NULL) {
 271     __ clear_reg(reg, true/*64bit*/, false/*set cc*/); // Must not kill cc set by cmove.
 272   } else {
 273     AddressLiteral a = __ allocate_oop_address(o);
 274     bool success = __ load_oop_from_toc(reg, a, reg);
 275     if (!success) {
 276       bailout("const section overflow");
 277     }
 278   }
 279 }
 280 
 281 void LIR_Assembler::jobject2reg_with_patching(Register reg, CodeEmitInfo *info) {
 282   // Allocate a new index in table to hold the object once it's been patched.
 283   int oop_index = __ oop_recorder()->allocate_oop_index(NULL);
 284   PatchingStub* patch = new PatchingStub(_masm, patching_id(info), oop_index);
 285 
 286   AddressLiteral addrlit((intptr_t)0, oop_Relocation::spec(oop_index));
 287   assert(addrlit.rspec().type() == relocInfo::oop_type, "must be an oop reloc");
 288   // The NULL will be dynamically patched later so the sequence to
 289   // load the address literal must not be optimized.
 290   __ load_const(reg, addrlit);
 291 
 292   patching_epilog(patch, lir_patch_normal, reg, info);
 293 }
 294 
 295 void LIR_Assembler::metadata2reg(Metadata* md, Register reg) {
 296   bool success = __ set_metadata_constant(md, reg);
 297   if (!success) {
 298     bailout("const section overflow");
 299     return;
 300   }
 301 }
 302 
 303 void LIR_Assembler::klass2reg_with_patching(Register reg, CodeEmitInfo *info) {
 304   // Allocate a new index in table to hold the klass once it's been patched.
 305   int index = __ oop_recorder()->allocate_metadata_index(NULL);
 306   PatchingStub* patch = new PatchingStub(_masm, PatchingStub::load_klass_id, index);
 307   AddressLiteral addrlit((intptr_t)0, metadata_Relocation::spec(index));
 308   assert(addrlit.rspec().type() == relocInfo::metadata_type, "must be an metadata reloc");
 309   // The NULL will be dynamically patched later so the sequence to
 310   // load the address literal must not be optimized.
 311   __ load_const(reg, addrlit);
 312 
 313   patching_epilog(patch, lir_patch_normal, reg, info);
 314 }
 315 
 316 void LIR_Assembler::emit_op3(LIR_Op3* op) {
 317   switch (op->code()) {
 318     case lir_idiv:
 319     case lir_irem:
 320       arithmetic_idiv(op->code(),
 321                       op->in_opr1(),
 322                       op->in_opr2(),
 323                       op->in_opr3(),
 324                       op->result_opr(),
 325                       op->info());
 326       break;
 327     default: ShouldNotReachHere(); break;
 328   }
 329 }
 330 
 331 
 332 void LIR_Assembler::emit_opBranch(LIR_OpBranch* op) {
 333 #ifdef ASSERT
 334   assert(op->block() == NULL || op->block()->label() == op->label(), "wrong label");
 335   if (op->block() != NULL)  { _branch_target_blocks.append(op->block()); }
 336   if (op->ublock() != NULL) { _branch_target_blocks.append(op->ublock()); }
 337 #endif
 338 
 339   if (op->cond() == lir_cond_always) {
 340     if (op->info() != NULL) { add_debug_info_for_branch(op->info()); }
 341     __ branch_optimized(Assembler::bcondAlways, *(op->label()));
 342   } else {
 343     Assembler::branch_condition acond = Assembler::bcondZero;
 344     if (op->code() == lir_cond_float_branch) {
 345       assert(op->ublock() != NULL, "must have unordered successor");
 346       __ branch_optimized(Assembler::bcondNotOrdered, *(op->ublock()->label()));
 347     }
 348     switch (op->cond()) {
 349       case lir_cond_equal:        acond = Assembler::bcondEqual;     break;
 350       case lir_cond_notEqual:     acond = Assembler::bcondNotEqual;  break;
 351       case lir_cond_less:         acond = Assembler::bcondLow;       break;
 352       case lir_cond_lessEqual:    acond = Assembler::bcondNotHigh;   break;
 353       case lir_cond_greaterEqual: acond = Assembler::bcondNotLow;    break;
 354       case lir_cond_greater:      acond = Assembler::bcondHigh;      break;
 355       case lir_cond_belowEqual:   acond = Assembler::bcondNotHigh;   break;
 356       case lir_cond_aboveEqual:   acond = Assembler::bcondNotLow;    break;
 357       default:                         ShouldNotReachHere();
 358     }
 359     __ branch_optimized(acond,*(op->label()));
 360   }
 361 }
 362 
 363 
 364 void LIR_Assembler::emit_opConvert(LIR_OpConvert* op) {
 365   LIR_Opr src  = op->in_opr();
 366   LIR_Opr dest = op->result_opr();
 367 
 368   switch (op->bytecode()) {
 369     case Bytecodes::_i2l:
 370       __ move_reg_if_needed(dest->as_register_lo(), T_LONG, src->as_register(), T_INT);
 371       break;
 372 
 373     case Bytecodes::_l2i:
 374       __ move_reg_if_needed(dest->as_register(), T_INT, src->as_register_lo(), T_LONG);
 375       break;
 376 
 377     case Bytecodes::_i2b:
 378       __ move_reg_if_needed(dest->as_register(), T_BYTE, src->as_register(), T_INT);
 379       break;
 380 
 381     case Bytecodes::_i2c:
 382       __ move_reg_if_needed(dest->as_register(), T_CHAR, src->as_register(), T_INT);
 383       break;
 384 
 385     case Bytecodes::_i2s:
 386       __ move_reg_if_needed(dest->as_register(), T_SHORT, src->as_register(), T_INT);
 387       break;
 388 
 389     case Bytecodes::_f2d:
 390       assert(dest->is_double_fpu(), "check");
 391       __ move_freg_if_needed(dest->as_double_reg(), T_DOUBLE, src->as_float_reg(), T_FLOAT);
 392       break;
 393 
 394     case Bytecodes::_d2f:
 395       assert(dest->is_single_fpu(), "check");
 396       __ move_freg_if_needed(dest->as_float_reg(), T_FLOAT, src->as_double_reg(), T_DOUBLE);
 397       break;
 398 
 399     case Bytecodes::_i2f:
 400       __ z_cefbr(dest->as_float_reg(), src->as_register());
 401       break;
 402 
 403     case Bytecodes::_i2d:
 404       __ z_cdfbr(dest->as_double_reg(), src->as_register());
 405       break;
 406 
 407     case Bytecodes::_l2f:
 408       __ z_cegbr(dest->as_float_reg(), src->as_register_lo());
 409       break;
 410     case Bytecodes::_l2d:
 411       __ z_cdgbr(dest->as_double_reg(), src->as_register_lo());
 412       break;
 413 
 414     case Bytecodes::_f2i:
 415     case Bytecodes::_f2l: {
 416       Label done;
 417       FloatRegister Rsrc = src->as_float_reg();
 418       Register Rdst = (op->bytecode() == Bytecodes::_f2i ? dest->as_register() : dest->as_register_lo());
 419       __ clear_reg(Rdst, true, false);
 420       __ z_cebr(Rsrc, Rsrc);
 421       __ z_brno(done); // NaN -> 0
 422       if (op->bytecode() == Bytecodes::_f2i) {
 423         __ z_cfebr(Rdst, Rsrc, Assembler::to_zero);
 424       } else { // op->bytecode() == Bytecodes::_f2l
 425         __ z_cgebr(Rdst, Rsrc, Assembler::to_zero);
 426       }
 427       __ bind(done);
 428     }
 429     break;
 430 
 431     case Bytecodes::_d2i:
 432     case Bytecodes::_d2l: {
 433       Label done;
 434       FloatRegister Rsrc = src->as_double_reg();
 435       Register Rdst = (op->bytecode() == Bytecodes::_d2i ? dest->as_register() : dest->as_register_lo());
 436       __ clear_reg(Rdst, true, false);  // Don't set CC.
 437       __ z_cdbr(Rsrc, Rsrc);
 438       __ z_brno(done); // NaN -> 0
 439       if (op->bytecode() == Bytecodes::_d2i) {
 440         __ z_cfdbr(Rdst, Rsrc, Assembler::to_zero);
 441       } else { // Bytecodes::_d2l
 442         __ z_cgdbr(Rdst, Rsrc, Assembler::to_zero);
 443       }
 444       __ bind(done);
 445     }
 446     break;
 447 
 448     default: ShouldNotReachHere();
 449   }
 450 }
 451 
 452 void LIR_Assembler::align_call(LIR_Code code) {
 453   // End of call instruction must be 4 byte aligned.
 454   int offset = __ offset();
 455   switch (code) {
 456     case lir_icvirtual_call:
 457       offset += MacroAssembler::load_const_from_toc_size();
 458       // no break
 459     case lir_static_call:
 460     case lir_optvirtual_call:
 461     case lir_dynamic_call:
 462       offset += NativeCall::call_far_pcrelative_displacement_offset;
 463       break;
 464     case lir_virtual_call:   // currently, sparc-specific for niagara
 465     default: ShouldNotReachHere();
 466   }
 467   if ((offset & (NativeCall::call_far_pcrelative_displacement_alignment-1)) != 0) {
 468     __ nop();
 469   }
 470 }
 471 
 472 void LIR_Assembler::call(LIR_OpJavaCall* op, relocInfo::relocType rtype) {
 473   assert((__ offset() + NativeCall::call_far_pcrelative_displacement_offset) % NativeCall::call_far_pcrelative_displacement_alignment == 0,
 474          "must be aligned (offset=%d)", __ offset());
 475   assert(rtype == relocInfo::none ||
 476          rtype == relocInfo::opt_virtual_call_type ||
 477          rtype == relocInfo::static_call_type, "unexpected rtype");
 478   // Prepend each BRASL with a nop.
 479   __ relocate(rtype);
 480   __ z_nop();
 481   __ z_brasl(Z_R14, op->addr());
 482   add_call_info(code_offset(), op->info());
 483 }
 484 
 485 void LIR_Assembler::ic_call(LIR_OpJavaCall* op) {
 486   address virtual_call_oop_addr = NULL;
 487   AddressLiteral empty_ic((address) Universe::non_oop_word());
 488   virtual_call_oop_addr = __ pc();
 489   bool success = __ load_const_from_toc(Z_inline_cache, empty_ic);
 490   if (!success) {
 491     bailout("const section overflow");
 492     return;
 493   }
 494 
 495   // CALL to fixup routine. Fixup routine uses ScopeDesc info
 496   // to determine who we intended to call.
 497   __ relocate(virtual_call_Relocation::spec(virtual_call_oop_addr));
 498   call(op, relocInfo::none);
 499 }
 500 
 501 // not supported
 502 void LIR_Assembler::vtable_call(LIR_OpJavaCall* op) {
 503   ShouldNotReachHere();
 504 }
 505 
 506 void LIR_Assembler::move_regs(Register from_reg, Register to_reg) {
 507   if (from_reg != to_reg) __ z_lgr(to_reg, from_reg);
 508 }
 509 
 510 void LIR_Assembler::const2stack(LIR_Opr src, LIR_Opr dest) {
 511   assert(src->is_constant(), "should not call otherwise");
 512   assert(dest->is_stack(), "should not call otherwise");
 513   LIR_Const* c = src->as_constant_ptr();
 514 
 515   unsigned int lmem = 0;
 516   unsigned int lcon = 0;
 517   int64_t cbits = 0;
 518   Address dest_addr;
 519   switch (c->type()) {
 520     case T_INT:  // fall through
 521     case T_FLOAT:
 522       dest_addr = frame_map()->address_for_slot(dest->single_stack_ix());
 523       lmem = 4; lcon = 4; cbits = c->as_jint_bits();
 524       break;
 525 
 526     case T_ADDRESS:
 527       dest_addr = frame_map()->address_for_slot(dest->single_stack_ix());
 528       lmem = 8; lcon = 4; cbits = c->as_jint_bits();
 529       break;
 530 
 531     case T_OBJECT:
 532       dest_addr = frame_map()->address_for_slot(dest->single_stack_ix());
 533       if (c->as_jobject() == NULL) {
 534         __ store_const(dest_addr, (int64_t)NULL_WORD, 8, 8);
 535       } else {
 536         jobject2reg(c->as_jobject(), Z_R1_scratch);
 537         __ reg2mem_opt(Z_R1_scratch, dest_addr, true);
 538       }
 539       return;
 540 
 541     case T_LONG:  // fall through
 542     case T_DOUBLE:
 543       dest_addr = frame_map()->address_for_slot(dest->double_stack_ix());
 544       lmem = 8; lcon = 8; cbits = (int64_t)(c->as_jlong_bits());
 545       break;
 546 
 547     default:
 548       ShouldNotReachHere();
 549   }
 550 
 551   __ store_const(dest_addr, cbits, lmem, lcon);
 552 }
 553 
 554 void LIR_Assembler::const2mem(LIR_Opr src, LIR_Opr dest, BasicType type, CodeEmitInfo* info, bool wide) {
 555   assert(src->is_constant(), "should not call otherwise");
 556   assert(dest->is_address(), "should not call otherwise");
 557   // See special case in LIRGenerator::do_StoreIndexed.
 558   // T_BYTE: Special case for card mark store.
 559   assert(type == T_BYTE || !dest->as_address_ptr()->index()->is_valid(), "not supported");
 560   LIR_Const* c = src->as_constant_ptr();
 561   Address addr = as_Address(dest->as_address_ptr());
 562 
 563   int store_offset = -1;
 564   unsigned int lmem = 0;
 565   unsigned int lcon = 0;
 566   int64_t cbits = 0;
 567   switch (type) {
 568     case T_INT:    // fall through
 569     case T_FLOAT:
 570       lmem = 4; lcon = 4; cbits = c->as_jint_bits();
 571       break;
 572 
 573     case T_ADDRESS:
 574       lmem = 8; lcon = 4; cbits = c->as_jint_bits();
 575       break;
 576 
 577     case T_OBJECT:  // fall through
 578     case T_ARRAY:
 579       if (c->as_jobject() == NULL) {
 580         if (UseCompressedOops && !wide) {
 581           store_offset = __ store_const(addr, (int32_t)NULL_WORD, 4, 4);
 582         } else {
 583           store_offset = __ store_const(addr, (int64_t)NULL_WORD, 8, 8);
 584         }
 585       } else {
 586         jobject2reg(c->as_jobject(), Z_R1_scratch);
 587         if (UseCompressedOops && !wide) {
 588           __ encode_heap_oop(Z_R1_scratch);
 589           store_offset = __ reg2mem_opt(Z_R1_scratch, addr, false);
 590         } else {
 591           store_offset = __ reg2mem_opt(Z_R1_scratch, addr, true);
 592         }
 593       }
 594       assert(store_offset >= 0, "check");
 595       break;
 596 
 597     case T_LONG:    // fall through
 598     case T_DOUBLE:
 599       lmem = 8; lcon = 8; cbits = (int64_t)(c->as_jlong_bits());
 600       break;
 601 
 602     case T_BOOLEAN: // fall through
 603     case T_BYTE:
 604       lmem = 1; lcon = 1; cbits = (int8_t)(c->as_jint());
 605       break;
 606 
 607     case T_CHAR:    // fall through
 608     case T_SHORT:
 609       lmem = 2; lcon = 2; cbits = (int16_t)(c->as_jint());
 610       break;
 611 
 612     default:
 613       ShouldNotReachHere();
 614   };
 615 
 616   // Index register is normally not supported, but for
 617   // LIRGenerator::CardTableModRef_post_barrier we make an exception.
 618   if (type == T_BYTE && dest->as_address_ptr()->index()->is_valid()) {
 619     __ load_const_optimized(Z_R0_scratch, (int8_t)(c->as_jint()));
 620     store_offset = __ offset();
 621     if (Immediate::is_uimm12(addr.disp())) {
 622       __ z_stc(Z_R0_scratch, addr);
 623     } else {
 624       __ z_stcy(Z_R0_scratch, addr);
 625     }
 626   }
 627 
 628   if (store_offset == -1) {
 629     store_offset = __ store_const(addr, cbits, lmem, lcon);
 630     assert(store_offset >= 0, "check");
 631   }
 632 
 633   if (info != NULL) {
 634     add_debug_info_for_null_check(store_offset, info);
 635   }
 636 }
 637 
 638 void LIR_Assembler::const2reg(LIR_Opr src, LIR_Opr dest, LIR_PatchCode patch_code, CodeEmitInfo* info) {
 639   assert(src->is_constant(), "should not call otherwise");
 640   assert(dest->is_register(), "should not call otherwise");
 641   LIR_Const* c = src->as_constant_ptr();
 642 
 643   switch (c->type()) {
 644     case T_INT: {
 645       assert(patch_code == lir_patch_none, "no patching handled here");
 646       __ load_const_optimized(dest->as_register(), c->as_jint());
 647       break;
 648     }
 649 
 650     case T_ADDRESS: {
 651       assert(patch_code == lir_patch_none, "no patching handled here");
 652       __ load_const_optimized(dest->as_register(), c->as_jint());
 653       break;
 654     }
 655 
 656     case T_LONG: {
 657       assert(patch_code == lir_patch_none, "no patching handled here");
 658       __ load_const_optimized(dest->as_register_lo(), (intptr_t)c->as_jlong());
 659       break;
 660     }
 661 
 662     case T_OBJECT: {
 663       if (patch_code != lir_patch_none) {
 664         jobject2reg_with_patching(dest->as_register(), info);
 665       } else {
 666         jobject2reg(c->as_jobject(), dest->as_register());
 667       }
 668       break;
 669     }
 670 
 671     case T_METADATA: {
 672       if (patch_code != lir_patch_none) {
 673         klass2reg_with_patching(dest->as_register(), info);
 674       } else {
 675         metadata2reg(c->as_metadata(), dest->as_register());
 676       }
 677       break;
 678     }
 679 
 680     case T_FLOAT: {
 681       Register toc_reg = Z_R1_scratch;
 682       __ load_toc(toc_reg);
 683       address const_addr = __ float_constant(c->as_jfloat());
 684       if (const_addr == NULL) {
 685         bailout("const section overflow");
 686         break;
 687       }
 688       int displ = const_addr - _masm->code()->consts()->start();
 689       if (dest->is_single_fpu()) {
 690         __ z_ley(dest->as_float_reg(), displ, toc_reg);
 691       } else {
 692         assert(dest->is_single_cpu(), "Must be a cpu register.");
 693         __ z_ly(dest->as_register(), displ, toc_reg);
 694       }
 695     }
 696     break;
 697 
 698     case T_DOUBLE: {
 699       Register toc_reg = Z_R1_scratch;
 700       __ load_toc(toc_reg);
 701       address const_addr = __ double_constant(c->as_jdouble());
 702       if (const_addr == NULL) {
 703         bailout("const section overflow");
 704         break;
 705       }
 706       int displ = const_addr - _masm->code()->consts()->start();
 707       if (dest->is_double_fpu()) {
 708         __ z_ldy(dest->as_double_reg(), displ, toc_reg);
 709       } else {
 710         assert(dest->is_double_cpu(), "Must be a long register.");
 711         __ z_lg(dest->as_register_lo(), displ, toc_reg);
 712       }
 713     }
 714     break;
 715 
 716     default:
 717       ShouldNotReachHere();
 718   }
 719 }
 720 
 721 Address LIR_Assembler::as_Address(LIR_Address* addr) {
 722   if (addr->base()->is_illegal()) {
 723     Unimplemented();
 724   }
 725 
 726   Register base = addr->base()->as_pointer_register();
 727 
 728   if (addr->index()->is_illegal()) {
 729     return Address(base, addr->disp());
 730   } else if (addr->index()->is_cpu_register()) {
 731     Register index = addr->index()->as_pointer_register();
 732     return Address(base, index, addr->disp());
 733   } else if (addr->index()->is_constant()) {
 734     intptr_t addr_offset = addr->index()->as_constant_ptr()->as_jint() + addr->disp();
 735     return Address(base, addr_offset);
 736   } else {
 737     ShouldNotReachHere();
 738     return Address();
 739   }
 740 }
 741 
 742 void LIR_Assembler::stack2stack(LIR_Opr src, LIR_Opr dest, BasicType type) {
 743   switch (type) {
 744     case T_INT:
 745     case T_FLOAT: {
 746       Register tmp = Z_R1_scratch;
 747       Address from = frame_map()->address_for_slot(src->single_stack_ix());
 748       Address to   = frame_map()->address_for_slot(dest->single_stack_ix());
 749       __ mem2reg_opt(tmp, from, false);
 750       __ reg2mem_opt(tmp, to, false);
 751       break;
 752     }
 753     case T_ADDRESS:
 754     case T_OBJECT: {
 755       Register tmp = Z_R1_scratch;
 756       Address from = frame_map()->address_for_slot(src->single_stack_ix());
 757       Address to   = frame_map()->address_for_slot(dest->single_stack_ix());
 758       __ mem2reg_opt(tmp, from, true);
 759       __ reg2mem_opt(tmp, to, true);
 760       break;
 761     }
 762     case T_LONG:
 763     case T_DOUBLE: {
 764       Register tmp = Z_R1_scratch;
 765       Address from = frame_map()->address_for_double_slot(src->double_stack_ix());
 766       Address to   = frame_map()->address_for_double_slot(dest->double_stack_ix());
 767       __ mem2reg_opt(tmp, from, true);
 768       __ reg2mem_opt(tmp, to, true);
 769       break;
 770     }
 771 
 772     default:
 773       ShouldNotReachHere();
 774   }
 775 }
 776 
 777 // 4-byte accesses only! Don't use it to access 8 bytes!
 778 Address LIR_Assembler::as_Address_hi(LIR_Address* addr) {
 779   ShouldNotCallThis();
 780   return 0; // unused
 781 }
 782 
 783 // 4-byte accesses only! Don't use it to access 8 bytes!
 784 Address LIR_Assembler::as_Address_lo(LIR_Address* addr) {
 785   ShouldNotCallThis();
 786   return 0; // unused
 787 }
 788 
 789 void LIR_Assembler::mem2reg(LIR_Opr src_opr, LIR_Opr dest, BasicType type, LIR_PatchCode patch_code,
 790                             CodeEmitInfo* info, bool wide, bool unaligned) {
 791 
 792   assert(type != T_METADATA, "load of metadata ptr not supported");
 793   LIR_Address* addr = src_opr->as_address_ptr();
 794   LIR_Opr to_reg = dest;
 795 
 796   Register src = addr->base()->as_pointer_register();
 797   Register disp_reg = Z_R0;
 798   int disp_value = addr->disp();
 799   bool needs_patching = (patch_code != lir_patch_none);
 800 
 801   if (addr->base()->type() == T_OBJECT) {
 802     __ verify_oop(src);
 803   }
 804 
 805   PatchingStub* patch = NULL;
 806   if (needs_patching) {
 807     patch = new PatchingStub(_masm, PatchingStub::access_field_id);
 808     assert(!to_reg->is_double_cpu() ||
 809            patch_code == lir_patch_none ||
 810            patch_code == lir_patch_normal, "patching doesn't match register");
 811   }
 812 
 813   if (addr->index()->is_illegal()) {
 814     if (!Immediate::is_simm20(disp_value)) {
 815       if (needs_patching) {
 816         __ load_const(Z_R1_scratch, (intptr_t)0);
 817       } else {
 818         __ load_const_optimized(Z_R1_scratch, disp_value);
 819       }
 820       disp_reg = Z_R1_scratch;
 821       disp_value = 0;
 822     }
 823   } else {
 824     if (!Immediate::is_simm20(disp_value)) {
 825       __ load_const_optimized(Z_R1_scratch, disp_value);
 826       __ z_la(Z_R1_scratch, 0, Z_R1_scratch, addr->index()->as_register());
 827       disp_reg = Z_R1_scratch;
 828       disp_value = 0;
 829     }
 830     disp_reg = addr->index()->as_pointer_register();
 831   }
 832 
 833   // Remember the offset of the load. The patching_epilog must be done
 834   // before the call to add_debug_info, otherwise the PcDescs don't get
 835   // entered in increasing order.
 836   int offset = code_offset();
 837 
 838   assert(disp_reg != Z_R0 || Immediate::is_simm20(disp_value), "should have set this up");
 839 
 840   bool short_disp = Immediate::is_uimm12(disp_value);
 841 
 842   switch (type) {
 843     case T_BOOLEAN: // fall through
 844     case T_BYTE  :  __ z_lb(dest->as_register(),   disp_value, disp_reg, src); break;
 845     case T_CHAR  :  __ z_llgh(dest->as_register(), disp_value, disp_reg, src); break;
 846     case T_SHORT :
 847       if (short_disp) {
 848                     __ z_lh(dest->as_register(),   disp_value, disp_reg, src);
 849       } else {
 850                     __ z_lhy(dest->as_register(),  disp_value, disp_reg, src);
 851       }
 852       break;
 853     case T_INT   :
 854       if (short_disp) {
 855                     __ z_l(dest->as_register(),    disp_value, disp_reg, src);
 856       } else {
 857                     __ z_ly(dest->as_register(),   disp_value, disp_reg, src);
 858       }
 859       break;
 860     case T_ADDRESS:
 861       if (UseCompressedClassPointers && addr->disp() == oopDesc::klass_offset_in_bytes()) {
 862         __ z_llgf(dest->as_register(), disp_value, disp_reg, src);
 863         __ decode_klass_not_null(dest->as_register());
 864       } else {
 865         __ z_lg(dest->as_register(), disp_value, disp_reg, src);
 866       }
 867       break;
 868     case T_ARRAY : // fall through
 869     case T_OBJECT:
 870     {
 871       if (UseCompressedOops && !wide) {
 872         __ z_llgf(dest->as_register(), disp_value, disp_reg, src);
 873         __ oop_decoder(dest->as_register(), dest->as_register(), true);
 874       } else {
 875         __ z_lg(dest->as_register(), disp_value, disp_reg, src);
 876       }
 877       break;
 878     }
 879     case T_FLOAT:
 880       if (short_disp) {
 881                     __ z_le(dest->as_float_reg(),  disp_value, disp_reg, src);
 882       } else {
 883                     __ z_ley(dest->as_float_reg(), disp_value, disp_reg, src);
 884       }
 885       break;
 886     case T_DOUBLE:
 887       if (short_disp) {
 888                     __ z_ld(dest->as_double_reg(),  disp_value, disp_reg, src);
 889       } else {
 890                     __ z_ldy(dest->as_double_reg(), disp_value, disp_reg, src);
 891       }
 892       break;
 893     case T_LONG  :  __ z_lg(dest->as_register_lo(), disp_value, disp_reg, src); break;
 894     default      : ShouldNotReachHere();
 895   }
 896   if (type == T_ARRAY || type == T_OBJECT) {
 897     __ verify_oop(dest->as_register());
 898   }
 899 
 900   if (patch != NULL) {
 901     patching_epilog(patch, patch_code, src, info);
 902   }
 903   if (info != NULL) add_debug_info_for_null_check(offset, info);
 904 }
 905 
 906 void LIR_Assembler::stack2reg(LIR_Opr src, LIR_Opr dest, BasicType type) {
 907   assert(src->is_stack(), "should not call otherwise");
 908   assert(dest->is_register(), "should not call otherwise");
 909 
 910   if (dest->is_single_cpu()) {
 911     if (type == T_ARRAY || type == T_OBJECT) {
 912       __ mem2reg_opt(dest->as_register(), frame_map()->address_for_slot(src->single_stack_ix()), true);
 913       __ verify_oop(dest->as_register());
 914     } else if (type == T_METADATA) {
 915       __ mem2reg_opt(dest->as_register(), frame_map()->address_for_slot(src->single_stack_ix()), true);
 916     } else {
 917       __ mem2reg_opt(dest->as_register(), frame_map()->address_for_slot(src->single_stack_ix()), false);
 918     }
 919   } else if (dest->is_double_cpu()) {
 920     Address src_addr_LO = frame_map()->address_for_slot(src->double_stack_ix());
 921     __ mem2reg_opt(dest->as_register_lo(), src_addr_LO, true);
 922   } else if (dest->is_single_fpu()) {
 923     Address src_addr = frame_map()->address_for_slot(src->single_stack_ix());
 924     __ mem2freg_opt(dest->as_float_reg(), src_addr, false);
 925   } else if (dest->is_double_fpu()) {
 926     Address src_addr = frame_map()->address_for_slot(src->double_stack_ix());
 927     __ mem2freg_opt(dest->as_double_reg(), src_addr, true);
 928   } else {
 929     ShouldNotReachHere();
 930   }
 931 }
 932 
 933 void LIR_Assembler::reg2stack(LIR_Opr src, LIR_Opr dest, BasicType type, bool pop_fpu_stack) {
 934   assert(src->is_register(), "should not call otherwise");
 935   assert(dest->is_stack(), "should not call otherwise");
 936 
 937   if (src->is_single_cpu()) {
 938     const Address dst = frame_map()->address_for_slot(dest->single_stack_ix());
 939     if (type == T_OBJECT || type == T_ARRAY) {
 940       __ verify_oop(src->as_register());
 941       __ reg2mem_opt(src->as_register(), dst, true);
 942     } else if (type == T_METADATA) {
 943       __ reg2mem_opt(src->as_register(), dst, true);
 944     } else {
 945       __ reg2mem_opt(src->as_register(), dst, false);
 946     }
 947   } else if (src->is_double_cpu()) {
 948     Address dstLO = frame_map()->address_for_slot(dest->double_stack_ix());
 949     __ reg2mem_opt(src->as_register_lo(), dstLO, true);
 950   } else if (src->is_single_fpu()) {
 951     Address dst_addr = frame_map()->address_for_slot(dest->single_stack_ix());
 952     __ freg2mem_opt(src->as_float_reg(), dst_addr, false);
 953   } else if (src->is_double_fpu()) {
 954     Address dst_addr = frame_map()->address_for_slot(dest->double_stack_ix());
 955     __ freg2mem_opt(src->as_double_reg(), dst_addr, true);
 956   } else {
 957     ShouldNotReachHere();
 958   }
 959 }
 960 
 961 void LIR_Assembler::reg2reg(LIR_Opr from_reg, LIR_Opr to_reg) {
 962   if (from_reg->is_float_kind() && to_reg->is_float_kind()) {
 963     if (from_reg->is_double_fpu()) {
 964       // double to double moves
 965       assert(to_reg->is_double_fpu(), "should match");
 966       __ z_ldr(to_reg->as_double_reg(), from_reg->as_double_reg());
 967     } else {
 968       // float to float moves
 969       assert(to_reg->is_single_fpu(), "should match");
 970       __ z_ler(to_reg->as_float_reg(), from_reg->as_float_reg());
 971     }
 972   } else if (!from_reg->is_float_kind() && !to_reg->is_float_kind()) {
 973     if (from_reg->is_double_cpu()) {
 974       __ z_lgr(to_reg->as_pointer_register(), from_reg->as_pointer_register());
 975     } else if (to_reg->is_double_cpu()) {
 976       // int to int moves
 977       __ z_lgr(to_reg->as_register_lo(), from_reg->as_register());
 978     } else {
 979       // int to int moves
 980       __ z_lgr(to_reg->as_register(), from_reg->as_register());
 981     }
 982   } else {
 983     ShouldNotReachHere();
 984   }
 985   if (to_reg->type() == T_OBJECT || to_reg->type() == T_ARRAY) {
 986     __ verify_oop(to_reg->as_register());
 987   }
 988 }
 989 
 990 void LIR_Assembler::reg2mem(LIR_Opr from, LIR_Opr dest_opr, BasicType type,
 991                             LIR_PatchCode patch_code, CodeEmitInfo* info, bool pop_fpu_stack,
 992                             bool wide, bool unaligned) {
 993   assert(type != T_METADATA, "store of metadata ptr not supported");
 994   LIR_Address* addr = dest_opr->as_address_ptr();
 995 
 996   Register dest = addr->base()->as_pointer_register();
 997   Register disp_reg = Z_R0;
 998   int disp_value = addr->disp();
 999   bool needs_patching = (patch_code != lir_patch_none);
1000 
1001   if (addr->base()->is_oop_register()) {
1002     __ verify_oop(dest);
1003   }
1004 
1005   PatchingStub* patch = NULL;
1006   if (needs_patching) {
1007     patch = new PatchingStub(_masm, PatchingStub::access_field_id);
1008     assert(!from->is_double_cpu() ||
1009            patch_code == lir_patch_none ||
1010            patch_code == lir_patch_normal, "patching doesn't match register");
1011   }
1012 
1013   assert(!needs_patching || (!Immediate::is_simm20(disp_value) && addr->index()->is_illegal()), "assumption");
1014   if (addr->index()->is_illegal()) {
1015     if (!Immediate::is_simm20(disp_value)) {
1016       if (needs_patching) {
1017         __ load_const(Z_R1_scratch, (intptr_t)0);
1018       } else {
1019         __ load_const_optimized(Z_R1_scratch, disp_value);
1020       }
1021       disp_reg = Z_R1_scratch;
1022       disp_value = 0;
1023     }
1024   } else {
1025     if (!Immediate::is_simm20(disp_value)) {
1026       __ load_const_optimized(Z_R1_scratch, disp_value);
1027       __ z_la(Z_R1_scratch, 0, Z_R1_scratch, addr->index()->as_register());
1028       disp_reg = Z_R1_scratch;
1029       disp_value = 0;
1030     }
1031     disp_reg = addr->index()->as_pointer_register();
1032   }
1033 
1034   assert(disp_reg != Z_R0 || Immediate::is_simm20(disp_value), "should have set this up");
1035 
1036   if (type == T_ARRAY || type == T_OBJECT) {
1037     __ verify_oop(from->as_register());
1038   }
1039 
1040   bool short_disp = Immediate::is_uimm12(disp_value);
1041 
1042   // Remember the offset of the store. The patching_epilog must be done
1043   // before the call to add_debug_info_for_null_check, otherwise the PcDescs don't get
1044   // entered in increasing order.
1045   int offset = code_offset();
1046   switch (type) {
1047     case T_BOOLEAN: // fall through
1048     case T_BYTE  :
1049       if (short_disp) {
1050                     __ z_stc(from->as_register(),  disp_value, disp_reg, dest);
1051       } else {
1052                     __ z_stcy(from->as_register(), disp_value, disp_reg, dest);
1053       }
1054       break;
1055     case T_CHAR  : // fall through
1056     case T_SHORT :
1057       if (short_disp) {
1058                     __ z_sth(from->as_register(),  disp_value, disp_reg, dest);
1059       } else {
1060                     __ z_sthy(from->as_register(), disp_value, disp_reg, dest);
1061       }
1062       break;
1063     case T_INT   :
1064       if (short_disp) {
1065                     __ z_st(from->as_register(),  disp_value, disp_reg, dest);
1066       } else {
1067                     __ z_sty(from->as_register(), disp_value, disp_reg, dest);
1068       }
1069       break;
1070     case T_LONG  :  __ z_stg(from->as_register_lo(), disp_value, disp_reg, dest); break;
1071     case T_ADDRESS: __ z_stg(from->as_register(),    disp_value, disp_reg, dest); break;
1072       break;
1073     case T_ARRAY : // fall through
1074     case T_OBJECT:
1075       {
1076         if (UseCompressedOops && !wide) {
1077           Register compressed_src = Z_R14;
1078           __ oop_encoder(compressed_src, from->as_register(), true, (disp_reg != Z_R1) ? Z_R1 : Z_R0, -1, true);
1079           offset = code_offset();
1080           if (short_disp) {
1081             __ z_st(compressed_src,  disp_value, disp_reg, dest);
1082           } else {
1083             __ z_sty(compressed_src, disp_value, disp_reg, dest);
1084           }
1085         } else {
1086           __ z_stg(from->as_register(), disp_value, disp_reg, dest);
1087         }
1088         break;
1089       }
1090     case T_FLOAT :
1091       if (short_disp) {
1092                     __ z_ste(from->as_float_reg(),  disp_value, disp_reg, dest);
1093       } else {
1094                     __ z_stey(from->as_float_reg(), disp_value, disp_reg, dest);
1095       }
1096       break;
1097     case T_DOUBLE:
1098       if (short_disp) {
1099                     __ z_std(from->as_double_reg(),  disp_value, disp_reg, dest);
1100       } else {
1101                     __ z_stdy(from->as_double_reg(), disp_value, disp_reg, dest);
1102       }
1103       break;
1104     default: ShouldNotReachHere();
1105   }
1106 
1107   if (patch != NULL) {
1108     patching_epilog(patch, patch_code, dest, info);
1109   }
1110 
1111   if (info != NULL) add_debug_info_for_null_check(offset, info);
1112 }
1113 
1114 
1115 void LIR_Assembler::return_op(LIR_Opr result) {
1116   assert(result->is_illegal() ||
1117          (result->is_single_cpu() && result->as_register() == Z_R2) ||
1118          (result->is_double_cpu() && result->as_register_lo() == Z_R2) ||
1119          (result->is_single_fpu() && result->as_float_reg() == Z_F0) ||
1120          (result->is_double_fpu() && result->as_double_reg() == Z_F0), "convention");
1121 
1122   AddressLiteral pp(os::get_polling_page());
1123   __ load_const_optimized(Z_R1_scratch, pp);
1124 
1125   // Pop the frame before the safepoint code.
1126   int retPC_offset = initial_frame_size_in_bytes() + _z_abi16(return_pc);
1127   if (Displacement::is_validDisp(retPC_offset)) {
1128     __ z_lg(Z_R14, retPC_offset, Z_SP);
1129     __ add2reg(Z_SP, initial_frame_size_in_bytes());
1130   } else {
1131     __ add2reg(Z_SP, initial_frame_size_in_bytes());
1132     __ restore_return_pc();
1133   }
1134 
1135   // We need to mark the code position where the load from the safepoint
1136   // polling page was emitted as relocInfo::poll_return_type here.
1137   __ relocate(relocInfo::poll_return_type);
1138   __ load_from_polling_page(Z_R1_scratch);
1139 
1140   __ z_br(Z_R14); // Return to caller.
1141 }
1142 
1143 int LIR_Assembler::safepoint_poll(LIR_Opr tmp, CodeEmitInfo* info) {
1144   AddressLiteral pp(os::get_polling_page());
1145   __ load_const_optimized(tmp->as_register_lo(), pp);
1146   guarantee(info != NULL, "Shouldn't be NULL");
1147   add_debug_info_for_branch(info);
1148   int offset = __ offset();
1149   __ relocate(relocInfo::poll_type);
1150   __ load_from_polling_page(tmp->as_register_lo());
1151   return offset;
1152 }
1153 
1154 void LIR_Assembler::emit_static_call_stub() {
1155 
1156   // Stub is fixed up when the corresponding call is converted from calling
1157   // compiled code to calling interpreted code.
1158 
1159   address call_pc = __ pc();
1160   address stub = __ start_a_stub(call_stub_size);
1161   if (stub == NULL) {
1162     bailout("static call stub overflow");
1163     return;
1164   }
1165 
1166   int start = __ offset();
1167 
1168   __ relocate(static_stub_Relocation::spec(call_pc));
1169 
1170   // See also Matcher::interpreter_method_oop_reg().
1171   AddressLiteral meta = __ allocate_metadata_address(NULL);
1172   bool success = __ load_const_from_toc(Z_method, meta);
1173 
1174   __ set_inst_mark();
1175   AddressLiteral a((address)-1);
1176   success = success && __ load_const_from_toc(Z_R1, a);
1177   if (!success) {
1178     bailout("const section overflow");
1179     return;
1180   }
1181 
1182   __ z_br(Z_R1);
1183   assert(__ offset() - start <= call_stub_size, "stub too big");
1184   __ end_a_stub(); // Update current stubs pointer and restore insts_end.
1185 }
1186 
1187 void LIR_Assembler::comp_op(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Op2* op) {
1188   bool unsigned_comp = condition == lir_cond_belowEqual || condition == lir_cond_aboveEqual;
1189   if (opr1->is_single_cpu()) {
1190     Register reg1 = opr1->as_register();
1191     if (opr2->is_single_cpu()) {
1192       // cpu register - cpu register
1193       if (opr1->type() == T_OBJECT || opr1->type() == T_ARRAY) {
1194         __ z_clgr(reg1, opr2->as_register());
1195       } else {
1196         assert(opr2->type() != T_OBJECT && opr2->type() != T_ARRAY, "cmp int, oop?");
1197         if (unsigned_comp) {
1198           __ z_clr(reg1, opr2->as_register());
1199         } else {
1200           __ z_cr(reg1, opr2->as_register());
1201         }
1202       }
1203     } else if (opr2->is_stack()) {
1204       // cpu register - stack
1205       if (opr1->type() == T_OBJECT || opr1->type() == T_ARRAY) {
1206         __ z_cg(reg1, frame_map()->address_for_slot(opr2->single_stack_ix()));
1207       } else {
1208         if (unsigned_comp) {
1209           __ z_cly(reg1, frame_map()->address_for_slot(opr2->single_stack_ix()));
1210         } else {
1211           __ z_cy(reg1, frame_map()->address_for_slot(opr2->single_stack_ix()));
1212         }
1213       }
1214     } else if (opr2->is_constant()) {
1215       // cpu register - constant
1216       LIR_Const* c = opr2->as_constant_ptr();
1217       if (c->type() == T_INT) {
1218         if (unsigned_comp) {
1219           __ z_clfi(reg1, c->as_jint());
1220         } else {
1221           __ z_cfi(reg1, c->as_jint());
1222         }
1223       } else if (c->type() == T_OBJECT || c->type() == T_ARRAY) {
1224         // In 64bit oops are single register.
1225         jobject o = c->as_jobject();
1226         if (o == NULL) {
1227           __ z_ltgr(reg1, reg1);
1228         } else {
1229           jobject2reg(o, Z_R1_scratch);
1230           __ z_cgr(reg1, Z_R1_scratch);
1231         }
1232       } else {
1233         fatal("unexpected type: %s", basictype_to_str(c->type()));
1234       }
1235       // cpu register - address
1236     } else if (opr2->is_address()) {
1237       if (op->info() != NULL) {
1238         add_debug_info_for_null_check_here(op->info());
1239       }
1240       if (unsigned_comp) {
1241         __ z_cly(reg1, as_Address(opr2->as_address_ptr()));
1242       } else {
1243         __ z_cy(reg1, as_Address(opr2->as_address_ptr()));
1244       }
1245     } else {
1246       ShouldNotReachHere();
1247     }
1248 
1249   } else if (opr1->is_double_cpu()) {
1250     assert(!unsigned_comp, "unexpected");
1251     Register xlo = opr1->as_register_lo();
1252     Register xhi = opr1->as_register_hi();
1253     if (opr2->is_double_cpu()) {
1254       __ z_cgr(xlo, opr2->as_register_lo());
1255     } else if (opr2->is_constant()) {
1256       // cpu register - constant 0
1257       assert(opr2->as_jlong() == (jlong)0, "only handles zero");
1258       __ z_ltgr(xlo, xlo);
1259     } else {
1260       ShouldNotReachHere();
1261     }
1262 
1263   } else if (opr1->is_single_fpu()) {
1264     if (opr2->is_single_fpu()) {
1265       __ z_cebr(opr1->as_float_reg(), opr2->as_float_reg());
1266     } else {
1267       // stack slot
1268       Address addr = frame_map()->address_for_slot(opr2->single_stack_ix());
1269       if (Immediate::is_uimm12(addr.disp())) {
1270         __ z_ceb(opr1->as_float_reg(), addr);
1271       } else {
1272         __ z_ley(Z_fscratch_1, addr);
1273         __ z_cebr(opr1->as_float_reg(), Z_fscratch_1);
1274       }
1275     }
1276   } else if (opr1->is_double_fpu()) {
1277     if (opr2->is_double_fpu()) {
1278     __ z_cdbr(opr1->as_double_reg(), opr2->as_double_reg());
1279     } else {
1280       // stack slot
1281       Address addr = frame_map()->address_for_slot(opr2->double_stack_ix());
1282       if (Immediate::is_uimm12(addr.disp())) {
1283         __ z_cdb(opr1->as_double_reg(), addr);
1284       } else {
1285         __ z_ldy(Z_fscratch_1, addr);
1286         __ z_cdbr(opr1->as_double_reg(), Z_fscratch_1);
1287       }
1288     }
1289   } else {
1290     ShouldNotReachHere();
1291   }
1292 }
1293 
1294 void LIR_Assembler::comp_fl2i(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dst, LIR_Op2* op) {
1295   Label    done;
1296   Register dreg = dst->as_register();
1297 
1298   if (code == lir_cmp_fd2i || code == lir_ucmp_fd2i) {
1299     assert((left->is_single_fpu() && right->is_single_fpu()) ||
1300            (left->is_double_fpu() && right->is_double_fpu()), "unexpected operand types");
1301     bool is_single = left->is_single_fpu();
1302     bool is_unordered_less = (code == lir_ucmp_fd2i);
1303     FloatRegister lreg = is_single ? left->as_float_reg() : left->as_double_reg();
1304     FloatRegister rreg = is_single ? right->as_float_reg() : right->as_double_reg();
1305     if (is_single) {
1306       __ z_cebr(lreg, rreg);
1307     } else {
1308       __ z_cdbr(lreg, rreg);
1309     }
1310     if (VM_Version::has_LoadStoreConditional()) {
1311       Register one       = Z_R0_scratch;
1312       Register minus_one = Z_R1_scratch;
1313       __ z_lghi(minus_one, -1);
1314       __ z_lghi(one,  1);
1315       __ z_lghi(dreg, 0);
1316       __ z_locgr(dreg, one,       is_unordered_less ? Assembler::bcondHigh            : Assembler::bcondHighOrNotOrdered);
1317       __ z_locgr(dreg, minus_one, is_unordered_less ? Assembler::bcondLowOrNotOrdered : Assembler::bcondLow);
1318     } else {
1319       __ clear_reg(dreg, true, false);
1320       __ z_bre(done); // if (left == right) dst = 0
1321 
1322       // if (left > right || ((code ~= cmpg) && (left <> right)) dst := 1
1323       __ z_lhi(dreg, 1);
1324       __ z_brc(is_unordered_less ? Assembler::bcondHigh : Assembler::bcondHighOrNotOrdered, done);
1325 
1326       // if (left < right || ((code ~= cmpl) && (left <> right)) dst := -1
1327       __ z_lhi(dreg, -1);
1328     }
1329   } else {
1330     assert(code == lir_cmp_l2i, "check");
1331     if (VM_Version::has_LoadStoreConditional()) {
1332       Register one       = Z_R0_scratch;
1333       Register minus_one = Z_R1_scratch;
1334       __ z_cgr(left->as_register_lo(), right->as_register_lo());
1335       __ z_lghi(minus_one, -1);
1336       __ z_lghi(one,  1);
1337       __ z_lghi(dreg, 0);
1338       __ z_locgr(dreg, one, Assembler::bcondHigh);
1339       __ z_locgr(dreg, minus_one, Assembler::bcondLow);
1340     } else {
1341       __ z_cgr(left->as_register_lo(), right->as_register_lo());
1342       __ z_lghi(dreg,  0);     // eq value
1343       __ z_bre(done);
1344       __ z_lghi(dreg,  1);     // gt value
1345       __ z_brh(done);
1346       __ z_lghi(dreg, -1);     // lt value
1347     }
1348   }
1349   __ bind(done);
1350 }
1351 
1352 // result = condition ? opr1 : opr2
1353 void LIR_Assembler::cmove(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Opr result, BasicType type) {
1354   Assembler::branch_condition acond = Assembler::bcondEqual, ncond = Assembler::bcondNotEqual;
1355   switch (condition) {
1356     case lir_cond_equal:        acond = Assembler::bcondEqual;    ncond = Assembler::bcondNotEqual; break;
1357     case lir_cond_notEqual:     acond = Assembler::bcondNotEqual; ncond = Assembler::bcondEqual;    break;
1358     case lir_cond_less:         acond = Assembler::bcondLow;      ncond = Assembler::bcondNotLow;   break;
1359     case lir_cond_lessEqual:    acond = Assembler::bcondNotHigh;  ncond = Assembler::bcondHigh;     break;
1360     case lir_cond_greaterEqual: acond = Assembler::bcondNotLow;   ncond = Assembler::bcondLow;      break;
1361     case lir_cond_greater:      acond = Assembler::bcondHigh;     ncond = Assembler::bcondNotHigh;  break;
1362     case lir_cond_belowEqual:   acond = Assembler::bcondNotHigh;  ncond = Assembler::bcondHigh;     break;
1363     case lir_cond_aboveEqual:   acond = Assembler::bcondNotLow;   ncond = Assembler::bcondLow;      break;
1364     default:                    ShouldNotReachHere();
1365   }
1366 
1367   if (opr1->is_cpu_register()) {
1368     reg2reg(opr1, result);
1369   } else if (opr1->is_stack()) {
1370     stack2reg(opr1, result, result->type());
1371   } else if (opr1->is_constant()) {
1372     const2reg(opr1, result, lir_patch_none, NULL);
1373   } else {
1374     ShouldNotReachHere();
1375   }
1376 
1377   if (VM_Version::has_LoadStoreConditional() && !opr2->is_constant()) {
1378     // Optimized version that does not require a branch.
1379     if (opr2->is_single_cpu()) {
1380       assert(opr2->cpu_regnr() != result->cpu_regnr(), "opr2 already overwritten by previous move");
1381       __ z_locgr(result->as_register(), opr2->as_register(), ncond);
1382     } else if (opr2->is_double_cpu()) {
1383       assert(opr2->cpu_regnrLo() != result->cpu_regnrLo() && opr2->cpu_regnrLo() != result->cpu_regnrHi(), "opr2 already overwritten by previous move");
1384       assert(opr2->cpu_regnrHi() != result->cpu_regnrLo() && opr2->cpu_regnrHi() != result->cpu_regnrHi(), "opr2 already overwritten by previous move");
1385       __ z_locgr(result->as_register_lo(), opr2->as_register_lo(), ncond);
1386     } else if (opr2->is_single_stack()) {
1387       __ z_loc(result->as_register(), frame_map()->address_for_slot(opr2->single_stack_ix()), ncond);
1388     } else if (opr2->is_double_stack()) {
1389       __ z_locg(result->as_register_lo(), frame_map()->address_for_slot(opr2->double_stack_ix()), ncond);
1390     } else {
1391       ShouldNotReachHere();
1392     }
1393   } else {
1394     Label skip;
1395     __ z_brc(acond, skip);
1396     if (opr2->is_cpu_register()) {
1397       reg2reg(opr2, result);
1398     } else if (opr2->is_stack()) {
1399       stack2reg(opr2, result, result->type());
1400     } else if (opr2->is_constant()) {
1401       const2reg(opr2, result, lir_patch_none, NULL);
1402     } else {
1403       ShouldNotReachHere();
1404     }
1405     __ bind(skip);
1406   }
1407 }
1408 
1409 void LIR_Assembler::arith_op(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dest,
1410                              CodeEmitInfo* info, bool pop_fpu_stack) {
1411   assert(info == NULL, "should never be used, idiv/irem and ldiv/lrem not handled by this method");
1412 
1413   if (left->is_single_cpu()) {
1414     assert(left == dest, "left and dest must be equal");
1415     Register lreg = left->as_register();
1416 
1417     if (right->is_single_cpu()) {
1418       // cpu register - cpu register
1419       Register rreg = right->as_register();
1420       switch (code) {
1421         case lir_add: __ z_ar (lreg, rreg); break;
1422         case lir_sub: __ z_sr (lreg, rreg); break;
1423         case lir_mul: __ z_msr(lreg, rreg); break;
1424         default: ShouldNotReachHere();
1425       }
1426 
1427     } else if (right->is_stack()) {
1428       // cpu register - stack
1429       Address raddr = frame_map()->address_for_slot(right->single_stack_ix());
1430       switch (code) {
1431         case lir_add: __ z_ay(lreg, raddr); break;
1432         case lir_sub: __ z_sy(lreg, raddr); break;
1433         default: ShouldNotReachHere();
1434       }
1435 
1436     } else if (right->is_constant()) {
1437       // cpu register - constant
1438       jint c = right->as_constant_ptr()->as_jint();
1439       switch (code) {
1440         case lir_add: __ z_agfi(lreg, c);  break;
1441         case lir_sub: __ z_agfi(lreg, -c); break; // note: -min_jint == min_jint
1442         case lir_mul: __ z_msfi(lreg, c);  break;
1443         default: ShouldNotReachHere();
1444       }
1445 
1446     } else {
1447       ShouldNotReachHere();
1448     }
1449 
1450   } else if (left->is_double_cpu()) {
1451     assert(left == dest, "left and dest must be equal");
1452     Register lreg_lo = left->as_register_lo();
1453     Register lreg_hi = left->as_register_hi();
1454 
1455     if (right->is_double_cpu()) {
1456       // cpu register - cpu register
1457       Register rreg_lo = right->as_register_lo();
1458       Register rreg_hi = right->as_register_hi();
1459       assert_different_registers(lreg_lo, rreg_lo);
1460       switch (code) {
1461         case lir_add:
1462           __ z_agr(lreg_lo, rreg_lo);
1463           break;
1464         case lir_sub:
1465           __ z_sgr(lreg_lo, rreg_lo);
1466           break;
1467         case lir_mul:
1468           __ z_msgr(lreg_lo, rreg_lo);
1469           break;
1470         default:
1471           ShouldNotReachHere();
1472       }
1473 
1474     } else if (right->is_constant()) {
1475       // cpu register - constant
1476       jlong c = right->as_constant_ptr()->as_jlong_bits();
1477       switch (code) {
1478         case lir_add: __ z_agfi(lreg_lo, c); break;
1479         case lir_sub:
1480           if (c != min_jint) {
1481                       __ z_agfi(lreg_lo, -c);
1482           } else {
1483             // -min_jint cannot be represented as simm32 in z_agfi
1484             // min_jint sign extended:      0xffffffff80000000
1485             // -min_jint as 64 bit integer: 0x0000000080000000
1486             // 0x80000000 can be represented as uimm32 in z_algfi
1487             // lreg_lo := lreg_lo + -min_jint == lreg_lo + 0x80000000
1488                       __ z_algfi(lreg_lo, UCONST64(0x80000000));
1489           }
1490           break;
1491         case lir_mul: __ z_msgfi(lreg_lo, c); break;
1492         default:
1493           ShouldNotReachHere();
1494       }
1495 
1496     } else {
1497       ShouldNotReachHere();
1498     }
1499 
1500   } else if (left->is_single_fpu()) {
1501     assert(left == dest, "left and dest must be equal");
1502     FloatRegister lreg = left->as_float_reg();
1503     FloatRegister rreg = right->is_single_fpu() ? right->as_float_reg() : fnoreg;
1504     Address raddr;
1505 
1506     if (rreg == fnoreg) {
1507       assert(right->is_single_stack(), "constants should be loaded into register");
1508       raddr = frame_map()->address_for_slot(right->single_stack_ix());
1509       if (!Immediate::is_uimm12(raddr.disp())) {
1510         __ mem2freg_opt(rreg = Z_fscratch_1, raddr, false);
1511       }
1512     }
1513 
1514     if (rreg != fnoreg) {
1515       switch (code) {
1516         case lir_add: __ z_aebr(lreg, rreg);  break;
1517         case lir_sub: __ z_sebr(lreg, rreg);  break;
1518         case lir_mul_strictfp: // fall through
1519         case lir_mul: __ z_meebr(lreg, rreg); break;
1520         case lir_div_strictfp: // fall through
1521         case lir_div: __ z_debr(lreg, rreg);  break;
1522         default: ShouldNotReachHere();
1523       }
1524     } else {
1525       switch (code) {
1526         case lir_add: __ z_aeb(lreg, raddr);  break;
1527         case lir_sub: __ z_seb(lreg, raddr);  break;
1528         case lir_mul_strictfp: // fall through
1529         case lir_mul: __ z_meeb(lreg, raddr);  break;
1530         case lir_div_strictfp: // fall through
1531         case lir_div: __ z_deb(lreg, raddr);  break;
1532         default: ShouldNotReachHere();
1533       }
1534     }
1535   } else if (left->is_double_fpu()) {
1536     assert(left == dest, "left and dest must be equal");
1537     FloatRegister lreg = left->as_double_reg();
1538     FloatRegister rreg = right->is_double_fpu() ? right->as_double_reg() : fnoreg;
1539     Address raddr;
1540 
1541     if (rreg == fnoreg) {
1542       assert(right->is_double_stack(), "constants should be loaded into register");
1543       raddr = frame_map()->address_for_slot(right->double_stack_ix());
1544       if (!Immediate::is_uimm12(raddr.disp())) {
1545         __ mem2freg_opt(rreg = Z_fscratch_1, raddr, true);
1546       }
1547     }
1548 
1549     if (rreg != fnoreg) {
1550       switch (code) {
1551         case lir_add: __ z_adbr(lreg, rreg); break;
1552         case lir_sub: __ z_sdbr(lreg, rreg); break;
1553         case lir_mul_strictfp: // fall through
1554         case lir_mul: __ z_mdbr(lreg, rreg); break;
1555         case lir_div_strictfp: // fall through
1556         case lir_div: __ z_ddbr(lreg, rreg); break;
1557         default: ShouldNotReachHere();
1558       }
1559     } else {
1560       switch (code) {
1561         case lir_add: __ z_adb(lreg, raddr); break;
1562         case lir_sub: __ z_sdb(lreg, raddr); break;
1563         case lir_mul_strictfp: // fall through
1564         case lir_mul: __ z_mdb(lreg, raddr); break;
1565         case lir_div_strictfp: // fall through
1566         case lir_div: __ z_ddb(lreg, raddr); break;
1567         default: ShouldNotReachHere();
1568       }
1569     }
1570   } else if (left->is_address()) {
1571     assert(left == dest, "left and dest must be equal");
1572     assert(code == lir_add, "unsupported operation");
1573     assert(right->is_constant(), "unsupported operand");
1574     jint c = right->as_constant_ptr()->as_jint();
1575     LIR_Address* lir_addr = left->as_address_ptr();
1576     Address addr = as_Address(lir_addr);
1577     switch (lir_addr->type()) {
1578       case T_INT:
1579         __ add2mem_32(addr, c, Z_R1_scratch);
1580         break;
1581       case T_LONG:
1582         __ add2mem_64(addr, c, Z_R1_scratch);
1583         break;
1584       default:
1585         ShouldNotReachHere();
1586     }
1587   } else {
1588     ShouldNotReachHere();
1589   }
1590 }
1591 
1592 void LIR_Assembler::fpop() {
1593   // do nothing
1594 }
1595 
1596 void LIR_Assembler::intrinsic_op(LIR_Code code, LIR_Opr value, LIR_Opr thread, LIR_Opr dest, LIR_Op* op) {
1597   switch (code) {
1598     case lir_sqrt: {
1599       assert(!thread->is_valid(), "there is no need for a thread_reg for dsqrt");
1600       FloatRegister src_reg = value->as_double_reg();
1601       FloatRegister dst_reg = dest->as_double_reg();
1602       __ z_sqdbr(dst_reg, src_reg);
1603       break;
1604     }
1605     case lir_abs: {
1606       assert(!thread->is_valid(), "there is no need for a thread_reg for fabs");
1607       FloatRegister src_reg = value->as_double_reg();
1608       FloatRegister dst_reg = dest->as_double_reg();
1609       __ z_lpdbr(dst_reg, src_reg);
1610       break;
1611     }
1612     default: {
1613       ShouldNotReachHere();
1614       break;
1615     }
1616   }
1617 }
1618 
1619 void LIR_Assembler::logic_op(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dst) {
1620   if (left->is_single_cpu()) {
1621     Register reg = left->as_register();
1622     if (right->is_constant()) {
1623       int val = right->as_constant_ptr()->as_jint();
1624       switch (code) {
1625         case lir_logic_and: __ z_nilf(reg, val); break;
1626         case lir_logic_or:  __ z_oilf(reg, val); break;
1627         case lir_logic_xor: __ z_xilf(reg, val); break;
1628         default: ShouldNotReachHere();
1629       }
1630     } else if (right->is_stack()) {
1631       Address raddr = frame_map()->address_for_slot(right->single_stack_ix());
1632       switch (code) {
1633         case lir_logic_and: __ z_ny(reg, raddr); break;
1634         case lir_logic_or:  __ z_oy(reg, raddr); break;
1635         case lir_logic_xor: __ z_xy(reg, raddr); break;
1636         default: ShouldNotReachHere();
1637       }
1638     } else {
1639       Register rright = right->as_register();
1640       switch (code) {
1641         case lir_logic_and: __ z_nr(reg, rright); break;
1642         case lir_logic_or : __ z_or(reg, rright); break;
1643         case lir_logic_xor: __ z_xr(reg, rright); break;
1644         default: ShouldNotReachHere();
1645       }
1646     }
1647     move_regs(reg, dst->as_register());
1648   } else {
1649     Register l_lo = left->as_register_lo();
1650     if (right->is_constant()) {
1651       __ load_const_optimized(Z_R1_scratch, right->as_constant_ptr()->as_jlong());
1652       switch (code) {
1653         case lir_logic_and:
1654           __ z_ngr(l_lo, Z_R1_scratch);
1655           break;
1656         case lir_logic_or:
1657           __ z_ogr(l_lo, Z_R1_scratch);
1658           break;
1659         case lir_logic_xor:
1660           __ z_xgr(l_lo, Z_R1_scratch);
1661           break;
1662         default: ShouldNotReachHere();
1663       }
1664     } else {
1665       Register r_lo;
1666       if (right->type() == T_OBJECT || right->type() == T_ARRAY) {
1667         r_lo = right->as_register();
1668       } else {
1669         r_lo = right->as_register_lo();
1670       }
1671       switch (code) {
1672         case lir_logic_and:
1673           __ z_ngr(l_lo, r_lo);
1674           break;
1675         case lir_logic_or:
1676           __ z_ogr(l_lo, r_lo);
1677           break;
1678         case lir_logic_xor:
1679           __ z_xgr(l_lo, r_lo);
1680           break;
1681         default: ShouldNotReachHere();
1682       }
1683     }
1684 
1685     Register dst_lo = dst->as_register_lo();
1686 
1687     move_regs(l_lo, dst_lo);
1688   }
1689 }
1690 
1691 // See operand selection in LIRGenerator::do_ArithmeticOp_Int().
1692 void LIR_Assembler::arithmetic_idiv(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr temp, LIR_Opr result, CodeEmitInfo* info) {
1693   if (left->is_double_cpu()) {
1694     // 64 bit integer case
1695     assert(left->is_double_cpu(), "left must be register");
1696     assert(right->is_double_cpu() || is_power_of_2_long(right->as_jlong()),
1697            "right must be register or power of 2 constant");
1698     assert(result->is_double_cpu(), "result must be register");
1699 
1700     Register lreg = left->as_register_lo();
1701     Register dreg = result->as_register_lo();
1702 
1703     if (right->is_constant()) {
1704       // Convert division by a power of two into some shifts and logical operations.
1705       Register treg1 = Z_R0_scratch;
1706       Register treg2 = Z_R1_scratch;
1707       jlong divisor = right->as_jlong();
1708       jlong log_divisor = log2_long(right->as_jlong());
1709 
1710       if (divisor == min_jlong) {
1711         // Min_jlong is special. Result is '0' except for min_jlong/min_jlong = 1.
1712         if (dreg == lreg) {
1713           NearLabel done;
1714           __ load_const_optimized(treg2, min_jlong);
1715           __ z_cgr(lreg, treg2);
1716           __ z_lghi(dreg, 0);           // Preserves condition code.
1717           __ z_brne(done);
1718           __ z_lghi(dreg, 1);           // min_jlong / min_jlong = 1
1719           __ bind(done);
1720         } else {
1721           assert_different_registers(dreg, lreg);
1722           NearLabel done;
1723           __ z_lghi(dreg, 0);
1724           __ compare64_and_branch(lreg, min_jlong, Assembler::bcondNotEqual, done);
1725           __ z_lghi(dreg, 1);
1726           __ bind(done);
1727         }
1728         return;
1729       }
1730       __ move_reg_if_needed(dreg, T_LONG, lreg, T_LONG);
1731       if (divisor == 2) {
1732         __ z_srlg(treg2, dreg, 63);     // dividend < 0 ? 1 : 0
1733       } else {
1734         __ z_srag(treg2, dreg, 63);     // dividend < 0 ? -1 : 0
1735         __ and_imm(treg2, divisor - 1, treg1, true);
1736       }
1737       if (code == lir_idiv) {
1738         __ z_agr(dreg, treg2);
1739         __ z_srag(dreg, dreg, log_divisor);
1740       } else {
1741         assert(code == lir_irem, "check");
1742         __ z_agr(treg2, dreg);
1743         __ and_imm(treg2, ~(divisor - 1), treg1, true);
1744         __ z_sgr(dreg, treg2);
1745       }
1746       return;
1747     }
1748 
1749     // Divisor is not a power of 2 constant.
1750     Register rreg = right->as_register_lo();
1751     Register treg = temp->as_register_lo();
1752     assert(right->is_double_cpu(), "right must be register");
1753     assert(lreg == Z_R11, "see ldivInOpr()");
1754     assert(rreg != lreg, "right register must not be same as left register");
1755     assert((code == lir_idiv && dreg == Z_R11 && treg == Z_R10) ||
1756            (code == lir_irem && dreg == Z_R10 && treg == Z_R11), "see ldivInOpr(), ldivOutOpr(), lremOutOpr()");
1757 
1758     Register R1 = lreg->predecessor();
1759     Register R2 = rreg;
1760     assert(code != lir_idiv || lreg==dreg, "see code below");
1761     if (code == lir_idiv) {
1762       __ z_lcgr(lreg, lreg);
1763     } else {
1764       __ clear_reg(dreg, true, false);
1765     }
1766     NearLabel done;
1767     __ compare64_and_branch(R2, -1, Assembler::bcondEqual, done);
1768     if (code == lir_idiv) {
1769       __ z_lcgr(lreg, lreg); // Revert lcgr above.
1770     }
1771     if (ImplicitDiv0Checks) {
1772       // No debug info because the idiv won't trap.
1773       // Add_debug_info_for_div0 would instantiate another DivByZeroStub,
1774       // which is unnecessary, too.
1775       add_debug_info_for_div0(__ offset(), info);
1776     }
1777     __ z_dsgr(R1, R2);
1778     __ bind(done);
1779     return;
1780   }
1781 
1782   // 32 bit integer case
1783 
1784   assert(left->is_single_cpu(), "left must be register");
1785   assert(right->is_single_cpu() || is_power_of_2(right->as_jint()), "right must be register or power of 2 constant");
1786   assert(result->is_single_cpu(), "result must be register");
1787 
1788   Register lreg = left->as_register();
1789   Register dreg = result->as_register();
1790 
1791   if (right->is_constant()) {
1792     // Convert division by a power of two into some shifts and logical operations.
1793     Register treg1 = Z_R0_scratch;
1794     Register treg2 = Z_R1_scratch;
1795     jlong divisor = right->as_jint();
1796     jlong log_divisor = log2_long(right->as_jint());
1797     __ move_reg_if_needed(dreg, T_LONG, lreg, T_INT); // sign extend
1798     if (divisor == 2) {
1799       __ z_srlg(treg2, dreg, 63);     // dividend < 0 ?  1 : 0
1800     } else {
1801       __ z_srag(treg2, dreg, 63);     // dividend < 0 ? -1 : 0
1802       __ and_imm(treg2, divisor - 1, treg1, true);
1803     }
1804     if (code == lir_idiv) {
1805       __ z_agr(dreg, treg2);
1806       __ z_srag(dreg, dreg, log_divisor);
1807     } else {
1808       assert(code == lir_irem, "check");
1809       __ z_agr(treg2, dreg);
1810       __ and_imm(treg2, ~(divisor - 1), treg1, true);
1811       __ z_sgr(dreg, treg2);
1812     }
1813     return;
1814   }
1815 
1816   // Divisor is not a power of 2 constant.
1817   Register rreg = right->as_register();
1818   Register treg = temp->as_register();
1819   assert(right->is_single_cpu(), "right must be register");
1820   assert(lreg == Z_R11, "left register must be rax,");
1821   assert(rreg != lreg, "right register must not be same as left register");
1822   assert((code == lir_idiv && dreg == Z_R11 && treg == Z_R10)
1823       || (code == lir_irem && dreg == Z_R10 && treg == Z_R11), "see divInOpr(), divOutOpr(), remOutOpr()");
1824 
1825   Register R1 = lreg->predecessor();
1826   Register R2 = rreg;
1827   __ move_reg_if_needed(lreg, T_LONG, lreg, T_INT); // sign extend
1828   if (ImplicitDiv0Checks) {
1829     // No debug info because the idiv won't trap.
1830     // Add_debug_info_for_div0 would instantiate another DivByZeroStub,
1831     // which is unnecessary, too.
1832     add_debug_info_for_div0(__ offset(), info);
1833   }
1834   __ z_dsgfr(R1, R2);
1835 }
1836 
1837 void LIR_Assembler::throw_op(LIR_Opr exceptionPC, LIR_Opr exceptionOop, CodeEmitInfo* info) {
1838   assert(exceptionOop->as_register() == Z_EXC_OOP, "should match");
1839   assert(exceptionPC->as_register() == Z_EXC_PC, "should match");
1840 
1841   // Exception object is not added to oop map by LinearScan
1842   // (LinearScan assumes that no oops are in fixed registers).
1843   info->add_register_oop(exceptionOop);
1844 
1845   // Reuse the debug info from the safepoint poll for the throw op itself.
1846   __ get_PC(Z_EXC_PC);
1847   add_call_info(__ offset(), info); // for exception handler
1848   address stub = Runtime1::entry_for (compilation()->has_fpu_code() ? Runtime1::handle_exception_id
1849                                                                     : Runtime1::handle_exception_nofpu_id);
1850   emit_call_c(stub);
1851 }
1852 
1853 void LIR_Assembler::unwind_op(LIR_Opr exceptionOop) {
1854   assert(exceptionOop->as_register() == Z_EXC_OOP, "should match");
1855 
1856   __ branch_optimized(Assembler::bcondAlways, _unwind_handler_entry);
1857 }
1858 
1859 void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) {
1860   ciArrayKlass* default_type = op->expected_type();
1861   Register src = op->src()->as_register();
1862   Register dst = op->dst()->as_register();
1863   Register src_pos = op->src_pos()->as_register();
1864   Register dst_pos = op->dst_pos()->as_register();
1865   Register length  = op->length()->as_register();
1866   Register tmp = op->tmp()->as_register();
1867 
1868   CodeStub* stub = op->stub();
1869   int flags = op->flags();
1870   BasicType basic_type = default_type != NULL ? default_type->element_type()->basic_type() : T_ILLEGAL;
1871   if (basic_type == T_ARRAY) basic_type = T_OBJECT;
1872 
1873   // If we don't know anything, just go through the generic arraycopy.
1874   if (default_type == NULL) {
1875     Label done;
1876     // Save outgoing arguments in callee saved registers (C convention) in case
1877     // a call to System.arraycopy is needed.
1878     Register callee_saved_src     = Z_R10;
1879     Register callee_saved_src_pos = Z_R11;
1880     Register callee_saved_dst     = Z_R12;
1881     Register callee_saved_dst_pos = Z_R13;
1882     Register callee_saved_length  = Z_ARG5; // Z_ARG5 == Z_R6 is callee saved.
1883 
1884     __ lgr_if_needed(callee_saved_src, src);
1885     __ lgr_if_needed(callee_saved_src_pos, src_pos);
1886     __ lgr_if_needed(callee_saved_dst, dst);
1887     __ lgr_if_needed(callee_saved_dst_pos, dst_pos);
1888     __ lgr_if_needed(callee_saved_length, length);
1889 
1890     // C function requires 64 bit values.
1891     __ z_lgfr(src_pos, src_pos);
1892     __ z_lgfr(dst_pos, dst_pos);
1893     __ z_lgfr(length, length);
1894 
1895     address C_entry = CAST_FROM_FN_PTR(address, Runtime1::arraycopy);
1896 
1897     address copyfunc_addr = StubRoutines::generic_arraycopy();
1898 
1899     // Pass arguments: may push as this is not a safepoint; SP must be fix at each safepoint.
1900 
1901     // The arguments are in the corresponding registers.
1902     assert(Z_ARG1 == src,     "assumption");
1903     assert(Z_ARG2 == src_pos, "assumption");
1904     assert(Z_ARG3 == dst,     "assumption");
1905     assert(Z_ARG4 == dst_pos, "assumption");
1906     assert(Z_ARG5 == length,  "assumption");
1907     if (copyfunc_addr == NULL) { // Use C version if stub was not generated.
1908       emit_call_c(C_entry);
1909     } else {
1910 #ifndef PRODUCT
1911       if (PrintC1Statistics) {
1912         __ load_const_optimized(Z_R1_scratch, (address)&Runtime1::_generic_arraycopystub_cnt);
1913         __ add2mem_32(Address(Z_R1_scratch), 1, Z_R0_scratch);
1914       }
1915 #endif
1916       emit_call_c(copyfunc_addr);
1917     }
1918     CHECK_BAILOUT();
1919 
1920     __ compare32_and_branch(Z_RET, (intptr_t)0, Assembler::bcondEqual, *stub->continuation());
1921 
1922     if (copyfunc_addr != NULL) {
1923       __ z_lgr(tmp, Z_RET);
1924       __ z_xilf(tmp, -1);
1925     }
1926 
1927     // Restore values from callee saved registers so they are where the stub
1928     // expects them.
1929     __ lgr_if_needed(src, callee_saved_src);
1930     __ lgr_if_needed(src_pos, callee_saved_src_pos);
1931     __ lgr_if_needed(dst, callee_saved_dst);
1932     __ lgr_if_needed(dst_pos, callee_saved_dst_pos);
1933     __ lgr_if_needed(length, callee_saved_length);
1934 
1935     if (copyfunc_addr != NULL) {
1936       __ z_sr(length, tmp);
1937       __ z_ar(src_pos, tmp);
1938       __ z_ar(dst_pos, tmp);
1939     }
1940     __ branch_optimized(Assembler::bcondAlways, *stub->entry());
1941 
1942     __ bind(*stub->continuation());
1943     return;
1944   }
1945 
1946   assert(default_type != NULL && default_type->is_array_klass() && default_type->is_loaded(), "must be true at this point");
1947 
1948   int elem_size = type2aelembytes(basic_type);
1949   int shift_amount;
1950 
1951   switch (elem_size) {
1952     case 1 :
1953       shift_amount = 0;
1954       break;
1955     case 2 :
1956       shift_amount = 1;
1957       break;
1958     case 4 :
1959       shift_amount = 2;
1960       break;
1961     case 8 :
1962       shift_amount = 3;
1963       break;
1964     default:
1965       shift_amount = -1;
1966       ShouldNotReachHere();
1967   }
1968 
1969   Address src_length_addr = Address(src, arrayOopDesc::length_offset_in_bytes());
1970   Address dst_length_addr = Address(dst, arrayOopDesc::length_offset_in_bytes());
1971   Address src_klass_addr = Address(src, oopDesc::klass_offset_in_bytes());
1972   Address dst_klass_addr = Address(dst, oopDesc::klass_offset_in_bytes());
1973 
1974   // Length and pos's are all sign extended at this point on 64bit.
1975 
1976   // test for NULL
1977   if (flags & LIR_OpArrayCopy::src_null_check) {
1978     __ compareU64_and_branch(src, (intptr_t)0, Assembler::bcondZero, *stub->entry());
1979   }
1980   if (flags & LIR_OpArrayCopy::dst_null_check) {
1981     __ compareU64_and_branch(dst, (intptr_t)0, Assembler::bcondZero, *stub->entry());
1982   }
1983 
1984   // Check if negative.
1985   if (flags & LIR_OpArrayCopy::src_pos_positive_check) {
1986     __ compare32_and_branch(src_pos, (intptr_t)0, Assembler::bcondLow, *stub->entry());
1987   }
1988   if (flags & LIR_OpArrayCopy::dst_pos_positive_check) {
1989     __ compare32_and_branch(dst_pos, (intptr_t)0, Assembler::bcondLow, *stub->entry());
1990   }
1991 
1992   // If the compiler was not able to prove that exact type of the source or the destination
1993   // of the arraycopy is an array type, check at runtime if the source or the destination is
1994   // an instance type.
1995   if (flags & LIR_OpArrayCopy::type_check) {
1996     assert(Klass::_lh_neutral_value == 0, "or replace z_lt instructions");
1997 
1998     if (!(flags & LIR_OpArrayCopy::dst_objarray)) {
1999       __ load_klass(tmp, dst);
2000       __ z_lt(tmp, Address(tmp, in_bytes(Klass::layout_helper_offset())));
2001       __ branch_optimized(Assembler::bcondNotLow, *stub->entry());
2002     }
2003 
2004     if (!(flags & LIR_OpArrayCopy::src_objarray)) {
2005       __ load_klass(tmp, src);
2006       __ z_lt(tmp, Address(tmp, in_bytes(Klass::layout_helper_offset())));
2007       __ branch_optimized(Assembler::bcondNotLow, *stub->entry());
2008     }
2009   }
2010 
2011   if (flags & LIR_OpArrayCopy::src_range_check) {
2012     __ z_la(tmp, Address(src_pos, length));
2013     __ z_cl(tmp, src_length_addr);
2014     __ branch_optimized(Assembler::bcondHigh, *stub->entry());
2015   }
2016   if (flags & LIR_OpArrayCopy::dst_range_check) {
2017     __ z_la(tmp, Address(dst_pos, length));
2018     __ z_cl(tmp, dst_length_addr);
2019     __ branch_optimized(Assembler::bcondHigh, *stub->entry());
2020   }
2021 
2022   if (flags & LIR_OpArrayCopy::length_positive_check) {
2023     __ z_ltr(length, length);
2024     __ branch_optimized(Assembler::bcondNegative, *stub->entry());
2025   }
2026 
2027   // Stubs require 64 bit values.
2028   __ z_lgfr(src_pos, src_pos); // int -> long
2029   __ z_lgfr(dst_pos, dst_pos); // int -> long
2030   __ z_lgfr(length, length);   // int -> long
2031 
2032   if (flags & LIR_OpArrayCopy::type_check) {
2033     // We don't know the array types are compatible.
2034     if (basic_type != T_OBJECT) {
2035       // Simple test for basic type arrays.
2036       if (UseCompressedClassPointers) {
2037         __ z_l(tmp, src_klass_addr);
2038         __ z_c(tmp, dst_klass_addr);
2039       } else {
2040         __ z_lg(tmp, src_klass_addr);
2041         __ z_cg(tmp, dst_klass_addr);
2042       }
2043       __ branch_optimized(Assembler::bcondNotEqual, *stub->entry());
2044     } else {
2045       // For object arrays, if src is a sub class of dst then we can
2046       // safely do the copy.
2047       NearLabel cont, slow;
2048       Register src_klass = Z_R1_scratch;
2049       Register dst_klass = Z_R10;
2050 
2051       __ load_klass(src_klass, src);
2052       __ load_klass(dst_klass, dst);
2053 
2054       __ check_klass_subtype_fast_path(src_klass, dst_klass, tmp, &cont, &slow, NULL);
2055 
2056       store_parameter(src_klass, 0); // sub
2057       store_parameter(dst_klass, 1); // super
2058       emit_call_c(Runtime1::entry_for (Runtime1::slow_subtype_check_id));
2059       CHECK_BAILOUT();
2060       // Sets condition code 0 for match (2 otherwise).
2061       __ branch_optimized(Assembler::bcondEqual, cont);
2062 
2063       __ bind(slow);
2064 
2065       address copyfunc_addr = StubRoutines::checkcast_arraycopy();
2066       if (copyfunc_addr != NULL) { // use stub if available
2067         // Src is not a sub class of dst so we have to do a
2068         // per-element check.
2069 
2070         int mask = LIR_OpArrayCopy::src_objarray|LIR_OpArrayCopy::dst_objarray;
2071         if ((flags & mask) != mask) {
2072           // Check that at least both of them object arrays.
2073           assert(flags & mask, "one of the two should be known to be an object array");
2074 
2075           if (!(flags & LIR_OpArrayCopy::src_objarray)) {
2076             __ load_klass(tmp, src);
2077           } else if (!(flags & LIR_OpArrayCopy::dst_objarray)) {
2078             __ load_klass(tmp, dst);
2079           }
2080           Address klass_lh_addr(tmp, Klass::layout_helper_offset());
2081           jint objArray_lh = Klass::array_layout_helper(T_OBJECT);
2082           __ load_const_optimized(Z_R1_scratch, objArray_lh);
2083           __ z_c(Z_R1_scratch, klass_lh_addr);
2084           __ branch_optimized(Assembler::bcondNotEqual, *stub->entry());
2085         }
2086 
2087         // Save outgoing arguments in callee saved registers (C convention) in case
2088         // a call to System.arraycopy is needed.
2089         Register callee_saved_src     = Z_R10;
2090         Register callee_saved_src_pos = Z_R11;
2091         Register callee_saved_dst     = Z_R12;
2092         Register callee_saved_dst_pos = Z_R13;
2093         Register callee_saved_length  = Z_ARG5; // Z_ARG5 == Z_R6 is callee saved.
2094 
2095         __ lgr_if_needed(callee_saved_src, src);
2096         __ lgr_if_needed(callee_saved_src_pos, src_pos);
2097         __ lgr_if_needed(callee_saved_dst, dst);
2098         __ lgr_if_needed(callee_saved_dst_pos, dst_pos);
2099         __ lgr_if_needed(callee_saved_length, length);
2100 
2101         __ z_llgfr(length, length); // Higher 32bits must be null.
2102 
2103         __ z_sllg(Z_ARG1, src_pos, shift_amount); // index -> byte offset
2104         __ z_sllg(Z_ARG2, dst_pos, shift_amount); // index -> byte offset
2105 
2106         __ z_la(Z_ARG1, Address(src, Z_ARG1, arrayOopDesc::base_offset_in_bytes(basic_type)));
2107         assert_different_registers(Z_ARG1, dst, dst_pos, length);
2108         __ z_la(Z_ARG2, Address(dst, Z_ARG2, arrayOopDesc::base_offset_in_bytes(basic_type)));
2109         assert_different_registers(Z_ARG2, dst, length);
2110 
2111         __ z_lgr(Z_ARG3, length);
2112         assert_different_registers(Z_ARG3, dst);
2113 
2114         __ load_klass(Z_ARG5, dst);
2115         __ z_lg(Z_ARG5, Address(Z_ARG5, ObjArrayKlass::element_klass_offset()));
2116         __ z_lg(Z_ARG4, Address(Z_ARG5, Klass::super_check_offset_offset()));
2117         emit_call_c(copyfunc_addr);
2118         CHECK_BAILOUT();
2119 
2120 #ifndef PRODUCT
2121         if (PrintC1Statistics) {
2122           NearLabel failed;
2123           __ compareU32_and_branch(Z_RET, (intptr_t)0, Assembler::bcondNotEqual, failed);
2124           __ load_const_optimized(Z_R1_scratch, (address)&Runtime1::_arraycopy_checkcast_cnt);
2125           __ add2mem_32(Address(Z_R1_scratch), 1, Z_R0_scratch);
2126           __ bind(failed);
2127         }
2128 #endif
2129 
2130         __ compareU32_and_branch(Z_RET, (intptr_t)0, Assembler::bcondEqual, *stub->continuation());
2131 
2132 #ifndef PRODUCT
2133         if (PrintC1Statistics) {
2134           __ load_const_optimized(Z_R1_scratch, (address)&Runtime1::_arraycopy_checkcast_attempt_cnt);
2135           __ add2mem_32(Address(Z_R1_scratch), 1, Z_R0_scratch);
2136         }
2137 #endif
2138 
2139         __ z_lgr(tmp, Z_RET);
2140         __ z_xilf(tmp, -1);
2141 
2142         // Restore previously spilled arguments
2143         __ lgr_if_needed(src, callee_saved_src);
2144         __ lgr_if_needed(src_pos, callee_saved_src_pos);
2145         __ lgr_if_needed(dst, callee_saved_dst);
2146         __ lgr_if_needed(dst_pos, callee_saved_dst_pos);
2147         __ lgr_if_needed(length, callee_saved_length);
2148 
2149         __ z_sr(length, tmp);
2150         __ z_ar(src_pos, tmp);
2151         __ z_ar(dst_pos, tmp);
2152       }
2153 
2154       __ branch_optimized(Assembler::bcondAlways, *stub->entry());
2155 
2156       __ bind(cont);
2157     }
2158   }
2159 
2160 #ifdef ASSERT
2161   if (basic_type != T_OBJECT || !(flags & LIR_OpArrayCopy::type_check)) {
2162     // Sanity check the known type with the incoming class. For the
2163     // primitive case the types must match exactly with src.klass and
2164     // dst.klass each exactly matching the default type. For the
2165     // object array case, if no type check is needed then either the
2166     // dst type is exactly the expected type and the src type is a
2167     // subtype which we can't check or src is the same array as dst
2168     // but not necessarily exactly of type default_type.
2169     NearLabel known_ok, halt;
2170     metadata2reg(default_type->constant_encoding(), tmp);
2171     if (UseCompressedClassPointers) {
2172       __ encode_klass_not_null(tmp);
2173     }
2174 
2175     if (basic_type != T_OBJECT) {
2176       if (UseCompressedClassPointers)         { __ z_c (tmp, dst_klass_addr); }
2177       else                                    { __ z_cg(tmp, dst_klass_addr); }
2178       __ branch_optimized(Assembler::bcondNotEqual, halt);
2179       if (UseCompressedClassPointers)         { __ z_c (tmp, src_klass_addr); }
2180       else                                    { __ z_cg(tmp, src_klass_addr); }
2181       __ branch_optimized(Assembler::bcondEqual, known_ok);
2182     } else {
2183       if (UseCompressedClassPointers)         { __ z_c (tmp, dst_klass_addr); }
2184       else                                    { __ z_cg(tmp, dst_klass_addr); }
2185       __ branch_optimized(Assembler::bcondEqual, known_ok);
2186       __ compareU64_and_branch(src, dst, Assembler::bcondEqual, known_ok);
2187     }
2188     __ bind(halt);
2189     __ stop("incorrect type information in arraycopy");
2190     __ bind(known_ok);
2191   }
2192 #endif
2193 
2194 #ifndef PRODUCT
2195   if (PrintC1Statistics) {
2196     __ load_const_optimized(Z_R1_scratch, Runtime1::arraycopy_count_address(basic_type));
2197     __ add2mem_32(Address(Z_R1_scratch), 1, Z_R0_scratch);
2198   }
2199 #endif
2200 
2201   __ z_sllg(tmp, src_pos, shift_amount); // index -> byte offset
2202   __ z_sllg(Z_R1_scratch, dst_pos, shift_amount); // index -> byte offset
2203 
2204   assert_different_registers(Z_ARG1, dst, dst_pos, length);
2205   __ z_la(Z_ARG1, Address(src, tmp, arrayOopDesc::base_offset_in_bytes(basic_type)));
2206   assert_different_registers(Z_ARG2, length);
2207   __ z_la(Z_ARG2, Address(dst, Z_R1_scratch, arrayOopDesc::base_offset_in_bytes(basic_type)));
2208   __ lgr_if_needed(Z_ARG3, length);
2209 
2210   bool disjoint = (flags & LIR_OpArrayCopy::overlapping) == 0;
2211   bool aligned = (flags & LIR_OpArrayCopy::unaligned) == 0;
2212   const char *name;
2213   address entry = StubRoutines::select_arraycopy_function(basic_type, aligned, disjoint, name, false);
2214   __ call_VM_leaf(entry);
2215 
2216   __ bind(*stub->continuation());
2217 }
2218 
2219 void LIR_Assembler::shift_op(LIR_Code code, LIR_Opr left, LIR_Opr count, LIR_Opr dest, LIR_Opr tmp) {
2220   if (dest->is_single_cpu()) {
2221     if (left->type() == T_OBJECT) {
2222       switch (code) {
2223         case lir_shl:  __ z_sllg (dest->as_register(), left->as_register(), 0, count->as_register()); break;
2224         case lir_shr:  __ z_srag (dest->as_register(), left->as_register(), 0, count->as_register()); break;
2225         case lir_ushr: __ z_srlg (dest->as_register(), left->as_register(), 0, count->as_register()); break;
2226         default: ShouldNotReachHere();
2227       }
2228     } else {
2229       assert(code == lir_shl || left == dest, "left and dest must be equal for 2 operand form right shifts");
2230       Register masked_count = Z_R1_scratch;
2231       __ z_lr(masked_count, count->as_register());
2232       __ z_nill(masked_count, 31);
2233       switch (code) {
2234         case lir_shl:  __ z_sllg (dest->as_register(), left->as_register(), 0, masked_count); break;
2235         case lir_shr:  __ z_sra  (dest->as_register(), 0, masked_count); break;
2236         case lir_ushr: __ z_srl  (dest->as_register(), 0, masked_count); break;
2237         default: ShouldNotReachHere();
2238       }
2239     }
2240   } else {
2241     switch (code) {
2242       case lir_shl:  __ z_sllg (dest->as_register_lo(), left->as_register_lo(), 0, count->as_register()); break;
2243       case lir_shr:  __ z_srag (dest->as_register_lo(), left->as_register_lo(), 0, count->as_register()); break;
2244       case lir_ushr: __ z_srlg (dest->as_register_lo(), left->as_register_lo(), 0, count->as_register()); break;
2245       default: ShouldNotReachHere();
2246     }
2247   }
2248 }
2249 
2250 void LIR_Assembler::shift_op(LIR_Code code, LIR_Opr left, jint count, LIR_Opr dest) {
2251   if (left->type() == T_OBJECT) {
2252     count = count & 63;  // Shouldn't shift by more than sizeof(intptr_t).
2253     Register l = left->as_register();
2254     Register d = dest->as_register_lo();
2255     switch (code) {
2256       case lir_shl:  __ z_sllg (d, l, count); break;
2257       case lir_shr:  __ z_srag (d, l, count); break;
2258       case lir_ushr: __ z_srlg (d, l, count); break;
2259       default: ShouldNotReachHere();
2260     }
2261     return;
2262   }
2263   if (dest->is_single_cpu()) {
2264     assert(code == lir_shl || left == dest, "left and dest must be equal for 2 operand form right shifts");
2265     count = count & 0x1F; // Java spec
2266     switch (code) {
2267       case lir_shl:  __ z_sllg (dest->as_register(), left->as_register(), count); break;
2268       case lir_shr:  __ z_sra  (dest->as_register(), count); break;
2269       case lir_ushr: __ z_srl  (dest->as_register(), count); break;
2270       default: ShouldNotReachHere();
2271     }
2272   } else if (dest->is_double_cpu()) {
2273     count = count & 63; // Java spec
2274     Register l = left->as_pointer_register();
2275     Register d = dest->as_pointer_register();
2276     switch (code) {
2277       case lir_shl:  __ z_sllg (d, l, count); break;
2278       case lir_shr:  __ z_srag (d, l, count); break;
2279       case lir_ushr: __ z_srlg (d, l, count); break;
2280       default: ShouldNotReachHere();
2281     }
2282   } else {
2283     ShouldNotReachHere();
2284   }
2285 }
2286 
2287 void LIR_Assembler::emit_alloc_obj(LIR_OpAllocObj* op) {
2288   if (op->init_check()) {
2289     // Make sure klass is initialized & doesn't have finalizer.
2290     const int state_offset = in_bytes(InstanceKlass::init_state_offset());
2291     Register iklass = op->klass()->as_register();
2292     add_debug_info_for_null_check_here(op->stub()->info());
2293     if (Immediate::is_uimm12(state_offset)) {
2294       __ z_cli(state_offset, iklass, InstanceKlass::fully_initialized);
2295     } else {
2296       __ z_cliy(state_offset, iklass, InstanceKlass::fully_initialized);
2297     }
2298     __ branch_optimized(Assembler::bcondNotEqual, *op->stub()->entry()); // Use long branch, because slow_case might be far.
2299   }
2300   __ allocate_object(op->obj()->as_register(),
2301                      op->tmp1()->as_register(),
2302                      op->tmp2()->as_register(),
2303                      op->header_size(),
2304                      op->object_size(),
2305                      op->klass()->as_register(),
2306                      *op->stub()->entry());
2307   __ bind(*op->stub()->continuation());
2308   __ verify_oop(op->obj()->as_register());
2309 }
2310 
2311 void LIR_Assembler::emit_alloc_array(LIR_OpAllocArray* op) {
2312   Register len = op->len()->as_register();
2313   __ move_reg_if_needed(len, T_LONG, len, T_INT); // sign extend
2314 
2315   if (UseSlowPath ||
2316       (!UseFastNewObjectArray && (op->type() == T_OBJECT || op->type() == T_ARRAY)) ||
2317       (!UseFastNewTypeArray   && (op->type() != T_OBJECT && op->type() != T_ARRAY))) {
2318     __ z_brul(*op->stub()->entry());
2319   } else {
2320     __ allocate_array(op->obj()->as_register(),
2321                       op->len()->as_register(),
2322                       op->tmp1()->as_register(),
2323                       op->tmp2()->as_register(),
2324                       arrayOopDesc::header_size(op->type()),
2325                       type2aelembytes(op->type()),
2326                       op->klass()->as_register(),
2327                       *op->stub()->entry());
2328   }
2329   __ bind(*op->stub()->continuation());
2330 }
2331 
2332 void LIR_Assembler::type_profile_helper(Register mdo, ciMethodData *md, ciProfileData *data,
2333                                         Register recv, Register tmp1, Label* update_done) {
2334   uint i;
2335   for (i = 0; i < VirtualCallData::row_limit(); i++) {
2336     Label next_test;
2337     // See if the receiver is receiver[n].
2338     Address receiver_addr(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_offset(i)));
2339     __ z_cg(recv, receiver_addr);
2340     __ z_brne(next_test);
2341     Address data_addr(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_count_offset(i)));
2342     __ add2mem_64(data_addr, DataLayout::counter_increment, tmp1);
2343     __ branch_optimized(Assembler::bcondAlways, *update_done);
2344     __ bind(next_test);
2345   }
2346 
2347   // Didn't find receiver; find next empty slot and fill it in.
2348   for (i = 0; i < VirtualCallData::row_limit(); i++) {
2349     Label next_test;
2350     Address recv_addr(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_offset(i)));
2351     __ z_ltg(Z_R0_scratch, recv_addr);
2352     __ z_brne(next_test);
2353     __ z_stg(recv, recv_addr);
2354     __ load_const_optimized(tmp1, DataLayout::counter_increment);
2355     __ z_stg(tmp1, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_count_offset(i)), mdo);
2356     __ branch_optimized(Assembler::bcondAlways, *update_done);
2357     __ bind(next_test);
2358   }
2359 }
2360 
2361 void LIR_Assembler::setup_md_access(ciMethod* method, int bci,
2362                                     ciMethodData*& md, ciProfileData*& data, int& mdo_offset_bias) {
2363   Unimplemented();
2364 }
2365 
2366 void LIR_Assembler::store_parameter(Register r, int param_num) {
2367   assert(param_num >= 0, "invalid num");
2368   int offset_in_bytes = param_num * BytesPerWord + FrameMap::first_available_sp_in_frame;
2369   assert(offset_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2370   __ z_stg(r, offset_in_bytes, Z_SP);
2371 }
2372 
2373 void LIR_Assembler::store_parameter(jint c, int param_num) {
2374   assert(param_num >= 0, "invalid num");
2375   int offset_in_bytes = param_num * BytesPerWord + FrameMap::first_available_sp_in_frame;
2376   assert(offset_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2377   __ store_const(Address(Z_SP, offset_in_bytes), c, Z_R1_scratch, true);
2378 }
2379 
2380 void LIR_Assembler::emit_typecheck_helper(LIR_OpTypeCheck *op, Label* success, Label* failure, Label* obj_is_null) {
2381   // We always need a stub for the failure case.
2382   CodeStub* stub = op->stub();
2383   Register obj = op->object()->as_register();
2384   Register k_RInfo = op->tmp1()->as_register();
2385   Register klass_RInfo = op->tmp2()->as_register();
2386   Register dst = op->result_opr()->as_register();
2387   Register Rtmp1 = Z_R1_scratch;
2388   ciKlass* k = op->klass();
2389 
2390   assert(!op->tmp3()->is_valid(), "tmp3's not needed");
2391 
2392   // Check if it needs to be profiled.
2393   ciMethodData* md = NULL;
2394   ciProfileData* data = NULL;
2395 
2396   if (op->should_profile()) {
2397     ciMethod* method = op->profiled_method();
2398     assert(method != NULL, "Should have method");
2399     int bci = op->profiled_bci();
2400     md = method->method_data_or_null();
2401     assert(md != NULL, "Sanity");
2402     data = md->bci_to_data(bci);
2403     assert(data != NULL,                "need data for type check");
2404     assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
2405   }
2406 
2407   // Temp operands do not overlap with inputs, if this is their last
2408   // use (end of range is exclusive), so a register conflict is possible.
2409   if (obj == k_RInfo) {
2410     k_RInfo = dst;
2411   } else if (obj == klass_RInfo) {
2412     klass_RInfo = dst;
2413   }
2414   assert_different_registers(obj, k_RInfo, klass_RInfo);
2415 
2416   if (op->should_profile()) {
2417     NearLabel not_null;
2418     __ compareU64_and_branch(obj, (intptr_t) 0, Assembler::bcondNotEqual, not_null);
2419     // Object is null; update MDO and exit.
2420     Register mdo = klass_RInfo;
2421     metadata2reg(md->constant_encoding(), mdo);
2422     Address data_addr(mdo, md->byte_offset_of_slot(data, DataLayout::header_offset()));
2423     int header_bits = DataLayout::flag_mask_to_header_mask(BitData::null_seen_byte_constant());
2424     __ or2mem_8(data_addr, header_bits);
2425     __ branch_optimized(Assembler::bcondAlways, *obj_is_null);
2426     __ bind(not_null);
2427   } else {
2428     __ compareU64_and_branch(obj, (intptr_t) 0, Assembler::bcondEqual, *obj_is_null);
2429   }
2430 
2431   NearLabel profile_cast_failure, profile_cast_success;
2432   Label *failure_target = op->should_profile() ? &profile_cast_failure : failure;
2433   Label *success_target = op->should_profile() ? &profile_cast_success : success;
2434 
2435   // Patching may screw with our temporaries on sparc,
2436   // so let's do it before loading the class.
2437   if (k->is_loaded()) {
2438     metadata2reg(k->constant_encoding(), k_RInfo);
2439   } else {
2440     klass2reg_with_patching(k_RInfo, op->info_for_patch());
2441   }
2442   assert(obj != k_RInfo, "must be different");
2443 
2444   __ verify_oop(obj);
2445 
2446   // Get object class.
2447   // Not a safepoint as obj null check happens earlier.
2448   if (op->fast_check()) {
2449     if (UseCompressedClassPointers) {
2450       __ load_klass(klass_RInfo, obj);
2451       __ compareU64_and_branch(k_RInfo, klass_RInfo, Assembler::bcondNotEqual, *failure_target);
2452     } else {
2453       __ z_cg(k_RInfo, Address(obj, oopDesc::klass_offset_in_bytes()));
2454       __ branch_optimized(Assembler::bcondNotEqual, *failure_target);
2455     }
2456     // Successful cast, fall through to profile or jump.
2457   } else {
2458     bool need_slow_path = !k->is_loaded() ||
2459                           ((int) k->super_check_offset() == in_bytes(Klass::secondary_super_cache_offset()));
2460     intptr_t super_check_offset = k->is_loaded() ? k->super_check_offset() : -1L;
2461     __ load_klass(klass_RInfo, obj);
2462     // Perform the fast part of the checking logic.
2463     __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1,
2464                                      (need_slow_path ? success_target : NULL),
2465                                      failure_target, NULL,
2466                                      RegisterOrConstant(super_check_offset));
2467     if (need_slow_path) {
2468       // Call out-of-line instance of __ check_klass_subtype_slow_path(...):
2469       address a = Runtime1::entry_for (Runtime1::slow_subtype_check_id);
2470       store_parameter(klass_RInfo, 0); // sub
2471       store_parameter(k_RInfo, 1);     // super
2472       emit_call_c(a); // Sets condition code 0 for match (2 otherwise).
2473       CHECK_BAILOUT();
2474       __ branch_optimized(Assembler::bcondNotEqual, *failure_target);
2475       // Fall through to success case.
2476     }
2477   }
2478 
2479   if (op->should_profile()) {
2480     Register mdo = klass_RInfo, recv = k_RInfo;
2481     assert_different_registers(obj, mdo, recv);
2482     __ bind(profile_cast_success);
2483     metadata2reg(md->constant_encoding(), mdo);
2484     __ load_klass(recv, obj);
2485     type_profile_helper(mdo, md, data, recv, Rtmp1, success);
2486     __ branch_optimized(Assembler::bcondAlways, *success);
2487 
2488     __ bind(profile_cast_failure);
2489     metadata2reg(md->constant_encoding(), mdo);
2490     __ add2mem_64(Address(mdo, md->byte_offset_of_slot(data, CounterData::count_offset())), -(int)DataLayout::counter_increment, Rtmp1);
2491     __ branch_optimized(Assembler::bcondAlways, *failure);
2492   } else {
2493     __ branch_optimized(Assembler::bcondAlways, *success);
2494   }
2495 }
2496 
2497 void LIR_Assembler::emit_opTypeCheck(LIR_OpTypeCheck* op) {
2498   LIR_Code code = op->code();
2499   if (code == lir_store_check) {
2500     Register value = op->object()->as_register();
2501     Register array = op->array()->as_register();
2502     Register k_RInfo = op->tmp1()->as_register();
2503     Register klass_RInfo = op->tmp2()->as_register();
2504     Register Rtmp1 = Z_R1_scratch;
2505 
2506     CodeStub* stub = op->stub();
2507 
2508     // Check if it needs to be profiled.
2509     ciMethodData* md = NULL;
2510     ciProfileData* data = NULL;
2511 
2512     assert_different_registers(value, k_RInfo, klass_RInfo);
2513 
2514     if (op->should_profile()) {
2515       ciMethod* method = op->profiled_method();
2516       assert(method != NULL, "Should have method");
2517       int bci = op->profiled_bci();
2518       md = method->method_data_or_null();
2519       assert(md != NULL, "Sanity");
2520       data = md->bci_to_data(bci);
2521       assert(data != NULL,                "need data for type check");
2522       assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
2523     }
2524     NearLabel profile_cast_success, profile_cast_failure, done;
2525     Label *success_target = op->should_profile() ? &profile_cast_success : &done;
2526     Label *failure_target = op->should_profile() ? &profile_cast_failure : stub->entry();
2527 
2528     if (op->should_profile()) {
2529       NearLabel not_null;
2530       __ compareU64_and_branch(value, (intptr_t) 0, Assembler::bcondNotEqual, not_null);
2531       // Object is null; update MDO and exit.
2532       Register mdo = klass_RInfo;
2533       metadata2reg(md->constant_encoding(), mdo);
2534       Address data_addr(mdo, md->byte_offset_of_slot(data, DataLayout::header_offset()));
2535       int header_bits = DataLayout::flag_mask_to_header_mask(BitData::null_seen_byte_constant());
2536       __ or2mem_8(data_addr, header_bits);
2537       __ branch_optimized(Assembler::bcondAlways, done);
2538       __ bind(not_null);
2539     } else {
2540       __ compareU64_and_branch(value, (intptr_t) 0, Assembler::bcondEqual, done);
2541     }
2542 
2543     add_debug_info_for_null_check_here(op->info_for_exception());
2544     __ load_klass(k_RInfo, array);
2545     __ load_klass(klass_RInfo, value);
2546 
2547     // Get instance klass (it's already uncompressed).
2548     __ z_lg(k_RInfo, Address(k_RInfo, ObjArrayKlass::element_klass_offset()));
2549     // Perform the fast part of the checking logic.
2550     __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, NULL);
2551     // Call out-of-line instance of __ check_klass_subtype_slow_path(...):
2552     address a = Runtime1::entry_for (Runtime1::slow_subtype_check_id);
2553     store_parameter(klass_RInfo, 0); // sub
2554     store_parameter(k_RInfo, 1);     // super
2555     emit_call_c(a); // Sets condition code 0 for match (2 otherwise).
2556     CHECK_BAILOUT();
2557     __ branch_optimized(Assembler::bcondNotEqual, *failure_target);
2558     // Fall through to success case.
2559 
2560     if (op->should_profile()) {
2561       Register mdo = klass_RInfo, recv = k_RInfo;
2562       assert_different_registers(value, mdo, recv);
2563       __ bind(profile_cast_success);
2564       metadata2reg(md->constant_encoding(), mdo);
2565       __ load_klass(recv, value);
2566       type_profile_helper(mdo, md, data, recv, Rtmp1, &done);
2567       __ branch_optimized(Assembler::bcondAlways, done);
2568 
2569       __ bind(profile_cast_failure);
2570       metadata2reg(md->constant_encoding(), mdo);
2571       __ add2mem_64(Address(mdo, md->byte_offset_of_slot(data, CounterData::count_offset())), -(int)DataLayout::counter_increment, Rtmp1);
2572       __ branch_optimized(Assembler::bcondAlways, *stub->entry());
2573     }
2574 
2575     __ bind(done);
2576   } else {
2577     if (code == lir_checkcast) {
2578       Register obj = op->object()->as_register();
2579       Register dst = op->result_opr()->as_register();
2580       NearLabel success;
2581       emit_typecheck_helper(op, &success, op->stub()->entry(), &success);
2582       __ bind(success);
2583       __ lgr_if_needed(dst, obj);
2584     } else {
2585       if (code == lir_instanceof) {
2586         Register obj = op->object()->as_register();
2587         Register dst = op->result_opr()->as_register();
2588         NearLabel success, failure, done;
2589         emit_typecheck_helper(op, &success, &failure, &failure);
2590         __ bind(failure);
2591         __ clear_reg(dst);
2592         __ branch_optimized(Assembler::bcondAlways, done);
2593         __ bind(success);
2594         __ load_const_optimized(dst, 1);
2595         __ bind(done);
2596       } else {
2597         ShouldNotReachHere();
2598       }
2599     }
2600   }
2601 }
2602 
2603 void LIR_Assembler::emit_compare_and_swap(LIR_OpCompareAndSwap* op) {
2604   Register addr = op->addr()->as_pointer_register();
2605   Register t1_cmp = Z_R1_scratch;
2606   if (op->code() == lir_cas_long) {
2607     assert(VM_Version::supports_cx8(), "wrong machine");
2608     Register cmp_value_lo = op->cmp_value()->as_register_lo();
2609     Register new_value_lo = op->new_value()->as_register_lo();
2610     __ z_lgr(t1_cmp, cmp_value_lo);
2611     // Perform the compare and swap operation.
2612     __ z_csg(t1_cmp, new_value_lo, 0, addr);
2613   } else if (op->code() == lir_cas_int || op->code() == lir_cas_obj) {
2614     Register cmp_value = op->cmp_value()->as_register();
2615     Register new_value = op->new_value()->as_register();
2616     if (op->code() == lir_cas_obj) {
2617       if (UseCompressedOops) {
2618                  t1_cmp = op->tmp1()->as_register();
2619         Register t2_new = op->tmp2()->as_register();
2620         assert_different_registers(cmp_value, new_value, addr, t1_cmp, t2_new);
2621         __ oop_encoder(t1_cmp, cmp_value, true /*maybe null*/);
2622         __ oop_encoder(t2_new, new_value, true /*maybe null*/);
2623         __ z_cs(t1_cmp, t2_new, 0, addr);
2624       } else {
2625         __ z_lgr(t1_cmp, cmp_value);
2626         __ z_csg(t1_cmp, new_value, 0, addr);
2627       }
2628     } else {
2629       __ z_lr(t1_cmp, cmp_value);
2630       __ z_cs(t1_cmp, new_value, 0, addr);
2631     }
2632   } else {
2633     ShouldNotReachHere(); // new lir_cas_??
2634   }
2635 }
2636 
2637 void LIR_Assembler::set_24bit_FPU() {
2638   ShouldNotCallThis(); // x86 only
2639 }
2640 
2641 void LIR_Assembler::reset_FPU() {
2642   ShouldNotCallThis(); // x86 only
2643 }
2644 
2645 void LIR_Assembler::breakpoint() {
2646   Unimplemented();
2647   //  __ breakpoint_trap();
2648 }
2649 
2650 void LIR_Assembler::push(LIR_Opr opr) {
2651   ShouldNotCallThis(); // unused
2652 }
2653 
2654 void LIR_Assembler::pop(LIR_Opr opr) {
2655   ShouldNotCallThis(); // unused
2656 }
2657 
2658 void LIR_Assembler::monitor_address(int monitor_no, LIR_Opr dst_opr) {
2659   Address addr = frame_map()->address_for_monitor_lock(monitor_no);
2660   __ add2reg(dst_opr->as_register(), addr.disp(), addr.base());
2661 }
2662 
2663 void LIR_Assembler::emit_lock(LIR_OpLock* op) {
2664   Register obj = op->obj_opr()->as_register();  // May not be an oop.
2665   Register hdr = op->hdr_opr()->as_register();
2666   Register lock = op->lock_opr()->as_register();
2667   if (!UseFastLocking) {
2668     __ branch_optimized(Assembler::bcondAlways, *op->stub()->entry());
2669   } else if (op->code() == lir_lock) {
2670     assert(BasicLock::displaced_header_offset_in_bytes() == 0, "lock_reg must point to the displaced header");
2671     // Add debug info for NullPointerException only if one is possible.
2672     if (op->info() != NULL) {
2673       add_debug_info_for_null_check_here(op->info());
2674     }
2675     __ lock_object(hdr, obj, lock, *op->stub()->entry());
2676     // done
2677   } else if (op->code() == lir_unlock) {
2678     assert(BasicLock::displaced_header_offset_in_bytes() == 0, "lock_reg must point to the displaced header");
2679     __ unlock_object(hdr, obj, lock, *op->stub()->entry());
2680   } else {
2681     ShouldNotReachHere();
2682   }
2683   __ bind(*op->stub()->continuation());
2684 }
2685 
2686 void LIR_Assembler::emit_profile_call(LIR_OpProfileCall* op) {
2687   ciMethod* method = op->profiled_method();
2688   int bci          = op->profiled_bci();
2689   ciMethod* callee = op->profiled_callee();
2690 
2691   // Update counter for all call types.
2692   ciMethodData* md = method->method_data_or_null();
2693   assert(md != NULL, "Sanity");
2694   ciProfileData* data = md->bci_to_data(bci);
2695   assert(data->is_CounterData(), "need CounterData for calls");
2696   assert(op->mdo()->is_single_cpu(),  "mdo must be allocated");
2697   Register mdo  = op->mdo()->as_register();
2698   assert(op->tmp1()->is_double_cpu(), "tmp1 must be allocated");
2699   Register tmp1 = op->tmp1()->as_register_lo();
2700   metadata2reg(md->constant_encoding(), mdo);
2701 
2702   Address counter_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
2703   Bytecodes::Code bc = method->java_code_at_bci(bci);
2704   const bool callee_is_static = callee->is_loaded() && callee->is_static();
2705   // Perform additional virtual call profiling for invokevirtual and
2706   // invokeinterface bytecodes.
2707   if ((bc == Bytecodes::_invokevirtual || bc == Bytecodes::_invokeinterface) &&
2708       !callee_is_static &&  // Required for optimized MH invokes.
2709       C1ProfileVirtualCalls) {
2710     assert(op->recv()->is_single_cpu(), "recv must be allocated");
2711     Register recv = op->recv()->as_register();
2712     assert_different_registers(mdo, tmp1, recv);
2713     assert(data->is_VirtualCallData(), "need VirtualCallData for virtual calls");
2714     ciKlass* known_klass = op->known_holder();
2715     if (C1OptimizeVirtualCallProfiling && known_klass != NULL) {
2716       // We know the type that will be seen at this call site; we can
2717       // statically update the MethodData* rather than needing to do
2718       // dynamic tests on the receiver type.
2719 
2720       // NOTE: we should probably put a lock around this search to
2721       // avoid collisions by concurrent compilations.
2722       ciVirtualCallData* vc_data = (ciVirtualCallData*) data;
2723       uint i;
2724       for (i = 0; i < VirtualCallData::row_limit(); i++) {
2725         ciKlass* receiver = vc_data->receiver(i);
2726         if (known_klass->equals(receiver)) {
2727           Address data_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_count_offset(i)));
2728           __ add2mem_64(data_addr, DataLayout::counter_increment, tmp1);
2729           return;
2730         }
2731       }
2732 
2733       // Receiver type not found in profile data. Select an empty slot.
2734 
2735       // Note that this is less efficient than it should be because it
2736       // always does a write to the receiver part of the
2737       // VirtualCallData rather than just the first time.
2738       for (i = 0; i < VirtualCallData::row_limit(); i++) {
2739         ciKlass* receiver = vc_data->receiver(i);
2740         if (receiver == NULL) {
2741           Address recv_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_offset(i)));
2742           metadata2reg(known_klass->constant_encoding(), tmp1);
2743           __ z_stg(tmp1, recv_addr);
2744           Address data_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_count_offset(i)));
2745           __ add2mem_64(data_addr, DataLayout::counter_increment, tmp1);
2746           return;
2747         }
2748       }
2749     } else {
2750       __ load_klass(recv, recv);
2751       NearLabel update_done;
2752       type_profile_helper(mdo, md, data, recv, tmp1, &update_done);
2753       // Receiver did not match any saved receiver and there is no empty row for it.
2754       // Increment total counter to indicate polymorphic case.
2755       __ add2mem_64(counter_addr, DataLayout::counter_increment, tmp1);
2756       __ bind(update_done);
2757     }
2758   } else {
2759     // static call
2760     __ add2mem_64(counter_addr, DataLayout::counter_increment, tmp1);
2761   }
2762 }
2763 
2764 void LIR_Assembler::align_backward_branch_target() {
2765   __ align(OptoLoopAlignment);
2766 }
2767 
2768 void LIR_Assembler::emit_delay(LIR_OpDelay* op) {
2769   ShouldNotCallThis(); // There are no delay slots on ZARCH_64.
2770 }
2771 
2772 void LIR_Assembler::negate(LIR_Opr left, LIR_Opr dest) {
2773   assert(left->is_register(), "can only handle registers");
2774 
2775   if (left->is_single_cpu()) {
2776     __ z_lcr(dest->as_register(), left->as_register());
2777   } else if (left->is_single_fpu()) {
2778     __ z_lcebr(dest->as_float_reg(), left->as_float_reg());
2779   } else if (left->is_double_fpu()) {
2780     __ z_lcdbr(dest->as_double_reg(), left->as_double_reg());
2781   } else {
2782     assert(left->is_double_cpu(), "Must be a long");
2783     __ z_lcgr(dest->as_register_lo(), left->as_register_lo());
2784   }
2785 }
2786 
2787 void LIR_Assembler::fxch(int i) {
2788   ShouldNotCallThis(); // x86 only
2789 }
2790 
2791 void LIR_Assembler::fld(int i) {
2792   ShouldNotCallThis(); // x86 only
2793 }
2794 
2795 void LIR_Assembler::ffree(int i) {
2796   ShouldNotCallThis(); // x86 only
2797 }
2798 
2799 void LIR_Assembler::rt_call(LIR_Opr result, address dest,
2800                             const LIR_OprList* args, LIR_Opr tmp, CodeEmitInfo* info) {
2801   assert(!tmp->is_valid(), "don't need temporary");
2802   emit_call_c(dest);
2803   CHECK_BAILOUT();
2804   if (info != NULL) {
2805     add_call_info_here(info);
2806   }
2807 }
2808 
2809 void LIR_Assembler::volatile_move_op(LIR_Opr src, LIR_Opr dest, BasicType type, CodeEmitInfo* info) {
2810   ShouldNotCallThis(); // not needed on ZARCH_64
2811 }
2812 
2813 void LIR_Assembler::membar() {
2814   __ z_fence();
2815 }
2816 
2817 void LIR_Assembler::membar_acquire() {
2818   __ z_acquire();
2819 }
2820 
2821 void LIR_Assembler::membar_release() {
2822   __ z_release();
2823 }
2824 
2825 void LIR_Assembler::membar_loadload() {
2826   __ z_acquire();
2827 }
2828 
2829 void LIR_Assembler::membar_storestore() {
2830   __ z_release();
2831 }
2832 
2833 void LIR_Assembler::membar_loadstore() {
2834   __ z_acquire();
2835 }
2836 
2837 void LIR_Assembler::membar_storeload() {
2838   __ z_fence();
2839 }
2840 
2841 void LIR_Assembler::on_spin_wait() {
2842   Unimplemented();
2843 }
2844 
2845 void LIR_Assembler::leal(LIR_Opr addr_opr, LIR_Opr dest) {
2846   LIR_Address* addr = addr_opr->as_address_ptr();
2847   assert(addr->scale() == LIR_Address::times_1, "scaling unsupported");
2848   __ load_address(dest->as_pointer_register(), as_Address(addr));
2849 }
2850 
2851 void LIR_Assembler::get_thread(LIR_Opr result_reg) {
2852   ShouldNotCallThis(); // unused
2853 }
2854 
2855 #ifdef ASSERT
2856 // Emit run-time assertion.
2857 void LIR_Assembler::emit_assert(LIR_OpAssert* op) {
2858   Unimplemented();
2859 }
2860 #endif
2861 
2862 void LIR_Assembler::peephole(LIR_List*) {
2863   // Do nothing for now.
2864 }
2865 
2866 void LIR_Assembler::atomic_op(LIR_Code code, LIR_Opr src, LIR_Opr data, LIR_Opr dest, LIR_Opr tmp) {
2867   assert(code == lir_xadd, "lir_xchg not supported");
2868   Address src_addr = as_Address(src->as_address_ptr());
2869   Register base = src_addr.base();
2870   intptr_t disp = src_addr.disp();
2871   if (src_addr.index()->is_valid()) {
2872     // LAA and LAAG do not support index register.
2873     __ load_address(Z_R1_scratch, src_addr);
2874     base = Z_R1_scratch;
2875     disp = 0;
2876   }
2877   if (data->type() == T_INT) {
2878     __ z_laa(dest->as_register(), data->as_register(), disp, base);
2879   } else if (data->type() == T_LONG) {
2880     assert(data->as_register_lo() == data->as_register_hi(), "should be a single register");
2881     __ z_laag(dest->as_register_lo(), data->as_register_lo(), disp, base);
2882   } else {
2883     ShouldNotReachHere();
2884   }
2885 }
2886 
2887 void LIR_Assembler::emit_profile_type(LIR_OpProfileType* op) {
2888   Register obj = op->obj()->as_register();
2889   Register tmp1 = op->tmp()->as_pointer_register();
2890   Register tmp2 = Z_R1_scratch;
2891   Address mdo_addr = as_Address(op->mdp()->as_address_ptr());
2892   ciKlass* exact_klass = op->exact_klass();
2893   intptr_t current_klass = op->current_klass();
2894   bool not_null = op->not_null();
2895   bool no_conflict = op->no_conflict();
2896 
2897   Label update, next, none, null_seen, init_klass;
2898 
2899   bool do_null = !not_null;
2900   bool exact_klass_set = exact_klass != NULL && ciTypeEntries::valid_ciklass(current_klass) == exact_klass;
2901   bool do_update = !TypeEntries::is_type_unknown(current_klass) && !exact_klass_set;
2902 
2903   assert(do_null || do_update, "why are we here?");
2904   assert(!TypeEntries::was_null_seen(current_klass) || do_update, "why are we here?");
2905 
2906   __ verify_oop(obj);
2907 
2908   if (do_null || tmp1 != obj DEBUG_ONLY(|| true)) {
2909     __ z_ltgr(tmp1, obj);
2910   }
2911   if (do_null) {
2912     __ z_brnz(update);
2913     if (!TypeEntries::was_null_seen(current_klass)) {
2914       __ z_lg(tmp1, mdo_addr);
2915       __ z_oill(tmp1, TypeEntries::null_seen);
2916       __ z_stg(tmp1, mdo_addr);
2917     }
2918     if (do_update) {
2919       __ z_bru(next);
2920     }
2921   } else {
2922     __ asm_assert_ne("unexpect null obj", __LINE__);
2923   }
2924 
2925   __ bind(update);
2926 
2927   if (do_update) {
2928 #ifdef ASSERT
2929     if (exact_klass != NULL) {
2930       __ load_klass(tmp1, tmp1);
2931       metadata2reg(exact_klass->constant_encoding(), tmp2);
2932       __ z_cgr(tmp1, tmp2);
2933       __ asm_assert_eq("exact klass and actual klass differ", __LINE__);
2934     }
2935 #endif
2936 
2937     Label do_update;
2938     __ z_lg(tmp2, mdo_addr);
2939 
2940     if (!no_conflict) {
2941       if (exact_klass == NULL || TypeEntries::is_type_none(current_klass)) {
2942         if (exact_klass != NULL) {
2943           metadata2reg(exact_klass->constant_encoding(), tmp1);
2944         } else {
2945           __ load_klass(tmp1, tmp1);
2946         }
2947 
2948         // Klass seen before: nothing to do (regardless of unknown bit).
2949         __ z_lgr(Z_R0_scratch, tmp2);
2950         assert(Immediate::is_uimm(~TypeEntries::type_klass_mask, 16), "or change following instruction");
2951         __ z_nill(Z_R0_scratch, TypeEntries::type_klass_mask & 0xFFFF);
2952         __ compareU64_and_branch(Z_R0_scratch, tmp1, Assembler::bcondEqual, next);
2953 
2954         // Already unknown: Nothing to do anymore.
2955         __ z_tmll(tmp2, TypeEntries::type_unknown);
2956         __ z_brc(Assembler::bcondAllOne, next);
2957 
2958         if (TypeEntries::is_type_none(current_klass)) {
2959           __ z_lgr(Z_R0_scratch, tmp2);
2960           assert(Immediate::is_uimm(~TypeEntries::type_mask, 16), "or change following instruction");
2961           __ z_nill(Z_R0_scratch, TypeEntries::type_mask & 0xFFFF);
2962           __ compareU64_and_branch(Z_R0_scratch, (intptr_t)0, Assembler::bcondEqual, init_klass);
2963         }
2964       } else {
2965         assert(ciTypeEntries::valid_ciklass(current_klass) != NULL &&
2966                ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "conflict only");
2967 
2968         // Already unknown: Nothing to do anymore.
2969         __ z_tmll(tmp2, TypeEntries::type_unknown);
2970         __ z_brc(Assembler::bcondAllOne, next);
2971       }
2972 
2973       // Different than before. Cannot keep accurate profile.
2974       __ z_oill(tmp2, TypeEntries::type_unknown);
2975       __ z_bru(do_update);
2976     } else {
2977       // There's a single possible klass at this profile point.
2978       assert(exact_klass != NULL, "should be");
2979       if (TypeEntries::is_type_none(current_klass)) {
2980         metadata2reg(exact_klass->constant_encoding(), tmp1);
2981         __ z_lgr(Z_R0_scratch, tmp2);
2982         assert(Immediate::is_uimm(~TypeEntries::type_klass_mask, 16), "or change following instruction");
2983         __ z_nill(Z_R0_scratch, TypeEntries::type_klass_mask & 0xFFFF);
2984         __ compareU64_and_branch(Z_R0_scratch, tmp1, Assembler::bcondEqual, next);
2985 #ifdef ASSERT
2986         {
2987           Label ok;
2988           __ z_lgr(Z_R0_scratch, tmp2);
2989           assert(Immediate::is_uimm(~TypeEntries::type_mask, 16), "or change following instruction");
2990           __ z_nill(Z_R0_scratch, TypeEntries::type_mask & 0xFFFF);
2991           __ compareU64_and_branch(Z_R0_scratch, (intptr_t)0, Assembler::bcondEqual, ok);
2992           __ stop("unexpected profiling mismatch");
2993           __ bind(ok);
2994         }
2995 #endif
2996 
2997       } else {
2998         assert(ciTypeEntries::valid_ciklass(current_klass) != NULL &&
2999                ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "inconsistent");
3000 
3001         // Already unknown: Nothing to do anymore.
3002         __ z_tmll(tmp2, TypeEntries::type_unknown);
3003         __ z_brc(Assembler::bcondAllOne, next);
3004         __ z_oill(tmp2, TypeEntries::type_unknown);
3005         __ z_bru(do_update);
3006       }
3007     }
3008 
3009     __ bind(init_klass);
3010     // Combine klass and null_seen bit (only used if (tmp & type_mask)==0).
3011     __ z_ogr(tmp2, tmp1);
3012 
3013     __ bind(do_update);
3014     __ z_stg(tmp2, mdo_addr);
3015 
3016     __ bind(next);
3017   }
3018 }
3019 
3020 void LIR_Assembler::emit_updatecrc32(LIR_OpUpdateCRC32* op) {
3021   assert(op->crc()->is_single_cpu(), "crc must be register");
3022   assert(op->val()->is_single_cpu(), "byte value must be register");
3023   assert(op->result_opr()->is_single_cpu(), "result must be register");
3024   Register crc = op->crc()->as_register();
3025   Register val = op->val()->as_register();
3026   Register res = op->result_opr()->as_register();
3027 
3028   assert_different_registers(val, crc, res);
3029 
3030   __ load_const_optimized(res, StubRoutines::crc_table_addr());
3031   __ not_(crc, noreg, false); // ~crc
3032   __ update_byte_crc32(crc, val, res);
3033   __ not_(res, crc, false); // ~crc
3034 }
3035 
3036 #undef __