1 /*
   2  * Copyright (c) 2016, 2018, Oracle and/or its affiliates. All rights reserved.
   3  * Copyright (c) 2016, 2017, SAP SE. All rights reserved.
   4  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   5  *
   6  * This code is free software; you can redistribute it and/or modify it
   7  * under the terms of the GNU General Public License version 2 only, as
   8  * published by the Free Software Foundation.
   9  *
  10  * This code is distributed in the hope that it will be useful, but WITHOUT
  11  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  12  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  13  * version 2 for more details (a copy is included in the LICENSE file that
  14  * accompanied this code).
  15  *
  16  * You should have received a copy of the GNU General Public License version
  17  * 2 along with this work; if not, write to the Free Software Foundation,
  18  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  19  *
  20  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  21  * or visit www.oracle.com if you need additional information or have any
  22  * questions.
  23  *
  24  */
  25 
  26 #include "precompiled.hpp"
  27 #include "asm/macroAssembler.inline.hpp"
  28 #include "c1/c1_Compilation.hpp"
  29 #include "c1/c1_LIRAssembler.hpp"
  30 #include "c1/c1_MacroAssembler.hpp"
  31 #include "c1/c1_Runtime1.hpp"
  32 #include "c1/c1_ValueStack.hpp"
  33 #include "ci/ciArrayKlass.hpp"
  34 #include "ci/ciInstance.hpp"
  35 #include "gc/shared/collectedHeap.hpp"
  36 #include "gc/shared/barrierSet.hpp"
  37 #include "gc/shared/cardTableBarrierSet.hpp"
  38 #include "nativeInst_s390.hpp"
  39 #include "oops/objArrayKlass.hpp"
  40 #include "runtime/frame.inline.hpp"
  41 #include "runtime/safepointMechanism.inline.hpp"
  42 #include "runtime/sharedRuntime.hpp"
  43 #include "vmreg_s390.inline.hpp"
  44 
  45 #define __ _masm->
  46 
  47 #ifndef PRODUCT
  48 #undef __
  49 #define __ (Verbose ? (_masm->block_comment(FILE_AND_LINE),_masm) : _masm)->
  50 #endif
  51 
  52 //------------------------------------------------------------
  53 
  54 bool LIR_Assembler::is_small_constant(LIR_Opr opr) {
  55   // Not used on ZARCH_64
  56   ShouldNotCallThis();
  57   return false;
  58 }
  59 
  60 LIR_Opr LIR_Assembler::receiverOpr() {
  61   return FrameMap::Z_R2_oop_opr;
  62 }
  63 
  64 LIR_Opr LIR_Assembler::osrBufferPointer() {
  65   return FrameMap::Z_R2_opr;
  66 }
  67 
  68 int LIR_Assembler::initial_frame_size_in_bytes() const {
  69   return in_bytes(frame_map()->framesize_in_bytes());
  70 }
  71 
  72 // Inline cache check: done before the frame is built.
  73 // The inline cached class is in Z_inline_cache(Z_R9).
  74 // We fetch the class of the receiver and compare it with the cached class.
  75 // If they do not match we jump to the slow case.
  76 int LIR_Assembler::check_icache() {
  77   Register receiver = receiverOpr()->as_register();
  78   int offset = __ offset();
  79   __ inline_cache_check(receiver, Z_inline_cache);
  80   return offset;
  81 }
  82 




  83 void LIR_Assembler::osr_entry() {
  84   // On-stack-replacement entry sequence (interpreter frame layout described in interpreter_sparc.cpp):
  85   //
  86   //   1. Create a new compiled activation.
  87   //   2. Initialize local variables in the compiled activation. The expression stack must be empty
  88   //      at the osr_bci; it is not initialized.
  89   //   3. Jump to the continuation address in compiled code to resume execution.
  90 
  91   // OSR entry point
  92   offsets()->set_value(CodeOffsets::OSR_Entry, code_offset());
  93   BlockBegin* osr_entry = compilation()->hir()->osr_entry();
  94   ValueStack* entry_state = osr_entry->end()->state();
  95   int number_of_locks = entry_state->locks_size();
  96 
  97   // Create a frame for the compiled activation.
  98   __ build_frame(initial_frame_size_in_bytes(), bang_size_in_bytes());
  99 
 100   // OSR buffer is
 101   //
 102   // locals[nlocals-1..0]
 103   // monitors[number_of_locks-1..0]
 104   //
 105   // Locals is a direct copy of the interpreter frame so in the osr buffer
 106   // the first slot in the local array is the last local from the interpreter
 107   // and the last slot is local[0] (receiver) from the interpreter
 108   //
 109   // Similarly with locks. The first lock slot in the osr buffer is the nth lock
 110   // from the interpreter frame, the nth lock slot in the osr buffer is 0th lock
 111   // in the interpreter frame (the method lock if a sync method)
 112 
 113   // Initialize monitors in the compiled activation.
 114   //   I0: pointer to osr buffer
 115   //
 116   // All other registers are dead at this point and the locals will be
 117   // copied into place by code emitted in the IR.
 118 
 119   Register OSR_buf = osrBufferPointer()->as_register();
 120   { assert(frame::interpreter_frame_monitor_size() == BasicObjectLock::size(), "adjust code below");
 121     int monitor_offset = BytesPerWord * method()->max_locals() +
 122       (2 * BytesPerWord) * (number_of_locks - 1);
 123     // SharedRuntime::OSR_migration_begin() packs BasicObjectLocks in
 124     // the OSR buffer using 2 word entries: first the lock and then
 125     // the oop.
 126     for (int i = 0; i < number_of_locks; i++) {
 127       int slot_offset = monitor_offset - ((i * 2) * BytesPerWord);
 128       // Verify the interpreter's monitor has a non-null object.
 129       __ asm_assert_mem8_isnot_zero(slot_offset + 1*BytesPerWord, OSR_buf, "locked object is NULL", __LINE__);
 130       // Copy the lock field into the compiled activation.
 131       __ z_lg(Z_R1_scratch, slot_offset + 0, OSR_buf);
 132       __ z_stg(Z_R1_scratch, frame_map()->address_for_monitor_lock(i));
 133       __ z_lg(Z_R1_scratch, slot_offset + 1*BytesPerWord, OSR_buf);
 134       __ z_stg(Z_R1_scratch, frame_map()->address_for_monitor_object(i));
 135     }
 136   }
 137 }
 138 
 139 // --------------------------------------------------------------------------------------------
 140 
 141 address LIR_Assembler::emit_call_c(address a) {
 142   __ align_call_far_patchable(__ pc());
 143   address call_addr = __ call_c_opt(a);
 144   if (call_addr == NULL) {
 145     bailout("const section overflow");
 146   }
 147   return call_addr;
 148 }
 149 
 150 int LIR_Assembler::emit_exception_handler() {
 151   // If the last instruction is a call (typically to do a throw which
 152   // is coming at the end after block reordering) the return address
 153   // must still point into the code area in order to avoid assertion
 154   // failures when searching for the corresponding bci. => Add a nop.
 155   // (was bug 5/14/1999 - gri)
 156   __ nop();
 157 
 158   // Generate code for exception handler.
 159   address handler_base = __ start_a_stub(exception_handler_size());
 160   if (handler_base == NULL) {
 161     // Not enough space left for the handler.
 162     bailout("exception handler overflow");
 163     return -1;
 164   }
 165 
 166   int offset = code_offset();
 167 
 168   address a = Runtime1::entry_for (Runtime1::handle_exception_from_callee_id);
 169   address call_addr = emit_call_c(a);
 170   CHECK_BAILOUT_(-1);
 171   __ should_not_reach_here();
 172   guarantee(code_offset() - offset <= exception_handler_size(), "overflow");
 173   __ end_a_stub();
 174 
 175   return offset;
 176 }
 177 
 178 // Emit the code to remove the frame from the stack in the exception
 179 // unwind path.
 180 int LIR_Assembler::emit_unwind_handler() {
 181 #ifndef PRODUCT
 182   if (CommentedAssembly) {
 183     _masm->block_comment("Unwind handler");
 184   }
 185 #endif
 186 
 187   int offset = code_offset();
 188   Register exception_oop_callee_saved = Z_R10; // Z_R10 is callee-saved.
 189   Register Rtmp1                      = Z_R11;
 190   Register Rtmp2                      = Z_R12;
 191 
 192   // Fetch the exception from TLS and clear out exception related thread state.
 193   Address exc_oop_addr = Address(Z_thread, JavaThread::exception_oop_offset());
 194   Address exc_pc_addr  = Address(Z_thread, JavaThread::exception_pc_offset());
 195   __ z_lg(Z_EXC_OOP, exc_oop_addr);
 196   __ clear_mem(exc_oop_addr, sizeof(oop));
 197   __ clear_mem(exc_pc_addr, sizeof(intptr_t));
 198 
 199   __ bind(_unwind_handler_entry);
 200   __ verify_not_null_oop(Z_EXC_OOP);
 201   if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
 202     __ lgr_if_needed(exception_oop_callee_saved, Z_EXC_OOP); // Preserve the exception.
 203   }
 204 
 205   // Preform needed unlocking.
 206   MonitorExitStub* stub = NULL;
 207   if (method()->is_synchronized()) {
 208     // Runtime1::monitorexit_id expects lock address in Z_R1_scratch.
 209     LIR_Opr lock = FrameMap::as_opr(Z_R1_scratch);
 210     monitor_address(0, lock);
 211     stub = new MonitorExitStub(lock, true, 0);
 212     __ unlock_object(Rtmp1, Rtmp2, lock->as_register(), *stub->entry());
 213     __ bind(*stub->continuation());
 214   }
 215 
 216   if (compilation()->env()->dtrace_method_probes()) {
 217     ShouldNotReachHere(); // Not supported.
 218 #if 0
 219     __ mov(rdi, r15_thread);
 220     __ mov_metadata(rsi, method()->constant_encoding());
 221     __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_exit)));
 222 #endif
 223   }
 224 
 225   if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
 226     __ lgr_if_needed(Z_EXC_OOP, exception_oop_callee_saved);  // Restore the exception.
 227   }
 228 
 229   // Remove the activation and dispatch to the unwind handler.
 230   __ pop_frame();
 231   __ z_lg(Z_EXC_PC, _z_abi16(return_pc), Z_SP);
 232 
 233   // Z_EXC_OOP: exception oop
 234   // Z_EXC_PC: exception pc
 235 
 236   // Dispatch to the unwind logic.
 237   __ load_const_optimized(Z_R5, Runtime1::entry_for (Runtime1::unwind_exception_id));
 238   __ z_br(Z_R5);
 239 
 240   // Emit the slow path assembly.
 241   if (stub != NULL) {
 242     stub->emit_code(this);
 243   }
 244 
 245   return offset;
 246 }
 247 
 248 int LIR_Assembler::emit_deopt_handler() {
 249   // If the last instruction is a call (typically to do a throw which
 250   // is coming at the end after block reordering) the return address
 251   // must still point into the code area in order to avoid assertion
 252   // failures when searching for the corresponding bci. => Add a nop.
 253   // (was bug 5/14/1999 - gri)
 254   __ nop();
 255 
 256   // Generate code for exception handler.
 257   address handler_base = __ start_a_stub(deopt_handler_size());
 258   if (handler_base == NULL) {
 259     // Not enough space left for the handler.
 260     bailout("deopt handler overflow");
 261     return -1;
 262   }  int offset = code_offset();
 263   // Size must be constant (see HandlerImpl::emit_deopt_handler).
 264   __ load_const(Z_R1_scratch, SharedRuntime::deopt_blob()->unpack());
 265   __ call(Z_R1_scratch);
 266   guarantee(code_offset() - offset <= deopt_handler_size(), "overflow");
 267   __ end_a_stub();
 268 
 269   return offset;
 270 }
 271 
 272 void LIR_Assembler::jobject2reg(jobject o, Register reg) {
 273   if (o == NULL) {
 274     __ clear_reg(reg, true/*64bit*/, false/*set cc*/); // Must not kill cc set by cmove.
 275   } else {
 276     AddressLiteral a = __ allocate_oop_address(o);
 277     bool success = __ load_oop_from_toc(reg, a, reg);
 278     if (!success) {
 279       bailout("const section overflow");
 280     }
 281   }
 282 }
 283 
 284 void LIR_Assembler::jobject2reg_with_patching(Register reg, CodeEmitInfo *info) {
 285   // Allocate a new index in table to hold the object once it's been patched.
 286   int oop_index = __ oop_recorder()->allocate_oop_index(NULL);
 287   PatchingStub* patch = new PatchingStub(_masm, patching_id(info), oop_index);
 288 
 289   AddressLiteral addrlit((intptr_t)0, oop_Relocation::spec(oop_index));
 290   assert(addrlit.rspec().type() == relocInfo::oop_type, "must be an oop reloc");
 291   // The NULL will be dynamically patched later so the sequence to
 292   // load the address literal must not be optimized.
 293   __ load_const(reg, addrlit);
 294 
 295   patching_epilog(patch, lir_patch_normal, reg, info);
 296 }
 297 
 298 void LIR_Assembler::metadata2reg(Metadata* md, Register reg) {
 299   bool success = __ set_metadata_constant(md, reg);
 300   if (!success) {
 301     bailout("const section overflow");
 302     return;
 303   }
 304 }
 305 
 306 void LIR_Assembler::klass2reg_with_patching(Register reg, CodeEmitInfo *info) {
 307   // Allocate a new index in table to hold the klass once it's been patched.
 308   int index = __ oop_recorder()->allocate_metadata_index(NULL);
 309   PatchingStub* patch = new PatchingStub(_masm, PatchingStub::load_klass_id, index);
 310   AddressLiteral addrlit((intptr_t)0, metadata_Relocation::spec(index));
 311   assert(addrlit.rspec().type() == relocInfo::metadata_type, "must be an metadata reloc");
 312   // The NULL will be dynamically patched later so the sequence to
 313   // load the address literal must not be optimized.
 314   __ load_const(reg, addrlit);
 315 
 316   patching_epilog(patch, lir_patch_normal, reg, info);
 317 }
 318 
 319 void LIR_Assembler::emit_op3(LIR_Op3* op) {
 320   switch (op->code()) {
 321     case lir_idiv:
 322     case lir_irem:
 323       arithmetic_idiv(op->code(),
 324                       op->in_opr1(),
 325                       op->in_opr2(),
 326                       op->in_opr3(),
 327                       op->result_opr(),
 328                       op->info());
 329       break;
 330     case lir_fmad: {
 331       const FloatRegister opr1 = op->in_opr1()->as_double_reg(),
 332                           opr2 = op->in_opr2()->as_double_reg(),
 333                           opr3 = op->in_opr3()->as_double_reg(),
 334                           res  = op->result_opr()->as_double_reg();
 335       __ z_madbr(opr3, opr1, opr2);
 336       if (res != opr3) { __ z_ldr(res, opr3); }
 337     } break;
 338     case lir_fmaf: {
 339       const FloatRegister opr1 = op->in_opr1()->as_float_reg(),
 340                           opr2 = op->in_opr2()->as_float_reg(),
 341                           opr3 = op->in_opr3()->as_float_reg(),
 342                           res  = op->result_opr()->as_float_reg();
 343       __ z_maebr(opr3, opr1, opr2);
 344       if (res != opr3) { __ z_ler(res, opr3); }
 345     } break;
 346     default: ShouldNotReachHere(); break;
 347   }
 348 }
 349 
 350 
 351 void LIR_Assembler::emit_opBranch(LIR_OpBranch* op) {
 352 #ifdef ASSERT
 353   assert(op->block() == NULL || op->block()->label() == op->label(), "wrong label");
 354   if (op->block() != NULL)  { _branch_target_blocks.append(op->block()); }
 355   if (op->ublock() != NULL) { _branch_target_blocks.append(op->ublock()); }
 356 #endif
 357 
 358   if (op->cond() == lir_cond_always) {
 359     if (op->info() != NULL) { add_debug_info_for_branch(op->info()); }
 360     __ branch_optimized(Assembler::bcondAlways, *(op->label()));
 361   } else {
 362     Assembler::branch_condition acond = Assembler::bcondZero;
 363     if (op->code() == lir_cond_float_branch) {
 364       assert(op->ublock() != NULL, "must have unordered successor");
 365       __ branch_optimized(Assembler::bcondNotOrdered, *(op->ublock()->label()));
 366     }
 367     switch (op->cond()) {
 368       case lir_cond_equal:        acond = Assembler::bcondEqual;     break;
 369       case lir_cond_notEqual:     acond = Assembler::bcondNotEqual;  break;
 370       case lir_cond_less:         acond = Assembler::bcondLow;       break;
 371       case lir_cond_lessEqual:    acond = Assembler::bcondNotHigh;   break;
 372       case lir_cond_greaterEqual: acond = Assembler::bcondNotLow;    break;
 373       case lir_cond_greater:      acond = Assembler::bcondHigh;      break;
 374       case lir_cond_belowEqual:   acond = Assembler::bcondNotHigh;   break;
 375       case lir_cond_aboveEqual:   acond = Assembler::bcondNotLow;    break;
 376       default:                         ShouldNotReachHere();
 377     }
 378     __ branch_optimized(acond,*(op->label()));
 379   }
 380 }
 381 
 382 
 383 void LIR_Assembler::emit_opConvert(LIR_OpConvert* op) {
 384   LIR_Opr src  = op->in_opr();
 385   LIR_Opr dest = op->result_opr();
 386 
 387   switch (op->bytecode()) {
 388     case Bytecodes::_i2l:
 389       __ move_reg_if_needed(dest->as_register_lo(), T_LONG, src->as_register(), T_INT);
 390       break;
 391 
 392     case Bytecodes::_l2i:
 393       __ move_reg_if_needed(dest->as_register(), T_INT, src->as_register_lo(), T_LONG);
 394       break;
 395 
 396     case Bytecodes::_i2b:
 397       __ move_reg_if_needed(dest->as_register(), T_BYTE, src->as_register(), T_INT);
 398       break;
 399 
 400     case Bytecodes::_i2c:
 401       __ move_reg_if_needed(dest->as_register(), T_CHAR, src->as_register(), T_INT);
 402       break;
 403 
 404     case Bytecodes::_i2s:
 405       __ move_reg_if_needed(dest->as_register(), T_SHORT, src->as_register(), T_INT);
 406       break;
 407 
 408     case Bytecodes::_f2d:
 409       assert(dest->is_double_fpu(), "check");
 410       __ move_freg_if_needed(dest->as_double_reg(), T_DOUBLE, src->as_float_reg(), T_FLOAT);
 411       break;
 412 
 413     case Bytecodes::_d2f:
 414       assert(dest->is_single_fpu(), "check");
 415       __ move_freg_if_needed(dest->as_float_reg(), T_FLOAT, src->as_double_reg(), T_DOUBLE);
 416       break;
 417 
 418     case Bytecodes::_i2f:
 419       __ z_cefbr(dest->as_float_reg(), src->as_register());
 420       break;
 421 
 422     case Bytecodes::_i2d:
 423       __ z_cdfbr(dest->as_double_reg(), src->as_register());
 424       break;
 425 
 426     case Bytecodes::_l2f:
 427       __ z_cegbr(dest->as_float_reg(), src->as_register_lo());
 428       break;
 429     case Bytecodes::_l2d:
 430       __ z_cdgbr(dest->as_double_reg(), src->as_register_lo());
 431       break;
 432 
 433     case Bytecodes::_f2i:
 434     case Bytecodes::_f2l: {
 435       Label done;
 436       FloatRegister Rsrc = src->as_float_reg();
 437       Register Rdst = (op->bytecode() == Bytecodes::_f2i ? dest->as_register() : dest->as_register_lo());
 438       __ clear_reg(Rdst, true, false);
 439       __ z_cebr(Rsrc, Rsrc);
 440       __ z_brno(done); // NaN -> 0
 441       if (op->bytecode() == Bytecodes::_f2i) {
 442         __ z_cfebr(Rdst, Rsrc, Assembler::to_zero);
 443       } else { // op->bytecode() == Bytecodes::_f2l
 444         __ z_cgebr(Rdst, Rsrc, Assembler::to_zero);
 445       }
 446       __ bind(done);
 447     }
 448     break;
 449 
 450     case Bytecodes::_d2i:
 451     case Bytecodes::_d2l: {
 452       Label done;
 453       FloatRegister Rsrc = src->as_double_reg();
 454       Register Rdst = (op->bytecode() == Bytecodes::_d2i ? dest->as_register() : dest->as_register_lo());
 455       __ clear_reg(Rdst, true, false);  // Don't set CC.
 456       __ z_cdbr(Rsrc, Rsrc);
 457       __ z_brno(done); // NaN -> 0
 458       if (op->bytecode() == Bytecodes::_d2i) {
 459         __ z_cfdbr(Rdst, Rsrc, Assembler::to_zero);
 460       } else { // Bytecodes::_d2l
 461         __ z_cgdbr(Rdst, Rsrc, Assembler::to_zero);
 462       }
 463       __ bind(done);
 464     }
 465     break;
 466 
 467     default: ShouldNotReachHere();
 468   }
 469 }
 470 
 471 void LIR_Assembler::align_call(LIR_Code code) {
 472   // End of call instruction must be 4 byte aligned.
 473   int offset = __ offset();
 474   switch (code) {
 475     case lir_icvirtual_call:
 476       offset += MacroAssembler::load_const_from_toc_size();
 477       // no break
 478     case lir_static_call:
 479     case lir_optvirtual_call:
 480     case lir_dynamic_call:
 481       offset += NativeCall::call_far_pcrelative_displacement_offset;
 482       break;
 483     case lir_virtual_call:   // currently, sparc-specific for niagara
 484     default: ShouldNotReachHere();
 485   }
 486   if ((offset & (NativeCall::call_far_pcrelative_displacement_alignment-1)) != 0) {
 487     __ nop();
 488   }
 489 }
 490 
 491 void LIR_Assembler::call(LIR_OpJavaCall* op, relocInfo::relocType rtype) {
 492   assert((__ offset() + NativeCall::call_far_pcrelative_displacement_offset) % NativeCall::call_far_pcrelative_displacement_alignment == 0,
 493          "must be aligned (offset=%d)", __ offset());
 494   assert(rtype == relocInfo::none ||
 495          rtype == relocInfo::opt_virtual_call_type ||
 496          rtype == relocInfo::static_call_type, "unexpected rtype");
 497   // Prepend each BRASL with a nop.
 498   __ relocate(rtype);
 499   __ z_nop();
 500   __ z_brasl(Z_R14, op->addr());
 501   add_call_info(code_offset(), op->info());
 502 }
 503 
 504 void LIR_Assembler::ic_call(LIR_OpJavaCall* op) {
 505   address virtual_call_oop_addr = NULL;
 506   AddressLiteral empty_ic((address) Universe::non_oop_word());
 507   virtual_call_oop_addr = __ pc();
 508   bool success = __ load_const_from_toc(Z_inline_cache, empty_ic);
 509   if (!success) {
 510     bailout("const section overflow");
 511     return;
 512   }
 513 
 514   // CALL to fixup routine. Fixup routine uses ScopeDesc info
 515   // to determine who we intended to call.
 516   __ relocate(virtual_call_Relocation::spec(virtual_call_oop_addr));
 517   call(op, relocInfo::none);
 518 }
 519 
 520 // not supported
 521 void LIR_Assembler::vtable_call(LIR_OpJavaCall* op) {
 522   ShouldNotReachHere();
 523 }
 524 
 525 void LIR_Assembler::move_regs(Register from_reg, Register to_reg) {
 526   if (from_reg != to_reg) __ z_lgr(to_reg, from_reg);
 527 }
 528 
 529 void LIR_Assembler::const2stack(LIR_Opr src, LIR_Opr dest) {
 530   assert(src->is_constant(), "should not call otherwise");
 531   assert(dest->is_stack(), "should not call otherwise");
 532   LIR_Const* c = src->as_constant_ptr();
 533 
 534   unsigned int lmem = 0;
 535   unsigned int lcon = 0;
 536   int64_t cbits = 0;
 537   Address dest_addr;
 538   switch (c->type()) {
 539     case T_INT:  // fall through
 540     case T_FLOAT:
 541       dest_addr = frame_map()->address_for_slot(dest->single_stack_ix());
 542       lmem = 4; lcon = 4; cbits = c->as_jint_bits();
 543       break;
 544 
 545     case T_ADDRESS:
 546       dest_addr = frame_map()->address_for_slot(dest->single_stack_ix());
 547       lmem = 8; lcon = 4; cbits = c->as_jint_bits();
 548       break;
 549 
 550     case T_OBJECT:
 551       dest_addr = frame_map()->address_for_slot(dest->single_stack_ix());
 552       if (c->as_jobject() == NULL) {
 553         __ store_const(dest_addr, (int64_t)NULL_WORD, 8, 8);
 554       } else {
 555         jobject2reg(c->as_jobject(), Z_R1_scratch);
 556         __ reg2mem_opt(Z_R1_scratch, dest_addr, true);
 557       }
 558       return;
 559 
 560     case T_LONG:  // fall through
 561     case T_DOUBLE:
 562       dest_addr = frame_map()->address_for_slot(dest->double_stack_ix());
 563       lmem = 8; lcon = 8; cbits = (int64_t)(c->as_jlong_bits());
 564       break;
 565 
 566     default:
 567       ShouldNotReachHere();
 568   }
 569 
 570   __ store_const(dest_addr, cbits, lmem, lcon);
 571 }
 572 
 573 void LIR_Assembler::const2mem(LIR_Opr src, LIR_Opr dest, BasicType type, CodeEmitInfo* info, bool wide) {
 574   assert(src->is_constant(), "should not call otherwise");
 575   assert(dest->is_address(), "should not call otherwise");
 576 
 577   LIR_Const* c = src->as_constant_ptr();
 578   Address addr = as_Address(dest->as_address_ptr());
 579 
 580   int store_offset = -1;
 581 
 582   if (dest->as_address_ptr()->index()->is_valid()) {
 583     switch (type) {
 584       case T_INT:    // fall through
 585       case T_FLOAT:
 586         __ load_const_optimized(Z_R0_scratch, c->as_jint_bits());
 587         store_offset = __ offset();
 588         if (Immediate::is_uimm12(addr.disp())) {
 589           __ z_st(Z_R0_scratch, addr);
 590         } else {
 591           __ z_sty(Z_R0_scratch, addr);
 592         }
 593         break;
 594 
 595       case T_ADDRESS:
 596         __ load_const_optimized(Z_R1_scratch, c->as_jint_bits());
 597         store_offset = __ reg2mem_opt(Z_R1_scratch, addr, true);
 598         break;
 599 
 600       case T_OBJECT:  // fall through
 601       case T_ARRAY:
 602         if (c->as_jobject() == NULL) {
 603           if (UseCompressedOops && !wide) {
 604             __ clear_reg(Z_R1_scratch, false);
 605             store_offset = __ reg2mem_opt(Z_R1_scratch, addr, false);
 606           } else {
 607             __ clear_reg(Z_R1_scratch, true);
 608             store_offset = __ reg2mem_opt(Z_R1_scratch, addr, true);
 609           }
 610         } else {
 611           jobject2reg(c->as_jobject(), Z_R1_scratch);
 612           if (UseCompressedOops && !wide) {
 613             __ encode_heap_oop(Z_R1_scratch);
 614             store_offset = __ reg2mem_opt(Z_R1_scratch, addr, false);
 615           } else {
 616             store_offset = __ reg2mem_opt(Z_R1_scratch, addr, true);
 617           }
 618         }
 619         assert(store_offset >= 0, "check");
 620         break;
 621 
 622       case T_LONG:    // fall through
 623       case T_DOUBLE:
 624         __ load_const_optimized(Z_R1_scratch, (int64_t)(c->as_jlong_bits()));
 625         store_offset = __ reg2mem_opt(Z_R1_scratch, addr, true);
 626         break;
 627 
 628       case T_BOOLEAN: // fall through
 629       case T_BYTE:
 630         __ load_const_optimized(Z_R0_scratch, (int8_t)(c->as_jint()));
 631         store_offset = __ offset();
 632         if (Immediate::is_uimm12(addr.disp())) {
 633           __ z_stc(Z_R0_scratch, addr);
 634         } else {
 635           __ z_stcy(Z_R0_scratch, addr);
 636         }
 637         break;
 638 
 639       case T_CHAR:    // fall through
 640       case T_SHORT:
 641         __ load_const_optimized(Z_R0_scratch, (int16_t)(c->as_jint()));
 642         store_offset = __ offset();
 643         if (Immediate::is_uimm12(addr.disp())) {
 644           __ z_sth(Z_R0_scratch, addr);
 645         } else {
 646           __ z_sthy(Z_R0_scratch, addr);
 647         }
 648         break;
 649 
 650       default:
 651         ShouldNotReachHere();
 652     }
 653 
 654   } else { // no index
 655 
 656     unsigned int lmem = 0;
 657     unsigned int lcon = 0;
 658     int64_t cbits = 0;
 659 
 660     switch (type) {
 661       case T_INT:    // fall through
 662       case T_FLOAT:
 663         lmem = 4; lcon = 4; cbits = c->as_jint_bits();
 664         break;
 665 
 666       case T_ADDRESS:
 667         lmem = 8; lcon = 4; cbits = c->as_jint_bits();
 668         break;
 669 
 670       case T_OBJECT:  // fall through
 671       case T_ARRAY:
 672         if (c->as_jobject() == NULL) {
 673           if (UseCompressedOops && !wide) {
 674             store_offset = __ store_const(addr, (int32_t)NULL_WORD, 4, 4);
 675           } else {
 676             store_offset = __ store_const(addr, (int64_t)NULL_WORD, 8, 8);
 677           }
 678         } else {
 679           jobject2reg(c->as_jobject(), Z_R1_scratch);
 680           if (UseCompressedOops && !wide) {
 681             __ encode_heap_oop(Z_R1_scratch);
 682             store_offset = __ reg2mem_opt(Z_R1_scratch, addr, false);
 683           } else {
 684             store_offset = __ reg2mem_opt(Z_R1_scratch, addr, true);
 685           }
 686         }
 687         assert(store_offset >= 0, "check");
 688         break;
 689 
 690       case T_LONG:    // fall through
 691       case T_DOUBLE:
 692         lmem = 8; lcon = 8; cbits = (int64_t)(c->as_jlong_bits());
 693         break;
 694 
 695       case T_BOOLEAN: // fall through
 696       case T_BYTE:
 697         lmem = 1; lcon = 1; cbits = (int8_t)(c->as_jint());
 698         break;
 699 
 700       case T_CHAR:    // fall through
 701       case T_SHORT:
 702         lmem = 2; lcon = 2; cbits = (int16_t)(c->as_jint());
 703         break;
 704 
 705       default:
 706         ShouldNotReachHere();
 707     }
 708 
 709     if (store_offset == -1) {
 710       store_offset = __ store_const(addr, cbits, lmem, lcon);
 711       assert(store_offset >= 0, "check");
 712     }
 713   }
 714 
 715   if (info != NULL) {
 716     add_debug_info_for_null_check(store_offset, info);
 717   }
 718 }
 719 
 720 void LIR_Assembler::const2reg(LIR_Opr src, LIR_Opr dest, LIR_PatchCode patch_code, CodeEmitInfo* info) {
 721   assert(src->is_constant(), "should not call otherwise");
 722   assert(dest->is_register(), "should not call otherwise");
 723   LIR_Const* c = src->as_constant_ptr();
 724 
 725   switch (c->type()) {
 726     case T_INT: {
 727       assert(patch_code == lir_patch_none, "no patching handled here");
 728       __ load_const_optimized(dest->as_register(), c->as_jint());
 729       break;
 730     }
 731 
 732     case T_ADDRESS: {
 733       assert(patch_code == lir_patch_none, "no patching handled here");
 734       __ load_const_optimized(dest->as_register(), c->as_jint());
 735       break;
 736     }
 737 
 738     case T_LONG: {
 739       assert(patch_code == lir_patch_none, "no patching handled here");
 740       __ load_const_optimized(dest->as_register_lo(), (intptr_t)c->as_jlong());
 741       break;
 742     }
 743 
 744     case T_OBJECT: {
 745       if (patch_code != lir_patch_none) {
 746         jobject2reg_with_patching(dest->as_register(), info);
 747       } else {
 748         jobject2reg(c->as_jobject(), dest->as_register());
 749       }
 750       break;
 751     }
 752 
 753     case T_METADATA: {
 754       if (patch_code != lir_patch_none) {
 755         klass2reg_with_patching(dest->as_register(), info);
 756       } else {
 757         metadata2reg(c->as_metadata(), dest->as_register());
 758       }
 759       break;
 760     }
 761 
 762     case T_FLOAT: {
 763       Register toc_reg = Z_R1_scratch;
 764       __ load_toc(toc_reg);
 765       address const_addr = __ float_constant(c->as_jfloat());
 766       if (const_addr == NULL) {
 767         bailout("const section overflow");
 768         break;
 769       }
 770       int displ = const_addr - _masm->code()->consts()->start();
 771       if (dest->is_single_fpu()) {
 772         __ z_ley(dest->as_float_reg(), displ, toc_reg);
 773       } else {
 774         assert(dest->is_single_cpu(), "Must be a cpu register.");
 775         __ z_ly(dest->as_register(), displ, toc_reg);
 776       }
 777     }
 778     break;
 779 
 780     case T_DOUBLE: {
 781       Register toc_reg = Z_R1_scratch;
 782       __ load_toc(toc_reg);
 783       address const_addr = __ double_constant(c->as_jdouble());
 784       if (const_addr == NULL) {
 785         bailout("const section overflow");
 786         break;
 787       }
 788       int displ = const_addr - _masm->code()->consts()->start();
 789       if (dest->is_double_fpu()) {
 790         __ z_ldy(dest->as_double_reg(), displ, toc_reg);
 791       } else {
 792         assert(dest->is_double_cpu(), "Must be a long register.");
 793         __ z_lg(dest->as_register_lo(), displ, toc_reg);
 794       }
 795     }
 796     break;
 797 
 798     default:
 799       ShouldNotReachHere();
 800   }
 801 }
 802 
 803 Address LIR_Assembler::as_Address(LIR_Address* addr) {
 804   if (addr->base()->is_illegal()) {
 805     Unimplemented();
 806   }
 807 
 808   Register base = addr->base()->as_pointer_register();
 809 
 810   if (addr->index()->is_illegal()) {
 811     return Address(base, addr->disp());
 812   } else if (addr->index()->is_cpu_register()) {
 813     Register index = addr->index()->as_pointer_register();
 814     return Address(base, index, addr->disp());
 815   } else if (addr->index()->is_constant()) {
 816     intptr_t addr_offset = addr->index()->as_constant_ptr()->as_jint() + addr->disp();
 817     return Address(base, addr_offset);
 818   } else {
 819     ShouldNotReachHere();
 820     return Address();
 821   }
 822 }
 823 
 824 void LIR_Assembler::stack2stack(LIR_Opr src, LIR_Opr dest, BasicType type) {
 825   switch (type) {
 826     case T_INT:
 827     case T_FLOAT: {
 828       Register tmp = Z_R1_scratch;
 829       Address from = frame_map()->address_for_slot(src->single_stack_ix());
 830       Address to   = frame_map()->address_for_slot(dest->single_stack_ix());
 831       __ mem2reg_opt(tmp, from, false);
 832       __ reg2mem_opt(tmp, to, false);
 833       break;
 834     }
 835     case T_ADDRESS:
 836     case T_OBJECT: {
 837       Register tmp = Z_R1_scratch;
 838       Address from = frame_map()->address_for_slot(src->single_stack_ix());
 839       Address to   = frame_map()->address_for_slot(dest->single_stack_ix());
 840       __ mem2reg_opt(tmp, from, true);
 841       __ reg2mem_opt(tmp, to, true);
 842       break;
 843     }
 844     case T_LONG:
 845     case T_DOUBLE: {
 846       Register tmp = Z_R1_scratch;
 847       Address from = frame_map()->address_for_double_slot(src->double_stack_ix());
 848       Address to   = frame_map()->address_for_double_slot(dest->double_stack_ix());
 849       __ mem2reg_opt(tmp, from, true);
 850       __ reg2mem_opt(tmp, to, true);
 851       break;
 852     }
 853 
 854     default:
 855       ShouldNotReachHere();
 856   }
 857 }
 858 
 859 // 4-byte accesses only! Don't use it to access 8 bytes!
 860 Address LIR_Assembler::as_Address_hi(LIR_Address* addr) {
 861   ShouldNotCallThis();
 862   return 0; // unused
 863 }
 864 
 865 // 4-byte accesses only! Don't use it to access 8 bytes!
 866 Address LIR_Assembler::as_Address_lo(LIR_Address* addr) {
 867   ShouldNotCallThis();
 868   return 0; // unused
 869 }
 870 
 871 void LIR_Assembler::mem2reg(LIR_Opr src_opr, LIR_Opr dest, BasicType type, LIR_PatchCode patch_code,
 872                             CodeEmitInfo* info, bool wide, bool unaligned) {
 873 
 874   assert(type != T_METADATA, "load of metadata ptr not supported");
 875   LIR_Address* addr = src_opr->as_address_ptr();
 876   LIR_Opr to_reg = dest;
 877 
 878   Register src = addr->base()->as_pointer_register();
 879   Register disp_reg = Z_R0;
 880   int disp_value = addr->disp();
 881   bool needs_patching = (patch_code != lir_patch_none);
 882 
 883   if (addr->base()->type() == T_OBJECT) {
 884     __ verify_oop(src);
 885   }
 886 
 887   PatchingStub* patch = NULL;
 888   if (needs_patching) {
 889     patch = new PatchingStub(_masm, PatchingStub::access_field_id);
 890     assert(!to_reg->is_double_cpu() ||
 891            patch_code == lir_patch_none ||
 892            patch_code == lir_patch_normal, "patching doesn't match register");
 893   }
 894 
 895   if (addr->index()->is_illegal()) {
 896     if (!Immediate::is_simm20(disp_value)) {
 897       if (needs_patching) {
 898         __ load_const(Z_R1_scratch, (intptr_t)0);
 899       } else {
 900         __ load_const_optimized(Z_R1_scratch, disp_value);
 901       }
 902       disp_reg = Z_R1_scratch;
 903       disp_value = 0;
 904     }
 905   } else {
 906     if (!Immediate::is_simm20(disp_value)) {
 907       __ load_const_optimized(Z_R1_scratch, disp_value);
 908       __ z_la(Z_R1_scratch, 0, Z_R1_scratch, addr->index()->as_register());
 909       disp_reg = Z_R1_scratch;
 910       disp_value = 0;
 911     }
 912     disp_reg = addr->index()->as_pointer_register();
 913   }
 914 
 915   // Remember the offset of the load. The patching_epilog must be done
 916   // before the call to add_debug_info, otherwise the PcDescs don't get
 917   // entered in increasing order.
 918   int offset = code_offset();
 919 
 920   assert(disp_reg != Z_R0 || Immediate::is_simm20(disp_value), "should have set this up");
 921 
 922   bool short_disp = Immediate::is_uimm12(disp_value);
 923 
 924   switch (type) {
 925     case T_BOOLEAN: // fall through
 926     case T_BYTE  :  __ z_lb(dest->as_register(),   disp_value, disp_reg, src); break;
 927     case T_CHAR  :  __ z_llgh(dest->as_register(), disp_value, disp_reg, src); break;
 928     case T_SHORT :
 929       if (short_disp) {
 930                     __ z_lh(dest->as_register(),   disp_value, disp_reg, src);
 931       } else {
 932                     __ z_lhy(dest->as_register(),  disp_value, disp_reg, src);
 933       }
 934       break;
 935     case T_INT   :
 936       if (short_disp) {
 937                     __ z_l(dest->as_register(),    disp_value, disp_reg, src);
 938       } else {
 939                     __ z_ly(dest->as_register(),   disp_value, disp_reg, src);
 940       }
 941       break;
 942     case T_ADDRESS:
 943       if (UseCompressedClassPointers && addr->disp() == oopDesc::klass_offset_in_bytes()) {
 944         __ z_llgf(dest->as_register(), disp_value, disp_reg, src);
 945         __ decode_klass_not_null(dest->as_register());
 946       } else {
 947         __ z_lg(dest->as_register(), disp_value, disp_reg, src);
 948       }
 949       break;
 950     case T_ARRAY : // fall through
 951     case T_OBJECT:
 952     {
 953       if (UseCompressedOops && !wide) {
 954         __ z_llgf(dest->as_register(), disp_value, disp_reg, src);
 955         __ oop_decoder(dest->as_register(), dest->as_register(), true);
 956       } else {
 957         __ z_lg(dest->as_register(), disp_value, disp_reg, src);
 958       }
 959       break;
 960     }
 961     case T_FLOAT:
 962       if (short_disp) {
 963                     __ z_le(dest->as_float_reg(),  disp_value, disp_reg, src);
 964       } else {
 965                     __ z_ley(dest->as_float_reg(), disp_value, disp_reg, src);
 966       }
 967       break;
 968     case T_DOUBLE:
 969       if (short_disp) {
 970                     __ z_ld(dest->as_double_reg(),  disp_value, disp_reg, src);
 971       } else {
 972                     __ z_ldy(dest->as_double_reg(), disp_value, disp_reg, src);
 973       }
 974       break;
 975     case T_LONG  :  __ z_lg(dest->as_register_lo(), disp_value, disp_reg, src); break;
 976     default      : ShouldNotReachHere();
 977   }
 978   if (type == T_ARRAY || type == T_OBJECT) {
 979     __ verify_oop(dest->as_register());
 980   }
 981 
 982   if (patch != NULL) {
 983     patching_epilog(patch, patch_code, src, info);
 984   }
 985   if (info != NULL) add_debug_info_for_null_check(offset, info);
 986 }
 987 
 988 void LIR_Assembler::stack2reg(LIR_Opr src, LIR_Opr dest, BasicType type) {
 989   assert(src->is_stack(), "should not call otherwise");
 990   assert(dest->is_register(), "should not call otherwise");
 991 
 992   if (dest->is_single_cpu()) {
 993     if (type == T_ARRAY || type == T_OBJECT) {
 994       __ mem2reg_opt(dest->as_register(), frame_map()->address_for_slot(src->single_stack_ix()), true);
 995       __ verify_oop(dest->as_register());
 996     } else if (type == T_METADATA) {
 997       __ mem2reg_opt(dest->as_register(), frame_map()->address_for_slot(src->single_stack_ix()), true);
 998     } else {
 999       __ mem2reg_opt(dest->as_register(), frame_map()->address_for_slot(src->single_stack_ix()), false);
1000     }
1001   } else if (dest->is_double_cpu()) {
1002     Address src_addr_LO = frame_map()->address_for_slot(src->double_stack_ix());
1003     __ mem2reg_opt(dest->as_register_lo(), src_addr_LO, true);
1004   } else if (dest->is_single_fpu()) {
1005     Address src_addr = frame_map()->address_for_slot(src->single_stack_ix());
1006     __ mem2freg_opt(dest->as_float_reg(), src_addr, false);
1007   } else if (dest->is_double_fpu()) {
1008     Address src_addr = frame_map()->address_for_slot(src->double_stack_ix());
1009     __ mem2freg_opt(dest->as_double_reg(), src_addr, true);
1010   } else {
1011     ShouldNotReachHere();
1012   }
1013 }
1014 
1015 void LIR_Assembler::reg2stack(LIR_Opr src, LIR_Opr dest, BasicType type, bool pop_fpu_stack) {
1016   assert(src->is_register(), "should not call otherwise");
1017   assert(dest->is_stack(), "should not call otherwise");
1018 
1019   if (src->is_single_cpu()) {
1020     const Address dst = frame_map()->address_for_slot(dest->single_stack_ix());
1021     if (type == T_OBJECT || type == T_ARRAY) {
1022       __ verify_oop(src->as_register());
1023       __ reg2mem_opt(src->as_register(), dst, true);
1024     } else if (type == T_METADATA) {
1025       __ reg2mem_opt(src->as_register(), dst, true);
1026     } else {
1027       __ reg2mem_opt(src->as_register(), dst, false);
1028     }
1029   } else if (src->is_double_cpu()) {
1030     Address dstLO = frame_map()->address_for_slot(dest->double_stack_ix());
1031     __ reg2mem_opt(src->as_register_lo(), dstLO, true);
1032   } else if (src->is_single_fpu()) {
1033     Address dst_addr = frame_map()->address_for_slot(dest->single_stack_ix());
1034     __ freg2mem_opt(src->as_float_reg(), dst_addr, false);
1035   } else if (src->is_double_fpu()) {
1036     Address dst_addr = frame_map()->address_for_slot(dest->double_stack_ix());
1037     __ freg2mem_opt(src->as_double_reg(), dst_addr, true);
1038   } else {
1039     ShouldNotReachHere();
1040   }
1041 }
1042 
1043 void LIR_Assembler::reg2reg(LIR_Opr from_reg, LIR_Opr to_reg) {
1044   if (from_reg->is_float_kind() && to_reg->is_float_kind()) {
1045     if (from_reg->is_double_fpu()) {
1046       // double to double moves
1047       assert(to_reg->is_double_fpu(), "should match");
1048       __ z_ldr(to_reg->as_double_reg(), from_reg->as_double_reg());
1049     } else {
1050       // float to float moves
1051       assert(to_reg->is_single_fpu(), "should match");
1052       __ z_ler(to_reg->as_float_reg(), from_reg->as_float_reg());
1053     }
1054   } else if (!from_reg->is_float_kind() && !to_reg->is_float_kind()) {
1055     if (from_reg->is_double_cpu()) {
1056       __ z_lgr(to_reg->as_pointer_register(), from_reg->as_pointer_register());
1057     } else if (to_reg->is_double_cpu()) {
1058       // int to int moves
1059       __ z_lgr(to_reg->as_register_lo(), from_reg->as_register());
1060     } else {
1061       // int to int moves
1062       __ z_lgr(to_reg->as_register(), from_reg->as_register());
1063     }
1064   } else {
1065     ShouldNotReachHere();
1066   }
1067   if (to_reg->type() == T_OBJECT || to_reg->type() == T_ARRAY) {
1068     __ verify_oop(to_reg->as_register());
1069   }
1070 }
1071 
1072 void LIR_Assembler::reg2mem(LIR_Opr from, LIR_Opr dest_opr, BasicType type,
1073                             LIR_PatchCode patch_code, CodeEmitInfo* info, bool pop_fpu_stack,
1074                             bool wide, bool unaligned) {
1075   assert(type != T_METADATA, "store of metadata ptr not supported");
1076   LIR_Address* addr = dest_opr->as_address_ptr();
1077 
1078   Register dest = addr->base()->as_pointer_register();
1079   Register disp_reg = Z_R0;
1080   int disp_value = addr->disp();
1081   bool needs_patching = (patch_code != lir_patch_none);
1082 
1083   if (addr->base()->is_oop_register()) {
1084     __ verify_oop(dest);
1085   }
1086 
1087   PatchingStub* patch = NULL;
1088   if (needs_patching) {
1089     patch = new PatchingStub(_masm, PatchingStub::access_field_id);
1090     assert(!from->is_double_cpu() ||
1091            patch_code == lir_patch_none ||
1092            patch_code == lir_patch_normal, "patching doesn't match register");
1093   }
1094 
1095   assert(!needs_patching || (!Immediate::is_simm20(disp_value) && addr->index()->is_illegal()), "assumption");
1096   if (addr->index()->is_illegal()) {
1097     if (!Immediate::is_simm20(disp_value)) {
1098       if (needs_patching) {
1099         __ load_const(Z_R1_scratch, (intptr_t)0);
1100       } else {
1101         __ load_const_optimized(Z_R1_scratch, disp_value);
1102       }
1103       disp_reg = Z_R1_scratch;
1104       disp_value = 0;
1105     }
1106   } else {
1107     if (!Immediate::is_simm20(disp_value)) {
1108       __ load_const_optimized(Z_R1_scratch, disp_value);
1109       __ z_la(Z_R1_scratch, 0, Z_R1_scratch, addr->index()->as_register());
1110       disp_reg = Z_R1_scratch;
1111       disp_value = 0;
1112     }
1113     disp_reg = addr->index()->as_pointer_register();
1114   }
1115 
1116   assert(disp_reg != Z_R0 || Immediate::is_simm20(disp_value), "should have set this up");
1117 
1118   if (type == T_ARRAY || type == T_OBJECT) {
1119     __ verify_oop(from->as_register());
1120   }
1121 
1122   bool short_disp = Immediate::is_uimm12(disp_value);
1123 
1124   // Remember the offset of the store. The patching_epilog must be done
1125   // before the call to add_debug_info_for_null_check, otherwise the PcDescs don't get
1126   // entered in increasing order.
1127   int offset = code_offset();
1128   switch (type) {
1129     case T_BOOLEAN: // fall through
1130     case T_BYTE  :
1131       if (short_disp) {
1132                     __ z_stc(from->as_register(),  disp_value, disp_reg, dest);
1133       } else {
1134                     __ z_stcy(from->as_register(), disp_value, disp_reg, dest);
1135       }
1136       break;
1137     case T_CHAR  : // fall through
1138     case T_SHORT :
1139       if (short_disp) {
1140                     __ z_sth(from->as_register(),  disp_value, disp_reg, dest);
1141       } else {
1142                     __ z_sthy(from->as_register(), disp_value, disp_reg, dest);
1143       }
1144       break;
1145     case T_INT   :
1146       if (short_disp) {
1147                     __ z_st(from->as_register(),  disp_value, disp_reg, dest);
1148       } else {
1149                     __ z_sty(from->as_register(), disp_value, disp_reg, dest);
1150       }
1151       break;
1152     case T_LONG  :  __ z_stg(from->as_register_lo(), disp_value, disp_reg, dest); break;
1153     case T_ADDRESS: __ z_stg(from->as_register(),    disp_value, disp_reg, dest); break;
1154       break;
1155     case T_ARRAY : // fall through
1156     case T_OBJECT:
1157       {
1158         if (UseCompressedOops && !wide) {
1159           Register compressed_src = Z_R14;
1160           __ oop_encoder(compressed_src, from->as_register(), true, (disp_reg != Z_R1) ? Z_R1 : Z_R0, -1, true);
1161           offset = code_offset();
1162           if (short_disp) {
1163             __ z_st(compressed_src,  disp_value, disp_reg, dest);
1164           } else {
1165             __ z_sty(compressed_src, disp_value, disp_reg, dest);
1166           }
1167         } else {
1168           __ z_stg(from->as_register(), disp_value, disp_reg, dest);
1169         }
1170         break;
1171       }
1172     case T_FLOAT :
1173       if (short_disp) {
1174         __ z_ste(from->as_float_reg(),  disp_value, disp_reg, dest);
1175       } else {
1176         __ z_stey(from->as_float_reg(), disp_value, disp_reg, dest);
1177       }
1178       break;
1179     case T_DOUBLE:
1180       if (short_disp) {
1181         __ z_std(from->as_double_reg(),  disp_value, disp_reg, dest);
1182       } else {
1183         __ z_stdy(from->as_double_reg(), disp_value, disp_reg, dest);
1184       }
1185       break;
1186     default: ShouldNotReachHere();
1187   }
1188 
1189   if (patch != NULL) {
1190     patching_epilog(patch, patch_code, dest, info);
1191   }
1192 
1193   if (info != NULL) add_debug_info_for_null_check(offset, info);
1194 }
1195 
1196 
1197 void LIR_Assembler::return_op(LIR_Opr result) {
1198   assert(result->is_illegal() ||
1199          (result->is_single_cpu() && result->as_register() == Z_R2) ||
1200          (result->is_double_cpu() && result->as_register_lo() == Z_R2) ||
1201          (result->is_single_fpu() && result->as_float_reg() == Z_F0) ||
1202          (result->is_double_fpu() && result->as_double_reg() == Z_F0), "convention");
1203 
1204   if (SafepointMechanism::uses_thread_local_poll()) {
1205     __ z_lg(Z_R1_scratch, Address(Z_thread, Thread::polling_page_offset()));
1206   } else {
1207     AddressLiteral pp(os::get_polling_page());
1208     __ load_const_optimized(Z_R1_scratch, pp);
1209   }
1210 
1211   // Pop the frame before the safepoint code.
1212   __ pop_frame_restore_retPC(initial_frame_size_in_bytes());
1213 
1214   if (StackReservedPages > 0 && compilation()->has_reserved_stack_access()) {
1215     __ reserved_stack_check(Z_R14);
1216   }
1217 
1218   // We need to mark the code position where the load from the safepoint
1219   // polling page was emitted as relocInfo::poll_return_type here.
1220   __ relocate(relocInfo::poll_return_type);
1221   __ load_from_polling_page(Z_R1_scratch);
1222 
1223   __ z_br(Z_R14); // Return to caller.
1224 }
1225 
1226 int LIR_Assembler::safepoint_poll(LIR_Opr tmp, CodeEmitInfo* info) {
1227   const Register poll_addr = tmp->as_register_lo();
1228   if (SafepointMechanism::uses_thread_local_poll()) {
1229     __ z_lg(poll_addr, Address(Z_thread, Thread::polling_page_offset()));
1230   } else {
1231     AddressLiteral pp(os::get_polling_page());
1232     __ load_const_optimized(poll_addr, pp);
1233   }
1234   guarantee(info != NULL, "Shouldn't be NULL");
1235   add_debug_info_for_branch(info);
1236   int offset = __ offset();
1237   __ relocate(relocInfo::poll_type);
1238   __ load_from_polling_page(poll_addr);
1239   return offset;
1240 }
1241 
1242 void LIR_Assembler::emit_static_call_stub() {
1243 
1244   // Stub is fixed up when the corresponding call is converted from calling
1245   // compiled code to calling interpreted code.
1246 
1247   address call_pc = __ pc();
1248   address stub = __ start_a_stub(call_stub_size());
1249   if (stub == NULL) {
1250     bailout("static call stub overflow");
1251     return;
1252   }
1253 
1254   int start = __ offset();
1255 
1256   __ relocate(static_stub_Relocation::spec(call_pc));
1257 
1258   // See also Matcher::interpreter_method_oop_reg().
1259   AddressLiteral meta = __ allocate_metadata_address(NULL);
1260   bool success = __ load_const_from_toc(Z_method, meta);
1261 
1262   __ set_inst_mark();
1263   AddressLiteral a((address)-1);
1264   success = success && __ load_const_from_toc(Z_R1, a);
1265   if (!success) {
1266     bailout("const section overflow");
1267     return;
1268   }
1269 
1270   __ z_br(Z_R1);
1271   assert(__ offset() - start <= call_stub_size(), "stub too big");
1272   __ end_a_stub(); // Update current stubs pointer and restore insts_end.
1273 }
1274 
1275 void LIR_Assembler::comp_op(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Op2* op) {
1276   bool unsigned_comp = condition == lir_cond_belowEqual || condition == lir_cond_aboveEqual;
1277   if (opr1->is_single_cpu()) {
1278     Register reg1 = opr1->as_register();
1279     if (opr2->is_single_cpu()) {
1280       // cpu register - cpu register
1281       if (opr1->type() == T_OBJECT || opr1->type() == T_ARRAY) {
1282         __ z_clgr(reg1, opr2->as_register());
1283       } else {
1284         assert(opr2->type() != T_OBJECT && opr2->type() != T_ARRAY, "cmp int, oop?");
1285         if (unsigned_comp) {
1286           __ z_clr(reg1, opr2->as_register());
1287         } else {
1288           __ z_cr(reg1, opr2->as_register());
1289         }
1290       }
1291     } else if (opr2->is_stack()) {
1292       // cpu register - stack
1293       if (opr1->type() == T_OBJECT || opr1->type() == T_ARRAY) {
1294         __ z_cg(reg1, frame_map()->address_for_slot(opr2->single_stack_ix()));
1295       } else {
1296         if (unsigned_comp) {
1297           __ z_cly(reg1, frame_map()->address_for_slot(opr2->single_stack_ix()));
1298         } else {
1299           __ z_cy(reg1, frame_map()->address_for_slot(opr2->single_stack_ix()));
1300         }
1301       }
1302     } else if (opr2->is_constant()) {
1303       // cpu register - constant
1304       LIR_Const* c = opr2->as_constant_ptr();
1305       if (c->type() == T_INT) {
1306         if (unsigned_comp) {
1307           __ z_clfi(reg1, c->as_jint());
1308         } else {
1309           __ z_cfi(reg1, c->as_jint());
1310         }
1311       } else if (c->type() == T_OBJECT || c->type() == T_ARRAY) {
1312         // In 64bit oops are single register.
1313         jobject o = c->as_jobject();
1314         if (o == NULL) {
1315           __ z_ltgr(reg1, reg1);
1316         } else {
1317           jobject2reg(o, Z_R1_scratch);
1318           __ z_cgr(reg1, Z_R1_scratch);
1319         }
1320       } else {
1321         fatal("unexpected type: %s", basictype_to_str(c->type()));
1322       }
1323       // cpu register - address
1324     } else if (opr2->is_address()) {
1325       if (op->info() != NULL) {
1326         add_debug_info_for_null_check_here(op->info());
1327       }
1328       if (unsigned_comp) {
1329         __ z_cly(reg1, as_Address(opr2->as_address_ptr()));
1330       } else {
1331         __ z_cy(reg1, as_Address(opr2->as_address_ptr()));
1332       }
1333     } else {
1334       ShouldNotReachHere();
1335     }
1336 
1337   } else if (opr1->is_double_cpu()) {
1338     assert(!unsigned_comp, "unexpected");
1339     Register xlo = opr1->as_register_lo();
1340     Register xhi = opr1->as_register_hi();
1341     if (opr2->is_double_cpu()) {
1342       __ z_cgr(xlo, opr2->as_register_lo());
1343     } else if (opr2->is_constant()) {
1344       // cpu register - constant 0
1345       assert(opr2->as_jlong() == (jlong)0, "only handles zero");
1346       __ z_ltgr(xlo, xlo);
1347     } else {
1348       ShouldNotReachHere();
1349     }
1350 
1351   } else if (opr1->is_single_fpu()) {
1352     if (opr2->is_single_fpu()) {
1353       __ z_cebr(opr1->as_float_reg(), opr2->as_float_reg());
1354     } else {
1355       // stack slot
1356       Address addr = frame_map()->address_for_slot(opr2->single_stack_ix());
1357       if (Immediate::is_uimm12(addr.disp())) {
1358         __ z_ceb(opr1->as_float_reg(), addr);
1359       } else {
1360         __ z_ley(Z_fscratch_1, addr);
1361         __ z_cebr(opr1->as_float_reg(), Z_fscratch_1);
1362       }
1363     }
1364   } else if (opr1->is_double_fpu()) {
1365     if (opr2->is_double_fpu()) {
1366     __ z_cdbr(opr1->as_double_reg(), opr2->as_double_reg());
1367     } else {
1368       // stack slot
1369       Address addr = frame_map()->address_for_slot(opr2->double_stack_ix());
1370       if (Immediate::is_uimm12(addr.disp())) {
1371         __ z_cdb(opr1->as_double_reg(), addr);
1372       } else {
1373         __ z_ldy(Z_fscratch_1, addr);
1374         __ z_cdbr(opr1->as_double_reg(), Z_fscratch_1);
1375       }
1376     }
1377   } else {
1378     ShouldNotReachHere();
1379   }
1380 }
1381 
1382 void LIR_Assembler::comp_fl2i(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dst, LIR_Op2* op) {
1383   Label    done;
1384   Register dreg = dst->as_register();
1385 
1386   if (code == lir_cmp_fd2i || code == lir_ucmp_fd2i) {
1387     assert((left->is_single_fpu() && right->is_single_fpu()) ||
1388            (left->is_double_fpu() && right->is_double_fpu()), "unexpected operand types");
1389     bool is_single = left->is_single_fpu();
1390     bool is_unordered_less = (code == lir_ucmp_fd2i);
1391     FloatRegister lreg = is_single ? left->as_float_reg() : left->as_double_reg();
1392     FloatRegister rreg = is_single ? right->as_float_reg() : right->as_double_reg();
1393     if (is_single) {
1394       __ z_cebr(lreg, rreg);
1395     } else {
1396       __ z_cdbr(lreg, rreg);
1397     }
1398     if (VM_Version::has_LoadStoreConditional()) {
1399       Register one       = Z_R0_scratch;
1400       Register minus_one = Z_R1_scratch;
1401       __ z_lghi(minus_one, -1);
1402       __ z_lghi(one,  1);
1403       __ z_lghi(dreg, 0);
1404       __ z_locgr(dreg, one,       is_unordered_less ? Assembler::bcondHigh            : Assembler::bcondHighOrNotOrdered);
1405       __ z_locgr(dreg, minus_one, is_unordered_less ? Assembler::bcondLowOrNotOrdered : Assembler::bcondLow);
1406     } else {
1407       __ clear_reg(dreg, true, false);
1408       __ z_bre(done); // if (left == right) dst = 0
1409 
1410       // if (left > right || ((code ~= cmpg) && (left <> right)) dst := 1
1411       __ z_lhi(dreg, 1);
1412       __ z_brc(is_unordered_less ? Assembler::bcondHigh : Assembler::bcondHighOrNotOrdered, done);
1413 
1414       // if (left < right || ((code ~= cmpl) && (left <> right)) dst := -1
1415       __ z_lhi(dreg, -1);
1416     }
1417   } else {
1418     assert(code == lir_cmp_l2i, "check");
1419     if (VM_Version::has_LoadStoreConditional()) {
1420       Register one       = Z_R0_scratch;
1421       Register minus_one = Z_R1_scratch;
1422       __ z_cgr(left->as_register_lo(), right->as_register_lo());
1423       __ z_lghi(minus_one, -1);
1424       __ z_lghi(one,  1);
1425       __ z_lghi(dreg, 0);
1426       __ z_locgr(dreg, one, Assembler::bcondHigh);
1427       __ z_locgr(dreg, minus_one, Assembler::bcondLow);
1428     } else {
1429       __ z_cgr(left->as_register_lo(), right->as_register_lo());
1430       __ z_lghi(dreg,  0);     // eq value
1431       __ z_bre(done);
1432       __ z_lghi(dreg,  1);     // gt value
1433       __ z_brh(done);
1434       __ z_lghi(dreg, -1);     // lt value
1435     }
1436   }
1437   __ bind(done);
1438 }
1439 
1440 // result = condition ? opr1 : opr2
1441 void LIR_Assembler::cmove(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Opr result, BasicType type) {
1442   Assembler::branch_condition acond = Assembler::bcondEqual, ncond = Assembler::bcondNotEqual;
1443   switch (condition) {
1444     case lir_cond_equal:        acond = Assembler::bcondEqual;    ncond = Assembler::bcondNotEqual; break;
1445     case lir_cond_notEqual:     acond = Assembler::bcondNotEqual; ncond = Assembler::bcondEqual;    break;
1446     case lir_cond_less:         acond = Assembler::bcondLow;      ncond = Assembler::bcondNotLow;   break;
1447     case lir_cond_lessEqual:    acond = Assembler::bcondNotHigh;  ncond = Assembler::bcondHigh;     break;
1448     case lir_cond_greaterEqual: acond = Assembler::bcondNotLow;   ncond = Assembler::bcondLow;      break;
1449     case lir_cond_greater:      acond = Assembler::bcondHigh;     ncond = Assembler::bcondNotHigh;  break;
1450     case lir_cond_belowEqual:   acond = Assembler::bcondNotHigh;  ncond = Assembler::bcondHigh;     break;
1451     case lir_cond_aboveEqual:   acond = Assembler::bcondNotLow;   ncond = Assembler::bcondLow;      break;
1452     default:                    ShouldNotReachHere();
1453   }
1454 
1455   if (opr1->is_cpu_register()) {
1456     reg2reg(opr1, result);
1457   } else if (opr1->is_stack()) {
1458     stack2reg(opr1, result, result->type());
1459   } else if (opr1->is_constant()) {
1460     const2reg(opr1, result, lir_patch_none, NULL);
1461   } else {
1462     ShouldNotReachHere();
1463   }
1464 
1465   if (VM_Version::has_LoadStoreConditional() && !opr2->is_constant()) {
1466     // Optimized version that does not require a branch.
1467     if (opr2->is_single_cpu()) {
1468       assert(opr2->cpu_regnr() != result->cpu_regnr(), "opr2 already overwritten by previous move");
1469       __ z_locgr(result->as_register(), opr2->as_register(), ncond);
1470     } else if (opr2->is_double_cpu()) {
1471       assert(opr2->cpu_regnrLo() != result->cpu_regnrLo() && opr2->cpu_regnrLo() != result->cpu_regnrHi(), "opr2 already overwritten by previous move");
1472       assert(opr2->cpu_regnrHi() != result->cpu_regnrLo() && opr2->cpu_regnrHi() != result->cpu_regnrHi(), "opr2 already overwritten by previous move");
1473       __ z_locgr(result->as_register_lo(), opr2->as_register_lo(), ncond);
1474     } else if (opr2->is_single_stack()) {
1475       __ z_loc(result->as_register(), frame_map()->address_for_slot(opr2->single_stack_ix()), ncond);
1476     } else if (opr2->is_double_stack()) {
1477       __ z_locg(result->as_register_lo(), frame_map()->address_for_slot(opr2->double_stack_ix()), ncond);
1478     } else {
1479       ShouldNotReachHere();
1480     }
1481   } else {
1482     Label skip;
1483     __ z_brc(acond, skip);
1484     if (opr2->is_cpu_register()) {
1485       reg2reg(opr2, result);
1486     } else if (opr2->is_stack()) {
1487       stack2reg(opr2, result, result->type());
1488     } else if (opr2->is_constant()) {
1489       const2reg(opr2, result, lir_patch_none, NULL);
1490     } else {
1491       ShouldNotReachHere();
1492     }
1493     __ bind(skip);
1494   }
1495 }
1496 
1497 void LIR_Assembler::arith_op(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dest,
1498                              CodeEmitInfo* info, bool pop_fpu_stack) {
1499   assert(info == NULL, "should never be used, idiv/irem and ldiv/lrem not handled by this method");
1500 
1501   if (left->is_single_cpu()) {
1502     assert(left == dest, "left and dest must be equal");
1503     Register lreg = left->as_register();
1504 
1505     if (right->is_single_cpu()) {
1506       // cpu register - cpu register
1507       Register rreg = right->as_register();
1508       switch (code) {
1509         case lir_add: __ z_ar (lreg, rreg); break;
1510         case lir_sub: __ z_sr (lreg, rreg); break;
1511         case lir_mul: __ z_msr(lreg, rreg); break;
1512         default: ShouldNotReachHere();
1513       }
1514 
1515     } else if (right->is_stack()) {
1516       // cpu register - stack
1517       Address raddr = frame_map()->address_for_slot(right->single_stack_ix());
1518       switch (code) {
1519         case lir_add: __ z_ay(lreg, raddr); break;
1520         case lir_sub: __ z_sy(lreg, raddr); break;
1521         default: ShouldNotReachHere();
1522       }
1523 
1524     } else if (right->is_constant()) {
1525       // cpu register - constant
1526       jint c = right->as_constant_ptr()->as_jint();
1527       switch (code) {
1528         case lir_add: __ z_agfi(lreg, c);  break;
1529         case lir_sub: __ z_agfi(lreg, -c); break; // note: -min_jint == min_jint
1530         case lir_mul: __ z_msfi(lreg, c);  break;
1531         default: ShouldNotReachHere();
1532       }
1533 
1534     } else {
1535       ShouldNotReachHere();
1536     }
1537 
1538   } else if (left->is_double_cpu()) {
1539     assert(left == dest, "left and dest must be equal");
1540     Register lreg_lo = left->as_register_lo();
1541     Register lreg_hi = left->as_register_hi();
1542 
1543     if (right->is_double_cpu()) {
1544       // cpu register - cpu register
1545       Register rreg_lo = right->as_register_lo();
1546       Register rreg_hi = right->as_register_hi();
1547       assert_different_registers(lreg_lo, rreg_lo);
1548       switch (code) {
1549         case lir_add:
1550           __ z_agr(lreg_lo, rreg_lo);
1551           break;
1552         case lir_sub:
1553           __ z_sgr(lreg_lo, rreg_lo);
1554           break;
1555         case lir_mul:
1556           __ z_msgr(lreg_lo, rreg_lo);
1557           break;
1558         default:
1559           ShouldNotReachHere();
1560       }
1561 
1562     } else if (right->is_constant()) {
1563       // cpu register - constant
1564       jlong c = right->as_constant_ptr()->as_jlong_bits();
1565       switch (code) {
1566         case lir_add: __ z_agfi(lreg_lo, c); break;
1567         case lir_sub:
1568           if (c != min_jint) {
1569                       __ z_agfi(lreg_lo, -c);
1570           } else {
1571             // -min_jint cannot be represented as simm32 in z_agfi
1572             // min_jint sign extended:      0xffffffff80000000
1573             // -min_jint as 64 bit integer: 0x0000000080000000
1574             // 0x80000000 can be represented as uimm32 in z_algfi
1575             // lreg_lo := lreg_lo + -min_jint == lreg_lo + 0x80000000
1576                       __ z_algfi(lreg_lo, UCONST64(0x80000000));
1577           }
1578           break;
1579         case lir_mul: __ z_msgfi(lreg_lo, c); break;
1580         default:
1581           ShouldNotReachHere();
1582       }
1583 
1584     } else {
1585       ShouldNotReachHere();
1586     }
1587 
1588   } else if (left->is_single_fpu()) {
1589     assert(left == dest, "left and dest must be equal");
1590     FloatRegister lreg = left->as_float_reg();
1591     FloatRegister rreg = right->is_single_fpu() ? right->as_float_reg() : fnoreg;
1592     Address raddr;
1593 
1594     if (rreg == fnoreg) {
1595       assert(right->is_single_stack(), "constants should be loaded into register");
1596       raddr = frame_map()->address_for_slot(right->single_stack_ix());
1597       if (!Immediate::is_uimm12(raddr.disp())) {
1598         __ mem2freg_opt(rreg = Z_fscratch_1, raddr, false);
1599       }
1600     }
1601 
1602     if (rreg != fnoreg) {
1603       switch (code) {
1604         case lir_add: __ z_aebr(lreg, rreg);  break;
1605         case lir_sub: __ z_sebr(lreg, rreg);  break;
1606         case lir_mul_strictfp: // fall through
1607         case lir_mul: __ z_meebr(lreg, rreg); break;
1608         case lir_div_strictfp: // fall through
1609         case lir_div: __ z_debr(lreg, rreg);  break;
1610         default: ShouldNotReachHere();
1611       }
1612     } else {
1613       switch (code) {
1614         case lir_add: __ z_aeb(lreg, raddr);  break;
1615         case lir_sub: __ z_seb(lreg, raddr);  break;
1616         case lir_mul_strictfp: // fall through
1617         case lir_mul: __ z_meeb(lreg, raddr);  break;
1618         case lir_div_strictfp: // fall through
1619         case lir_div: __ z_deb(lreg, raddr);  break;
1620         default: ShouldNotReachHere();
1621       }
1622     }
1623   } else if (left->is_double_fpu()) {
1624     assert(left == dest, "left and dest must be equal");
1625     FloatRegister lreg = left->as_double_reg();
1626     FloatRegister rreg = right->is_double_fpu() ? right->as_double_reg() : fnoreg;
1627     Address raddr;
1628 
1629     if (rreg == fnoreg) {
1630       assert(right->is_double_stack(), "constants should be loaded into register");
1631       raddr = frame_map()->address_for_slot(right->double_stack_ix());
1632       if (!Immediate::is_uimm12(raddr.disp())) {
1633         __ mem2freg_opt(rreg = Z_fscratch_1, raddr, true);
1634       }
1635     }
1636 
1637     if (rreg != fnoreg) {
1638       switch (code) {
1639         case lir_add: __ z_adbr(lreg, rreg); break;
1640         case lir_sub: __ z_sdbr(lreg, rreg); break;
1641         case lir_mul_strictfp: // fall through
1642         case lir_mul: __ z_mdbr(lreg, rreg); break;
1643         case lir_div_strictfp: // fall through
1644         case lir_div: __ z_ddbr(lreg, rreg); break;
1645         default: ShouldNotReachHere();
1646       }
1647     } else {
1648       switch (code) {
1649         case lir_add: __ z_adb(lreg, raddr); break;
1650         case lir_sub: __ z_sdb(lreg, raddr); break;
1651         case lir_mul_strictfp: // fall through
1652         case lir_mul: __ z_mdb(lreg, raddr); break;
1653         case lir_div_strictfp: // fall through
1654         case lir_div: __ z_ddb(lreg, raddr); break;
1655         default: ShouldNotReachHere();
1656       }
1657     }
1658   } else if (left->is_address()) {
1659     assert(left == dest, "left and dest must be equal");
1660     assert(code == lir_add, "unsupported operation");
1661     assert(right->is_constant(), "unsupported operand");
1662     jint c = right->as_constant_ptr()->as_jint();
1663     LIR_Address* lir_addr = left->as_address_ptr();
1664     Address addr = as_Address(lir_addr);
1665     switch (lir_addr->type()) {
1666       case T_INT:
1667         __ add2mem_32(addr, c, Z_R1_scratch);
1668         break;
1669       case T_LONG:
1670         __ add2mem_64(addr, c, Z_R1_scratch);
1671         break;
1672       default:
1673         ShouldNotReachHere();
1674     }
1675   } else {
1676     ShouldNotReachHere();
1677   }
1678 }
1679 
1680 void LIR_Assembler::fpop() {
1681   // do nothing
1682 }
1683 
1684 void LIR_Assembler::intrinsic_op(LIR_Code code, LIR_Opr value, LIR_Opr thread, LIR_Opr dest, LIR_Op* op) {
1685   switch (code) {
1686     case lir_sqrt: {
1687       assert(!thread->is_valid(), "there is no need for a thread_reg for dsqrt");
1688       FloatRegister src_reg = value->as_double_reg();
1689       FloatRegister dst_reg = dest->as_double_reg();
1690       __ z_sqdbr(dst_reg, src_reg);
1691       break;
1692     }
1693     case lir_abs: {
1694       assert(!thread->is_valid(), "there is no need for a thread_reg for fabs");
1695       FloatRegister src_reg = value->as_double_reg();
1696       FloatRegister dst_reg = dest->as_double_reg();
1697       __ z_lpdbr(dst_reg, src_reg);
1698       break;
1699     }
1700     default: {
1701       ShouldNotReachHere();
1702       break;
1703     }
1704   }
1705 }
1706 
1707 void LIR_Assembler::logic_op(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dst) {
1708   if (left->is_single_cpu()) {
1709     Register reg = left->as_register();
1710     if (right->is_constant()) {
1711       int val = right->as_constant_ptr()->as_jint();
1712       switch (code) {
1713         case lir_logic_and: __ z_nilf(reg, val); break;
1714         case lir_logic_or:  __ z_oilf(reg, val); break;
1715         case lir_logic_xor: __ z_xilf(reg, val); break;
1716         default: ShouldNotReachHere();
1717       }
1718     } else if (right->is_stack()) {
1719       Address raddr = frame_map()->address_for_slot(right->single_stack_ix());
1720       switch (code) {
1721         case lir_logic_and: __ z_ny(reg, raddr); break;
1722         case lir_logic_or:  __ z_oy(reg, raddr); break;
1723         case lir_logic_xor: __ z_xy(reg, raddr); break;
1724         default: ShouldNotReachHere();
1725       }
1726     } else {
1727       Register rright = right->as_register();
1728       switch (code) {
1729         case lir_logic_and: __ z_nr(reg, rright); break;
1730         case lir_logic_or : __ z_or(reg, rright); break;
1731         case lir_logic_xor: __ z_xr(reg, rright); break;
1732         default: ShouldNotReachHere();
1733       }
1734     }
1735     move_regs(reg, dst->as_register());
1736   } else {
1737     Register l_lo = left->as_register_lo();
1738     if (right->is_constant()) {
1739       __ load_const_optimized(Z_R1_scratch, right->as_constant_ptr()->as_jlong());
1740       switch (code) {
1741         case lir_logic_and:
1742           __ z_ngr(l_lo, Z_R1_scratch);
1743           break;
1744         case lir_logic_or:
1745           __ z_ogr(l_lo, Z_R1_scratch);
1746           break;
1747         case lir_logic_xor:
1748           __ z_xgr(l_lo, Z_R1_scratch);
1749           break;
1750         default: ShouldNotReachHere();
1751       }
1752     } else {
1753       Register r_lo;
1754       if (right->type() == T_OBJECT || right->type() == T_ARRAY) {
1755         r_lo = right->as_register();
1756       } else {
1757         r_lo = right->as_register_lo();
1758       }
1759       switch (code) {
1760         case lir_logic_and:
1761           __ z_ngr(l_lo, r_lo);
1762           break;
1763         case lir_logic_or:
1764           __ z_ogr(l_lo, r_lo);
1765           break;
1766         case lir_logic_xor:
1767           __ z_xgr(l_lo, r_lo);
1768           break;
1769         default: ShouldNotReachHere();
1770       }
1771     }
1772 
1773     Register dst_lo = dst->as_register_lo();
1774 
1775     move_regs(l_lo, dst_lo);
1776   }
1777 }
1778 
1779 // See operand selection in LIRGenerator::do_ArithmeticOp_Int().
1780 void LIR_Assembler::arithmetic_idiv(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr temp, LIR_Opr result, CodeEmitInfo* info) {
1781   if (left->is_double_cpu()) {
1782     // 64 bit integer case
1783     assert(left->is_double_cpu(), "left must be register");
1784     assert(right->is_double_cpu() || is_power_of_2_long(right->as_jlong()),
1785            "right must be register or power of 2 constant");
1786     assert(result->is_double_cpu(), "result must be register");
1787 
1788     Register lreg = left->as_register_lo();
1789     Register dreg = result->as_register_lo();
1790 
1791     if (right->is_constant()) {
1792       // Convert division by a power of two into some shifts and logical operations.
1793       Register treg1 = Z_R0_scratch;
1794       Register treg2 = Z_R1_scratch;
1795       jlong divisor = right->as_jlong();
1796       jlong log_divisor = log2_long(right->as_jlong());
1797 
1798       if (divisor == min_jlong) {
1799         // Min_jlong is special. Result is '0' except for min_jlong/min_jlong = 1.
1800         if (dreg == lreg) {
1801           NearLabel done;
1802           __ load_const_optimized(treg2, min_jlong);
1803           __ z_cgr(lreg, treg2);
1804           __ z_lghi(dreg, 0);           // Preserves condition code.
1805           __ z_brne(done);
1806           __ z_lghi(dreg, 1);           // min_jlong / min_jlong = 1
1807           __ bind(done);
1808         } else {
1809           assert_different_registers(dreg, lreg);
1810           NearLabel done;
1811           __ z_lghi(dreg, 0);
1812           __ compare64_and_branch(lreg, min_jlong, Assembler::bcondNotEqual, done);
1813           __ z_lghi(dreg, 1);
1814           __ bind(done);
1815         }
1816         return;
1817       }
1818       __ move_reg_if_needed(dreg, T_LONG, lreg, T_LONG);
1819       if (divisor == 2) {
1820         __ z_srlg(treg2, dreg, 63);     // dividend < 0 ? 1 : 0
1821       } else {
1822         __ z_srag(treg2, dreg, 63);     // dividend < 0 ? -1 : 0
1823         __ and_imm(treg2, divisor - 1, treg1, true);
1824       }
1825       if (code == lir_idiv) {
1826         __ z_agr(dreg, treg2);
1827         __ z_srag(dreg, dreg, log_divisor);
1828       } else {
1829         assert(code == lir_irem, "check");
1830         __ z_agr(treg2, dreg);
1831         __ and_imm(treg2, ~(divisor - 1), treg1, true);
1832         __ z_sgr(dreg, treg2);
1833       }
1834       return;
1835     }
1836 
1837     // Divisor is not a power of 2 constant.
1838     Register rreg = right->as_register_lo();
1839     Register treg = temp->as_register_lo();
1840     assert(right->is_double_cpu(), "right must be register");
1841     assert(lreg == Z_R11, "see ldivInOpr()");
1842     assert(rreg != lreg, "right register must not be same as left register");
1843     assert((code == lir_idiv && dreg == Z_R11 && treg == Z_R10) ||
1844            (code == lir_irem && dreg == Z_R10 && treg == Z_R11), "see ldivInOpr(), ldivOutOpr(), lremOutOpr()");
1845 
1846     Register R1 = lreg->predecessor();
1847     Register R2 = rreg;
1848     assert(code != lir_idiv || lreg==dreg, "see code below");
1849     if (code == lir_idiv) {
1850       __ z_lcgr(lreg, lreg);
1851     } else {
1852       __ clear_reg(dreg, true, false);
1853     }
1854     NearLabel done;
1855     __ compare64_and_branch(R2, -1, Assembler::bcondEqual, done);
1856     if (code == lir_idiv) {
1857       __ z_lcgr(lreg, lreg); // Revert lcgr above.
1858     }
1859     if (ImplicitDiv0Checks) {
1860       // No debug info because the idiv won't trap.
1861       // Add_debug_info_for_div0 would instantiate another DivByZeroStub,
1862       // which is unnecessary, too.
1863       add_debug_info_for_div0(__ offset(), info);
1864     }
1865     __ z_dsgr(R1, R2);
1866     __ bind(done);
1867     return;
1868   }
1869 
1870   // 32 bit integer case
1871 
1872   assert(left->is_single_cpu(), "left must be register");
1873   assert(right->is_single_cpu() || is_power_of_2(right->as_jint()), "right must be register or power of 2 constant");
1874   assert(result->is_single_cpu(), "result must be register");
1875 
1876   Register lreg = left->as_register();
1877   Register dreg = result->as_register();
1878 
1879   if (right->is_constant()) {
1880     // Convert division by a power of two into some shifts and logical operations.
1881     Register treg1 = Z_R0_scratch;
1882     Register treg2 = Z_R1_scratch;
1883     jlong divisor = right->as_jint();
1884     jlong log_divisor = log2_long(right->as_jint());
1885     __ move_reg_if_needed(dreg, T_LONG, lreg, T_INT); // sign extend
1886     if (divisor == 2) {
1887       __ z_srlg(treg2, dreg, 63);     // dividend < 0 ?  1 : 0
1888     } else {
1889       __ z_srag(treg2, dreg, 63);     // dividend < 0 ? -1 : 0
1890       __ and_imm(treg2, divisor - 1, treg1, true);
1891     }
1892     if (code == lir_idiv) {
1893       __ z_agr(dreg, treg2);
1894       __ z_srag(dreg, dreg, log_divisor);
1895     } else {
1896       assert(code == lir_irem, "check");
1897       __ z_agr(treg2, dreg);
1898       __ and_imm(treg2, ~(divisor - 1), treg1, true);
1899       __ z_sgr(dreg, treg2);
1900     }
1901     return;
1902   }
1903 
1904   // Divisor is not a power of 2 constant.
1905   Register rreg = right->as_register();
1906   Register treg = temp->as_register();
1907   assert(right->is_single_cpu(), "right must be register");
1908   assert(lreg == Z_R11, "left register must be rax,");
1909   assert(rreg != lreg, "right register must not be same as left register");
1910   assert((code == lir_idiv && dreg == Z_R11 && treg == Z_R10)
1911       || (code == lir_irem && dreg == Z_R10 && treg == Z_R11), "see divInOpr(), divOutOpr(), remOutOpr()");
1912 
1913   Register R1 = lreg->predecessor();
1914   Register R2 = rreg;
1915   __ move_reg_if_needed(lreg, T_LONG, lreg, T_INT); // sign extend
1916   if (ImplicitDiv0Checks) {
1917     // No debug info because the idiv won't trap.
1918     // Add_debug_info_for_div0 would instantiate another DivByZeroStub,
1919     // which is unnecessary, too.
1920     add_debug_info_for_div0(__ offset(), info);
1921   }
1922   __ z_dsgfr(R1, R2);
1923 }
1924 
1925 void LIR_Assembler::throw_op(LIR_Opr exceptionPC, LIR_Opr exceptionOop, CodeEmitInfo* info) {
1926   assert(exceptionOop->as_register() == Z_EXC_OOP, "should match");
1927   assert(exceptionPC->as_register() == Z_EXC_PC, "should match");
1928 
1929   // Exception object is not added to oop map by LinearScan
1930   // (LinearScan assumes that no oops are in fixed registers).
1931   info->add_register_oop(exceptionOop);
1932 
1933   // Reuse the debug info from the safepoint poll for the throw op itself.
1934   __ get_PC(Z_EXC_PC);
1935   add_call_info(__ offset(), info); // for exception handler
1936   address stub = Runtime1::entry_for (compilation()->has_fpu_code() ? Runtime1::handle_exception_id
1937                                                                     : Runtime1::handle_exception_nofpu_id);
1938   emit_call_c(stub);
1939 }
1940 
1941 void LIR_Assembler::unwind_op(LIR_Opr exceptionOop) {
1942   assert(exceptionOop->as_register() == Z_EXC_OOP, "should match");
1943 
1944   __ branch_optimized(Assembler::bcondAlways, _unwind_handler_entry);
1945 }
1946 
1947 void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) {
1948   ciArrayKlass* default_type = op->expected_type();
1949   Register src = op->src()->as_register();
1950   Register dst = op->dst()->as_register();
1951   Register src_pos = op->src_pos()->as_register();
1952   Register dst_pos = op->dst_pos()->as_register();
1953   Register length  = op->length()->as_register();
1954   Register tmp = op->tmp()->as_register();
1955 
1956   CodeStub* stub = op->stub();
1957   int flags = op->flags();
1958   BasicType basic_type = default_type != NULL ? default_type->element_type()->basic_type() : T_ILLEGAL;
1959   if (basic_type == T_ARRAY) basic_type = T_OBJECT;
1960 
1961   // If we don't know anything, just go through the generic arraycopy.
1962   if (default_type == NULL) {
1963     address copyfunc_addr = StubRoutines::generic_arraycopy();
1964 
1965     if (copyfunc_addr == NULL) {
1966       // Take a slow path for generic arraycopy.
1967       __ branch_optimized(Assembler::bcondAlways, *stub->entry());
1968       __ bind(*stub->continuation());
1969       return;
1970     }
1971 
1972     Label done;
1973     // Save outgoing arguments in callee saved registers (C convention) in case
1974     // a call to System.arraycopy is needed.
1975     Register callee_saved_src     = Z_R10;
1976     Register callee_saved_src_pos = Z_R11;
1977     Register callee_saved_dst     = Z_R12;
1978     Register callee_saved_dst_pos = Z_R13;
1979     Register callee_saved_length  = Z_ARG5; // Z_ARG5 == Z_R6 is callee saved.
1980 
1981     __ lgr_if_needed(callee_saved_src, src);
1982     __ lgr_if_needed(callee_saved_src_pos, src_pos);
1983     __ lgr_if_needed(callee_saved_dst, dst);
1984     __ lgr_if_needed(callee_saved_dst_pos, dst_pos);
1985     __ lgr_if_needed(callee_saved_length, length);
1986 
1987     // C function requires 64 bit values.
1988     __ z_lgfr(src_pos, src_pos);
1989     __ z_lgfr(dst_pos, dst_pos);
1990     __ z_lgfr(length, length);
1991 
1992     // Pass arguments: may push as this is not a safepoint; SP must be fix at each safepoint.
1993 
1994     // The arguments are in the corresponding registers.
1995     assert(Z_ARG1 == src,     "assumption");
1996     assert(Z_ARG2 == src_pos, "assumption");
1997     assert(Z_ARG3 == dst,     "assumption");
1998     assert(Z_ARG4 == dst_pos, "assumption");
1999     assert(Z_ARG5 == length,  "assumption");
2000 #ifndef PRODUCT
2001     if (PrintC1Statistics) {
2002       __ load_const_optimized(Z_R1_scratch, (address)&Runtime1::_generic_arraycopystub_cnt);
2003       __ add2mem_32(Address(Z_R1_scratch), 1, Z_R0_scratch);
2004     }
2005 #endif
2006     emit_call_c(copyfunc_addr);
2007     CHECK_BAILOUT();
2008 
2009     __ compare32_and_branch(Z_RET, (intptr_t)0, Assembler::bcondEqual, *stub->continuation());
2010 
2011     __ z_lgr(tmp, Z_RET);
2012     __ z_xilf(tmp, -1);
2013 
2014     // Restore values from callee saved registers so they are where the stub
2015     // expects them.
2016     __ lgr_if_needed(src, callee_saved_src);
2017     __ lgr_if_needed(src_pos, callee_saved_src_pos);
2018     __ lgr_if_needed(dst, callee_saved_dst);
2019     __ lgr_if_needed(dst_pos, callee_saved_dst_pos);
2020     __ lgr_if_needed(length, callee_saved_length);
2021 
2022     __ z_sr(length, tmp);
2023     __ z_ar(src_pos, tmp);
2024     __ z_ar(dst_pos, tmp);
2025     __ branch_optimized(Assembler::bcondAlways, *stub->entry());
2026 
2027     __ bind(*stub->continuation());
2028     return;
2029   }
2030 
2031   assert(default_type != NULL && default_type->is_array_klass() && default_type->is_loaded(), "must be true at this point");
2032 
2033   int elem_size = type2aelembytes(basic_type);
2034   int shift_amount;
2035 
2036   switch (elem_size) {
2037     case 1 :
2038       shift_amount = 0;
2039       break;
2040     case 2 :
2041       shift_amount = 1;
2042       break;
2043     case 4 :
2044       shift_amount = 2;
2045       break;
2046     case 8 :
2047       shift_amount = 3;
2048       break;
2049     default:
2050       shift_amount = -1;
2051       ShouldNotReachHere();
2052   }
2053 
2054   Address src_length_addr = Address(src, arrayOopDesc::length_offset_in_bytes());
2055   Address dst_length_addr = Address(dst, arrayOopDesc::length_offset_in_bytes());
2056   Address src_klass_addr = Address(src, oopDesc::klass_offset_in_bytes());
2057   Address dst_klass_addr = Address(dst, oopDesc::klass_offset_in_bytes());
2058 
2059   // Length and pos's are all sign extended at this point on 64bit.
2060 
2061   // test for NULL
2062   if (flags & LIR_OpArrayCopy::src_null_check) {
2063     __ compareU64_and_branch(src, (intptr_t)0, Assembler::bcondZero, *stub->entry());
2064   }
2065   if (flags & LIR_OpArrayCopy::dst_null_check) {
2066     __ compareU64_and_branch(dst, (intptr_t)0, Assembler::bcondZero, *stub->entry());
2067   }
2068 
2069   // Check if negative.
2070   if (flags & LIR_OpArrayCopy::src_pos_positive_check) {
2071     __ compare32_and_branch(src_pos, (intptr_t)0, Assembler::bcondLow, *stub->entry());
2072   }
2073   if (flags & LIR_OpArrayCopy::dst_pos_positive_check) {
2074     __ compare32_and_branch(dst_pos, (intptr_t)0, Assembler::bcondLow, *stub->entry());
2075   }
2076 
2077   // If the compiler was not able to prove that exact type of the source or the destination
2078   // of the arraycopy is an array type, check at runtime if the source or the destination is
2079   // an instance type.
2080   if (flags & LIR_OpArrayCopy::type_check) {
2081     assert(Klass::_lh_neutral_value == 0, "or replace z_lt instructions");
2082 
2083     if (!(flags & LIR_OpArrayCopy::dst_objarray)) {
2084       __ load_klass(tmp, dst);
2085       __ z_lt(tmp, Address(tmp, in_bytes(Klass::layout_helper_offset())));
2086       __ branch_optimized(Assembler::bcondNotLow, *stub->entry());
2087     }
2088 
2089     if (!(flags & LIR_OpArrayCopy::src_objarray)) {
2090       __ load_klass(tmp, src);
2091       __ z_lt(tmp, Address(tmp, in_bytes(Klass::layout_helper_offset())));
2092       __ branch_optimized(Assembler::bcondNotLow, *stub->entry());
2093     }
2094   }
2095 
2096   if (flags & LIR_OpArrayCopy::src_range_check) {
2097     __ z_la(tmp, Address(src_pos, length));
2098     __ z_cl(tmp, src_length_addr);
2099     __ branch_optimized(Assembler::bcondHigh, *stub->entry());
2100   }
2101   if (flags & LIR_OpArrayCopy::dst_range_check) {
2102     __ z_la(tmp, Address(dst_pos, length));
2103     __ z_cl(tmp, dst_length_addr);
2104     __ branch_optimized(Assembler::bcondHigh, *stub->entry());
2105   }
2106 
2107   if (flags & LIR_OpArrayCopy::length_positive_check) {
2108     __ z_ltr(length, length);
2109     __ branch_optimized(Assembler::bcondNegative, *stub->entry());
2110   }
2111 
2112   // Stubs require 64 bit values.
2113   __ z_lgfr(src_pos, src_pos); // int -> long
2114   __ z_lgfr(dst_pos, dst_pos); // int -> long
2115   __ z_lgfr(length, length);   // int -> long
2116 
2117   if (flags & LIR_OpArrayCopy::type_check) {
2118     // We don't know the array types are compatible.
2119     if (basic_type != T_OBJECT) {
2120       // Simple test for basic type arrays.
2121       if (UseCompressedClassPointers) {
2122         __ z_l(tmp, src_klass_addr);
2123         __ z_c(tmp, dst_klass_addr);
2124       } else {
2125         __ z_lg(tmp, src_klass_addr);
2126         __ z_cg(tmp, dst_klass_addr);
2127       }
2128       __ branch_optimized(Assembler::bcondNotEqual, *stub->entry());
2129     } else {
2130       // For object arrays, if src is a sub class of dst then we can
2131       // safely do the copy.
2132       NearLabel cont, slow;
2133       Register src_klass = Z_R1_scratch;
2134       Register dst_klass = Z_R10;
2135 
2136       __ load_klass(src_klass, src);
2137       __ load_klass(dst_klass, dst);
2138 
2139       __ check_klass_subtype_fast_path(src_klass, dst_klass, tmp, &cont, &slow, NULL);
2140 
2141       store_parameter(src_klass, 0); // sub
2142       store_parameter(dst_klass, 1); // super
2143       emit_call_c(Runtime1::entry_for (Runtime1::slow_subtype_check_id));
2144       CHECK_BAILOUT();
2145       // Sets condition code 0 for match (2 otherwise).
2146       __ branch_optimized(Assembler::bcondEqual, cont);
2147 
2148       __ bind(slow);
2149 
2150       address copyfunc_addr = StubRoutines::checkcast_arraycopy();
2151       if (copyfunc_addr != NULL) { // use stub if available
2152         // Src is not a sub class of dst so we have to do a
2153         // per-element check.
2154 
2155         int mask = LIR_OpArrayCopy::src_objarray|LIR_OpArrayCopy::dst_objarray;
2156         if ((flags & mask) != mask) {
2157           // Check that at least both of them object arrays.
2158           assert(flags & mask, "one of the two should be known to be an object array");
2159 
2160           if (!(flags & LIR_OpArrayCopy::src_objarray)) {
2161             __ load_klass(tmp, src);
2162           } else if (!(flags & LIR_OpArrayCopy::dst_objarray)) {
2163             __ load_klass(tmp, dst);
2164           }
2165           Address klass_lh_addr(tmp, Klass::layout_helper_offset());
2166           jint objArray_lh = Klass::array_layout_helper(T_OBJECT);
2167           __ load_const_optimized(Z_R1_scratch, objArray_lh);
2168           __ z_c(Z_R1_scratch, klass_lh_addr);
2169           __ branch_optimized(Assembler::bcondNotEqual, *stub->entry());
2170         }
2171 
2172         // Save outgoing arguments in callee saved registers (C convention) in case
2173         // a call to System.arraycopy is needed.
2174         Register callee_saved_src     = Z_R10;
2175         Register callee_saved_src_pos = Z_R11;
2176         Register callee_saved_dst     = Z_R12;
2177         Register callee_saved_dst_pos = Z_R13;
2178         Register callee_saved_length  = Z_ARG5; // Z_ARG5 == Z_R6 is callee saved.
2179 
2180         __ lgr_if_needed(callee_saved_src, src);
2181         __ lgr_if_needed(callee_saved_src_pos, src_pos);
2182         __ lgr_if_needed(callee_saved_dst, dst);
2183         __ lgr_if_needed(callee_saved_dst_pos, dst_pos);
2184         __ lgr_if_needed(callee_saved_length, length);
2185 
2186         __ z_llgfr(length, length); // Higher 32bits must be null.
2187 
2188         __ z_sllg(Z_ARG1, src_pos, shift_amount); // index -> byte offset
2189         __ z_sllg(Z_ARG2, dst_pos, shift_amount); // index -> byte offset
2190 
2191         __ z_la(Z_ARG1, Address(src, Z_ARG1, arrayOopDesc::base_offset_in_bytes(basic_type)));
2192         assert_different_registers(Z_ARG1, dst, dst_pos, length);
2193         __ z_la(Z_ARG2, Address(dst, Z_ARG2, arrayOopDesc::base_offset_in_bytes(basic_type)));
2194         assert_different_registers(Z_ARG2, dst, length);
2195 
2196         __ z_lgr(Z_ARG3, length);
2197         assert_different_registers(Z_ARG3, dst);
2198 
2199         __ load_klass(Z_ARG5, dst);
2200         __ z_lg(Z_ARG5, Address(Z_ARG5, ObjArrayKlass::element_klass_offset()));
2201         __ z_lg(Z_ARG4, Address(Z_ARG5, Klass::super_check_offset_offset()));
2202         emit_call_c(copyfunc_addr);
2203         CHECK_BAILOUT();
2204 
2205 #ifndef PRODUCT
2206         if (PrintC1Statistics) {
2207           NearLabel failed;
2208           __ compareU32_and_branch(Z_RET, (intptr_t)0, Assembler::bcondNotEqual, failed);
2209           __ load_const_optimized(Z_R1_scratch, (address)&Runtime1::_arraycopy_checkcast_cnt);
2210           __ add2mem_32(Address(Z_R1_scratch), 1, Z_R0_scratch);
2211           __ bind(failed);
2212         }
2213 #endif
2214 
2215         __ compareU32_and_branch(Z_RET, (intptr_t)0, Assembler::bcondEqual, *stub->continuation());
2216 
2217 #ifndef PRODUCT
2218         if (PrintC1Statistics) {
2219           __ load_const_optimized(Z_R1_scratch, (address)&Runtime1::_arraycopy_checkcast_attempt_cnt);
2220           __ add2mem_32(Address(Z_R1_scratch), 1, Z_R0_scratch);
2221         }
2222 #endif
2223 
2224         __ z_lgr(tmp, Z_RET);
2225         __ z_xilf(tmp, -1);
2226 
2227         // Restore previously spilled arguments
2228         __ lgr_if_needed(src, callee_saved_src);
2229         __ lgr_if_needed(src_pos, callee_saved_src_pos);
2230         __ lgr_if_needed(dst, callee_saved_dst);
2231         __ lgr_if_needed(dst_pos, callee_saved_dst_pos);
2232         __ lgr_if_needed(length, callee_saved_length);
2233 
2234         __ z_sr(length, tmp);
2235         __ z_ar(src_pos, tmp);
2236         __ z_ar(dst_pos, tmp);
2237       }
2238 
2239       __ branch_optimized(Assembler::bcondAlways, *stub->entry());
2240 
2241       __ bind(cont);
2242     }
2243   }
2244 
2245 #ifdef ASSERT
2246   if (basic_type != T_OBJECT || !(flags & LIR_OpArrayCopy::type_check)) {
2247     // Sanity check the known type with the incoming class. For the
2248     // primitive case the types must match exactly with src.klass and
2249     // dst.klass each exactly matching the default type. For the
2250     // object array case, if no type check is needed then either the
2251     // dst type is exactly the expected type and the src type is a
2252     // subtype which we can't check or src is the same array as dst
2253     // but not necessarily exactly of type default_type.
2254     NearLabel known_ok, halt;
2255     metadata2reg(default_type->constant_encoding(), tmp);
2256     if (UseCompressedClassPointers) {
2257       __ encode_klass_not_null(tmp);
2258     }
2259 
2260     if (basic_type != T_OBJECT) {
2261       if (UseCompressedClassPointers)         { __ z_c (tmp, dst_klass_addr); }
2262       else                                    { __ z_cg(tmp, dst_klass_addr); }
2263       __ branch_optimized(Assembler::bcondNotEqual, halt);
2264       if (UseCompressedClassPointers)         { __ z_c (tmp, src_klass_addr); }
2265       else                                    { __ z_cg(tmp, src_klass_addr); }
2266       __ branch_optimized(Assembler::bcondEqual, known_ok);
2267     } else {
2268       if (UseCompressedClassPointers)         { __ z_c (tmp, dst_klass_addr); }
2269       else                                    { __ z_cg(tmp, dst_klass_addr); }
2270       __ branch_optimized(Assembler::bcondEqual, known_ok);
2271       __ compareU64_and_branch(src, dst, Assembler::bcondEqual, known_ok);
2272     }
2273     __ bind(halt);
2274     __ stop("incorrect type information in arraycopy");
2275     __ bind(known_ok);
2276   }
2277 #endif
2278 
2279 #ifndef PRODUCT
2280   if (PrintC1Statistics) {
2281     __ load_const_optimized(Z_R1_scratch, Runtime1::arraycopy_count_address(basic_type));
2282     __ add2mem_32(Address(Z_R1_scratch), 1, Z_R0_scratch);
2283   }
2284 #endif
2285 
2286   __ z_sllg(tmp, src_pos, shift_amount); // index -> byte offset
2287   __ z_sllg(Z_R1_scratch, dst_pos, shift_amount); // index -> byte offset
2288 
2289   assert_different_registers(Z_ARG1, dst, dst_pos, length);
2290   __ z_la(Z_ARG1, Address(src, tmp, arrayOopDesc::base_offset_in_bytes(basic_type)));
2291   assert_different_registers(Z_ARG2, length);
2292   __ z_la(Z_ARG2, Address(dst, Z_R1_scratch, arrayOopDesc::base_offset_in_bytes(basic_type)));
2293   __ lgr_if_needed(Z_ARG3, length);
2294 
2295   bool disjoint = (flags & LIR_OpArrayCopy::overlapping) == 0;
2296   bool aligned = (flags & LIR_OpArrayCopy::unaligned) == 0;
2297   const char *name;
2298   address entry = StubRoutines::select_arraycopy_function(basic_type, aligned, disjoint, name, false);
2299   __ call_VM_leaf(entry);
2300 
2301   __ bind(*stub->continuation());
2302 }
2303 
2304 void LIR_Assembler::shift_op(LIR_Code code, LIR_Opr left, LIR_Opr count, LIR_Opr dest, LIR_Opr tmp) {
2305   if (dest->is_single_cpu()) {
2306     if (left->type() == T_OBJECT) {
2307       switch (code) {
2308         case lir_shl:  __ z_sllg (dest->as_register(), left->as_register(), 0, count->as_register()); break;
2309         case lir_shr:  __ z_srag (dest->as_register(), left->as_register(), 0, count->as_register()); break;
2310         case lir_ushr: __ z_srlg (dest->as_register(), left->as_register(), 0, count->as_register()); break;
2311         default: ShouldNotReachHere();
2312       }
2313     } else {
2314       assert(code == lir_shl || left == dest, "left and dest must be equal for 2 operand form right shifts");
2315       Register masked_count = Z_R1_scratch;
2316       __ z_lr(masked_count, count->as_register());
2317       __ z_nill(masked_count, 31);
2318       switch (code) {
2319         case lir_shl:  __ z_sllg (dest->as_register(), left->as_register(), 0, masked_count); break;
2320         case lir_shr:  __ z_sra  (dest->as_register(), 0, masked_count); break;
2321         case lir_ushr: __ z_srl  (dest->as_register(), 0, masked_count); break;
2322         default: ShouldNotReachHere();
2323       }
2324     }
2325   } else {
2326     switch (code) {
2327       case lir_shl:  __ z_sllg (dest->as_register_lo(), left->as_register_lo(), 0, count->as_register()); break;
2328       case lir_shr:  __ z_srag (dest->as_register_lo(), left->as_register_lo(), 0, count->as_register()); break;
2329       case lir_ushr: __ z_srlg (dest->as_register_lo(), left->as_register_lo(), 0, count->as_register()); break;
2330       default: ShouldNotReachHere();
2331     }
2332   }
2333 }
2334 
2335 void LIR_Assembler::shift_op(LIR_Code code, LIR_Opr left, jint count, LIR_Opr dest) {
2336   if (left->type() == T_OBJECT) {
2337     count = count & 63;  // Shouldn't shift by more than sizeof(intptr_t).
2338     Register l = left->as_register();
2339     Register d = dest->as_register_lo();
2340     switch (code) {
2341       case lir_shl:  __ z_sllg (d, l, count); break;
2342       case lir_shr:  __ z_srag (d, l, count); break;
2343       case lir_ushr: __ z_srlg (d, l, count); break;
2344       default: ShouldNotReachHere();
2345     }
2346     return;
2347   }
2348   if (dest->is_single_cpu()) {
2349     assert(code == lir_shl || left == dest, "left and dest must be equal for 2 operand form right shifts");
2350     count = count & 0x1F; // Java spec
2351     switch (code) {
2352       case lir_shl:  __ z_sllg (dest->as_register(), left->as_register(), count); break;
2353       case lir_shr:  __ z_sra  (dest->as_register(), count); break;
2354       case lir_ushr: __ z_srl  (dest->as_register(), count); break;
2355       default: ShouldNotReachHere();
2356     }
2357   } else if (dest->is_double_cpu()) {
2358     count = count & 63; // Java spec
2359     Register l = left->as_pointer_register();
2360     Register d = dest->as_pointer_register();
2361     switch (code) {
2362       case lir_shl:  __ z_sllg (d, l, count); break;
2363       case lir_shr:  __ z_srag (d, l, count); break;
2364       case lir_ushr: __ z_srlg (d, l, count); break;
2365       default: ShouldNotReachHere();
2366     }
2367   } else {
2368     ShouldNotReachHere();
2369   }
2370 }
2371 
2372 void LIR_Assembler::emit_alloc_obj(LIR_OpAllocObj* op) {
2373   if (op->init_check()) {
2374     // Make sure klass is initialized & doesn't have finalizer.
2375     const int state_offset = in_bytes(InstanceKlass::init_state_offset());
2376     Register iklass = op->klass()->as_register();
2377     add_debug_info_for_null_check_here(op->stub()->info());
2378     if (Immediate::is_uimm12(state_offset)) {
2379       __ z_cli(state_offset, iklass, InstanceKlass::fully_initialized);
2380     } else {
2381       __ z_cliy(state_offset, iklass, InstanceKlass::fully_initialized);
2382     }
2383     __ branch_optimized(Assembler::bcondNotEqual, *op->stub()->entry()); // Use long branch, because slow_case might be far.
2384   }
2385   __ allocate_object(op->obj()->as_register(),
2386                      op->tmp1()->as_register(),
2387                      op->tmp2()->as_register(),
2388                      op->header_size(),
2389                      op->object_size(),
2390                      op->klass()->as_register(),
2391                      *op->stub()->entry());
2392   __ bind(*op->stub()->continuation());
2393   __ verify_oop(op->obj()->as_register());
2394 }
2395 
2396 void LIR_Assembler::emit_alloc_array(LIR_OpAllocArray* op) {
2397   Register len = op->len()->as_register();
2398   __ move_reg_if_needed(len, T_LONG, len, T_INT); // sign extend
2399 
2400   if (UseSlowPath ||
2401       (!UseFastNewObjectArray && (op->type() == T_OBJECT || op->type() == T_ARRAY)) ||
2402       (!UseFastNewTypeArray   && (op->type() != T_OBJECT && op->type() != T_ARRAY))) {
2403     __ z_brul(*op->stub()->entry());
2404   } else {
2405     __ allocate_array(op->obj()->as_register(),
2406                       op->len()->as_register(),
2407                       op->tmp1()->as_register(),
2408                       op->tmp2()->as_register(),
2409                       arrayOopDesc::header_size(op->type()),
2410                       type2aelembytes(op->type()),
2411                       op->klass()->as_register(),
2412                       *op->stub()->entry());
2413   }
2414   __ bind(*op->stub()->continuation());
2415 }
2416 
2417 void LIR_Assembler::type_profile_helper(Register mdo, ciMethodData *md, ciProfileData *data,
2418                                         Register recv, Register tmp1, Label* update_done) {
2419   uint i;
2420   for (i = 0; i < VirtualCallData::row_limit(); i++) {
2421     Label next_test;
2422     // See if the receiver is receiver[n].
2423     Address receiver_addr(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_offset(i)));
2424     __ z_cg(recv, receiver_addr);
2425     __ z_brne(next_test);
2426     Address data_addr(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_count_offset(i)));
2427     __ add2mem_64(data_addr, DataLayout::counter_increment, tmp1);
2428     __ branch_optimized(Assembler::bcondAlways, *update_done);
2429     __ bind(next_test);
2430   }
2431 
2432   // Didn't find receiver; find next empty slot and fill it in.
2433   for (i = 0; i < VirtualCallData::row_limit(); i++) {
2434     Label next_test;
2435     Address recv_addr(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_offset(i)));
2436     __ z_ltg(Z_R0_scratch, recv_addr);
2437     __ z_brne(next_test);
2438     __ z_stg(recv, recv_addr);
2439     __ load_const_optimized(tmp1, DataLayout::counter_increment);
2440     __ z_stg(tmp1, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_count_offset(i)), mdo);
2441     __ branch_optimized(Assembler::bcondAlways, *update_done);
2442     __ bind(next_test);
2443   }
2444 }
2445 
2446 void LIR_Assembler::setup_md_access(ciMethod* method, int bci,
2447                                     ciMethodData*& md, ciProfileData*& data, int& mdo_offset_bias) {
2448   Unimplemented();
2449 }
2450 
2451 void LIR_Assembler::store_parameter(Register r, int param_num) {
2452   assert(param_num >= 0, "invalid num");
2453   int offset_in_bytes = param_num * BytesPerWord + FrameMap::first_available_sp_in_frame;
2454   assert(offset_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2455   __ z_stg(r, offset_in_bytes, Z_SP);
2456 }
2457 
2458 void LIR_Assembler::store_parameter(jint c, int param_num) {
2459   assert(param_num >= 0, "invalid num");
2460   int offset_in_bytes = param_num * BytesPerWord + FrameMap::first_available_sp_in_frame;
2461   assert(offset_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2462   __ store_const(Address(Z_SP, offset_in_bytes), c, Z_R1_scratch, true);
2463 }
2464 
2465 void LIR_Assembler::emit_typecheck_helper(LIR_OpTypeCheck *op, Label* success, Label* failure, Label* obj_is_null) {
2466   // We always need a stub for the failure case.
2467   CodeStub* stub = op->stub();
2468   Register obj = op->object()->as_register();
2469   Register k_RInfo = op->tmp1()->as_register();
2470   Register klass_RInfo = op->tmp2()->as_register();
2471   Register dst = op->result_opr()->as_register();
2472   Register Rtmp1 = Z_R1_scratch;
2473   ciKlass* k = op->klass();
2474 
2475   assert(!op->tmp3()->is_valid(), "tmp3's not needed");
2476 
2477   // Check if it needs to be profiled.
2478   ciMethodData* md = NULL;
2479   ciProfileData* data = NULL;
2480 
2481   if (op->should_profile()) {
2482     ciMethod* method = op->profiled_method();
2483     assert(method != NULL, "Should have method");
2484     int bci = op->profiled_bci();
2485     md = method->method_data_or_null();
2486     assert(md != NULL, "Sanity");
2487     data = md->bci_to_data(bci);
2488     assert(data != NULL,                "need data for type check");
2489     assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
2490   }
2491 
2492   // Temp operands do not overlap with inputs, if this is their last
2493   // use (end of range is exclusive), so a register conflict is possible.
2494   if (obj == k_RInfo) {
2495     k_RInfo = dst;
2496   } else if (obj == klass_RInfo) {
2497     klass_RInfo = dst;
2498   }
2499   assert_different_registers(obj, k_RInfo, klass_RInfo);
2500 
2501   if (op->should_profile()) {
2502     NearLabel not_null;
2503     __ compareU64_and_branch(obj, (intptr_t) 0, Assembler::bcondNotEqual, not_null);
2504     // Object is null; update MDO and exit.
2505     Register mdo = klass_RInfo;
2506     metadata2reg(md->constant_encoding(), mdo);
2507     Address data_addr(mdo, md->byte_offset_of_slot(data, DataLayout::header_offset()));
2508     int header_bits = DataLayout::flag_mask_to_header_mask(BitData::null_seen_byte_constant());
2509     __ or2mem_8(data_addr, header_bits);
2510     __ branch_optimized(Assembler::bcondAlways, *obj_is_null);
2511     __ bind(not_null);
2512   } else {
2513     __ compareU64_and_branch(obj, (intptr_t) 0, Assembler::bcondEqual, *obj_is_null);
2514   }
2515 
2516   NearLabel profile_cast_failure, profile_cast_success;
2517   Label *failure_target = op->should_profile() ? &profile_cast_failure : failure;
2518   Label *success_target = op->should_profile() ? &profile_cast_success : success;
2519 
2520   // Patching may screw with our temporaries on sparc,
2521   // so let's do it before loading the class.
2522   if (k->is_loaded()) {
2523     metadata2reg(k->constant_encoding(), k_RInfo);
2524   } else {
2525     klass2reg_with_patching(k_RInfo, op->info_for_patch());
2526   }
2527   assert(obj != k_RInfo, "must be different");
2528 
2529   __ verify_oop(obj);
2530 
2531   // Get object class.
2532   // Not a safepoint as obj null check happens earlier.
2533   if (op->fast_check()) {
2534     if (UseCompressedClassPointers) {
2535       __ load_klass(klass_RInfo, obj);
2536       __ compareU64_and_branch(k_RInfo, klass_RInfo, Assembler::bcondNotEqual, *failure_target);
2537     } else {
2538       __ z_cg(k_RInfo, Address(obj, oopDesc::klass_offset_in_bytes()));
2539       __ branch_optimized(Assembler::bcondNotEqual, *failure_target);
2540     }
2541     // Successful cast, fall through to profile or jump.
2542   } else {
2543     bool need_slow_path = !k->is_loaded() ||
2544                           ((int) k->super_check_offset() == in_bytes(Klass::secondary_super_cache_offset()));
2545     intptr_t super_check_offset = k->is_loaded() ? k->super_check_offset() : -1L;
2546     __ load_klass(klass_RInfo, obj);
2547     // Perform the fast part of the checking logic.
2548     __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1,
2549                                      (need_slow_path ? success_target : NULL),
2550                                      failure_target, NULL,
2551                                      RegisterOrConstant(super_check_offset));
2552     if (need_slow_path) {
2553       // Call out-of-line instance of __ check_klass_subtype_slow_path(...):
2554       address a = Runtime1::entry_for (Runtime1::slow_subtype_check_id);
2555       store_parameter(klass_RInfo, 0); // sub
2556       store_parameter(k_RInfo, 1);     // super
2557       emit_call_c(a); // Sets condition code 0 for match (2 otherwise).
2558       CHECK_BAILOUT();
2559       __ branch_optimized(Assembler::bcondNotEqual, *failure_target);
2560       // Fall through to success case.
2561     }
2562   }
2563 
2564   if (op->should_profile()) {
2565     Register mdo = klass_RInfo, recv = k_RInfo;
2566     assert_different_registers(obj, mdo, recv);
2567     __ bind(profile_cast_success);
2568     metadata2reg(md->constant_encoding(), mdo);
2569     __ load_klass(recv, obj);
2570     type_profile_helper(mdo, md, data, recv, Rtmp1, success);
2571     __ branch_optimized(Assembler::bcondAlways, *success);
2572 
2573     __ bind(profile_cast_failure);
2574     metadata2reg(md->constant_encoding(), mdo);
2575     __ add2mem_64(Address(mdo, md->byte_offset_of_slot(data, CounterData::count_offset())), -(int)DataLayout::counter_increment, Rtmp1);
2576     __ branch_optimized(Assembler::bcondAlways, *failure);
2577   } else {
2578     __ branch_optimized(Assembler::bcondAlways, *success);
2579   }
2580 }
2581 
2582 void LIR_Assembler::emit_opTypeCheck(LIR_OpTypeCheck* op) {
2583   LIR_Code code = op->code();
2584   if (code == lir_store_check) {
2585     Register value = op->object()->as_register();
2586     Register array = op->array()->as_register();
2587     Register k_RInfo = op->tmp1()->as_register();
2588     Register klass_RInfo = op->tmp2()->as_register();
2589     Register Rtmp1 = Z_R1_scratch;
2590 
2591     CodeStub* stub = op->stub();
2592 
2593     // Check if it needs to be profiled.
2594     ciMethodData* md = NULL;
2595     ciProfileData* data = NULL;
2596 
2597     assert_different_registers(value, k_RInfo, klass_RInfo);
2598 
2599     if (op->should_profile()) {
2600       ciMethod* method = op->profiled_method();
2601       assert(method != NULL, "Should have method");
2602       int bci = op->profiled_bci();
2603       md = method->method_data_or_null();
2604       assert(md != NULL, "Sanity");
2605       data = md->bci_to_data(bci);
2606       assert(data != NULL,                "need data for type check");
2607       assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
2608     }
2609     NearLabel profile_cast_success, profile_cast_failure, done;
2610     Label *success_target = op->should_profile() ? &profile_cast_success : &done;
2611     Label *failure_target = op->should_profile() ? &profile_cast_failure : stub->entry();
2612 
2613     if (op->should_profile()) {
2614       NearLabel not_null;
2615       __ compareU64_and_branch(value, (intptr_t) 0, Assembler::bcondNotEqual, not_null);
2616       // Object is null; update MDO and exit.
2617       Register mdo = klass_RInfo;
2618       metadata2reg(md->constant_encoding(), mdo);
2619       Address data_addr(mdo, md->byte_offset_of_slot(data, DataLayout::header_offset()));
2620       int header_bits = DataLayout::flag_mask_to_header_mask(BitData::null_seen_byte_constant());
2621       __ or2mem_8(data_addr, header_bits);
2622       __ branch_optimized(Assembler::bcondAlways, done);
2623       __ bind(not_null);
2624     } else {
2625       __ compareU64_and_branch(value, (intptr_t) 0, Assembler::bcondEqual, done);
2626     }
2627 
2628     add_debug_info_for_null_check_here(op->info_for_exception());
2629     __ load_klass(k_RInfo, array);
2630     __ load_klass(klass_RInfo, value);
2631 
2632     // Get instance klass (it's already uncompressed).
2633     __ z_lg(k_RInfo, Address(k_RInfo, ObjArrayKlass::element_klass_offset()));
2634     // Perform the fast part of the checking logic.
2635     __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, NULL);
2636     // Call out-of-line instance of __ check_klass_subtype_slow_path(...):
2637     address a = Runtime1::entry_for (Runtime1::slow_subtype_check_id);
2638     store_parameter(klass_RInfo, 0); // sub
2639     store_parameter(k_RInfo, 1);     // super
2640     emit_call_c(a); // Sets condition code 0 for match (2 otherwise).
2641     CHECK_BAILOUT();
2642     __ branch_optimized(Assembler::bcondNotEqual, *failure_target);
2643     // Fall through to success case.
2644 
2645     if (op->should_profile()) {
2646       Register mdo = klass_RInfo, recv = k_RInfo;
2647       assert_different_registers(value, mdo, recv);
2648       __ bind(profile_cast_success);
2649       metadata2reg(md->constant_encoding(), mdo);
2650       __ load_klass(recv, value);
2651       type_profile_helper(mdo, md, data, recv, Rtmp1, &done);
2652       __ branch_optimized(Assembler::bcondAlways, done);
2653 
2654       __ bind(profile_cast_failure);
2655       metadata2reg(md->constant_encoding(), mdo);
2656       __ add2mem_64(Address(mdo, md->byte_offset_of_slot(data, CounterData::count_offset())), -(int)DataLayout::counter_increment, Rtmp1);
2657       __ branch_optimized(Assembler::bcondAlways, *stub->entry());
2658     }
2659 
2660     __ bind(done);
2661   } else {
2662     if (code == lir_checkcast) {
2663       Register obj = op->object()->as_register();
2664       Register dst = op->result_opr()->as_register();
2665       NearLabel success;
2666       emit_typecheck_helper(op, &success, op->stub()->entry(), &success);
2667       __ bind(success);
2668       __ lgr_if_needed(dst, obj);
2669     } else {
2670       if (code == lir_instanceof) {
2671         Register obj = op->object()->as_register();
2672         Register dst = op->result_opr()->as_register();
2673         NearLabel success, failure, done;
2674         emit_typecheck_helper(op, &success, &failure, &failure);
2675         __ bind(failure);
2676         __ clear_reg(dst);
2677         __ branch_optimized(Assembler::bcondAlways, done);
2678         __ bind(success);
2679         __ load_const_optimized(dst, 1);
2680         __ bind(done);
2681       } else {
2682         ShouldNotReachHere();
2683       }
2684     }
2685   }
2686 }
2687 
2688 void LIR_Assembler::emit_compare_and_swap(LIR_OpCompareAndSwap* op) {
2689   Register addr = op->addr()->as_pointer_register();
2690   Register t1_cmp = Z_R1_scratch;
2691   if (op->code() == lir_cas_long) {
2692     assert(VM_Version::supports_cx8(), "wrong machine");
2693     Register cmp_value_lo = op->cmp_value()->as_register_lo();
2694     Register new_value_lo = op->new_value()->as_register_lo();
2695     __ z_lgr(t1_cmp, cmp_value_lo);
2696     // Perform the compare and swap operation.
2697     __ z_csg(t1_cmp, new_value_lo, 0, addr);
2698   } else if (op->code() == lir_cas_int || op->code() == lir_cas_obj) {
2699     Register cmp_value = op->cmp_value()->as_register();
2700     Register new_value = op->new_value()->as_register();
2701     if (op->code() == lir_cas_obj) {
2702       if (UseCompressedOops) {
2703                  t1_cmp = op->tmp1()->as_register();
2704         Register t2_new = op->tmp2()->as_register();
2705         assert_different_registers(cmp_value, new_value, addr, t1_cmp, t2_new);
2706         __ oop_encoder(t1_cmp, cmp_value, true /*maybe null*/);
2707         __ oop_encoder(t2_new, new_value, true /*maybe null*/);
2708         __ z_cs(t1_cmp, t2_new, 0, addr);
2709       } else {
2710         __ z_lgr(t1_cmp, cmp_value);
2711         __ z_csg(t1_cmp, new_value, 0, addr);
2712       }
2713     } else {
2714       __ z_lr(t1_cmp, cmp_value);
2715       __ z_cs(t1_cmp, new_value, 0, addr);
2716     }
2717   } else {
2718     ShouldNotReachHere(); // new lir_cas_??
2719   }
2720 }
2721 
2722 void LIR_Assembler::set_24bit_FPU() {
2723   ShouldNotCallThis(); // x86 only
2724 }
2725 
2726 void LIR_Assembler::reset_FPU() {
2727   ShouldNotCallThis(); // x86 only
2728 }
2729 
2730 void LIR_Assembler::breakpoint() {
2731   Unimplemented();
2732   //  __ breakpoint_trap();
2733 }
2734 
2735 void LIR_Assembler::push(LIR_Opr opr) {
2736   ShouldNotCallThis(); // unused
2737 }
2738 
2739 void LIR_Assembler::pop(LIR_Opr opr) {
2740   ShouldNotCallThis(); // unused
2741 }
2742 
2743 void LIR_Assembler::monitor_address(int monitor_no, LIR_Opr dst_opr) {
2744   Address addr = frame_map()->address_for_monitor_lock(monitor_no);
2745   __ add2reg(dst_opr->as_register(), addr.disp(), addr.base());
2746 }
2747 
2748 void LIR_Assembler::emit_lock(LIR_OpLock* op) {
2749   Register obj = op->obj_opr()->as_register();  // May not be an oop.
2750   Register hdr = op->hdr_opr()->as_register();
2751   Register lock = op->lock_opr()->as_register();
2752   if (!UseFastLocking) {
2753     __ branch_optimized(Assembler::bcondAlways, *op->stub()->entry());
2754   } else if (op->code() == lir_lock) {
2755     assert(BasicLock::displaced_header_offset_in_bytes() == 0, "lock_reg must point to the displaced header");
2756     // Add debug info for NullPointerException only if one is possible.
2757     if (op->info() != NULL) {
2758       add_debug_info_for_null_check_here(op->info());
2759     }
2760     __ lock_object(hdr, obj, lock, *op->stub()->entry());
2761     // done
2762   } else if (op->code() == lir_unlock) {
2763     assert(BasicLock::displaced_header_offset_in_bytes() == 0, "lock_reg must point to the displaced header");
2764     __ unlock_object(hdr, obj, lock, *op->stub()->entry());
2765   } else {
2766     ShouldNotReachHere();
2767   }
2768   __ bind(*op->stub()->continuation());
2769 }
2770 
2771 void LIR_Assembler::emit_profile_call(LIR_OpProfileCall* op) {
2772   ciMethod* method = op->profiled_method();
2773   int bci          = op->profiled_bci();
2774   ciMethod* callee = op->profiled_callee();
2775 
2776   // Update counter for all call types.
2777   ciMethodData* md = method->method_data_or_null();
2778   assert(md != NULL, "Sanity");
2779   ciProfileData* data = md->bci_to_data(bci);
2780   assert(data != NULL && data->is_CounterData(), "need CounterData for calls");
2781   assert(op->mdo()->is_single_cpu(),  "mdo must be allocated");
2782   Register mdo  = op->mdo()->as_register();
2783   assert(op->tmp1()->is_double_cpu(), "tmp1 must be allocated");
2784   Register tmp1 = op->tmp1()->as_register_lo();
2785   metadata2reg(md->constant_encoding(), mdo);
2786 
2787   Address counter_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
2788   // Perform additional virtual call profiling for invokevirtual and
2789   // invokeinterface bytecodes
2790   if (op->should_profile_receiver_type()) {
2791     assert(op->recv()->is_single_cpu(), "recv must be allocated");
2792     Register recv = op->recv()->as_register();
2793     assert_different_registers(mdo, tmp1, recv);
2794     assert(data->is_VirtualCallData(), "need VirtualCallData for virtual calls");
2795     ciKlass* known_klass = op->known_holder();
2796     if (C1OptimizeVirtualCallProfiling && known_klass != NULL) {
2797       // We know the type that will be seen at this call site; we can
2798       // statically update the MethodData* rather than needing to do
2799       // dynamic tests on the receiver type.
2800 
2801       // NOTE: we should probably put a lock around this search to
2802       // avoid collisions by concurrent compilations.
2803       ciVirtualCallData* vc_data = (ciVirtualCallData*) data;
2804       uint i;
2805       for (i = 0; i < VirtualCallData::row_limit(); i++) {
2806         ciKlass* receiver = vc_data->receiver(i);
2807         if (known_klass->equals(receiver)) {
2808           Address data_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_count_offset(i)));
2809           __ add2mem_64(data_addr, DataLayout::counter_increment, tmp1);
2810           return;
2811         }
2812       }
2813 
2814       // Receiver type not found in profile data. Select an empty slot.
2815 
2816       // Note that this is less efficient than it should be because it
2817       // always does a write to the receiver part of the
2818       // VirtualCallData rather than just the first time.
2819       for (i = 0; i < VirtualCallData::row_limit(); i++) {
2820         ciKlass* receiver = vc_data->receiver(i);
2821         if (receiver == NULL) {
2822           Address recv_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_offset(i)));
2823           metadata2reg(known_klass->constant_encoding(), tmp1);
2824           __ z_stg(tmp1, recv_addr);
2825           Address data_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_count_offset(i)));
2826           __ add2mem_64(data_addr, DataLayout::counter_increment, tmp1);
2827           return;
2828         }
2829       }
2830     } else {
2831       __ load_klass(recv, recv);
2832       NearLabel update_done;
2833       type_profile_helper(mdo, md, data, recv, tmp1, &update_done);
2834       // Receiver did not match any saved receiver and there is no empty row for it.
2835       // Increment total counter to indicate polymorphic case.
2836       __ add2mem_64(counter_addr, DataLayout::counter_increment, tmp1);
2837       __ bind(update_done);
2838     }
2839   } else {
2840     // static call
2841     __ add2mem_64(counter_addr, DataLayout::counter_increment, tmp1);
2842   }
2843 }
2844 
2845 void LIR_Assembler::align_backward_branch_target() {
2846   __ align(OptoLoopAlignment);
2847 }
2848 
2849 void LIR_Assembler::emit_delay(LIR_OpDelay* op) {
2850   ShouldNotCallThis(); // There are no delay slots on ZARCH_64.
2851 }
2852 
2853 void LIR_Assembler::negate(LIR_Opr left, LIR_Opr dest, LIR_Opr tmp) {
2854   // tmp must be unused
2855   assert(tmp->is_illegal(), "wasting a register if tmp is allocated");
2856   assert(left->is_register(), "can only handle registers");
2857 
2858   if (left->is_single_cpu()) {
2859     __ z_lcr(dest->as_register(), left->as_register());
2860   } else if (left->is_single_fpu()) {
2861     __ z_lcebr(dest->as_float_reg(), left->as_float_reg());
2862   } else if (left->is_double_fpu()) {
2863     __ z_lcdbr(dest->as_double_reg(), left->as_double_reg());
2864   } else {
2865     assert(left->is_double_cpu(), "Must be a long");
2866     __ z_lcgr(dest->as_register_lo(), left->as_register_lo());
2867   }
2868 }
2869 
2870 void LIR_Assembler::fxch(int i) {
2871   ShouldNotCallThis(); // x86 only
2872 }
2873 
2874 void LIR_Assembler::fld(int i) {
2875   ShouldNotCallThis(); // x86 only
2876 }
2877 
2878 void LIR_Assembler::ffree(int i) {
2879   ShouldNotCallThis(); // x86 only
2880 }
2881 
2882 void LIR_Assembler::rt_call(LIR_Opr result, address dest,
2883                             const LIR_OprList* args, LIR_Opr tmp, CodeEmitInfo* info) {
2884   assert(!tmp->is_valid(), "don't need temporary");
2885   emit_call_c(dest);
2886   CHECK_BAILOUT();
2887   if (info != NULL) {
2888     add_call_info_here(info);
2889   }
2890 }
2891 
2892 void LIR_Assembler::volatile_move_op(LIR_Opr src, LIR_Opr dest, BasicType type, CodeEmitInfo* info) {
2893   ShouldNotCallThis(); // not needed on ZARCH_64
2894 }
2895 
2896 void LIR_Assembler::membar() {
2897   __ z_fence();
2898 }
2899 
2900 void LIR_Assembler::membar_acquire() {
2901   __ z_acquire();
2902 }
2903 
2904 void LIR_Assembler::membar_release() {
2905   __ z_release();
2906 }
2907 
2908 void LIR_Assembler::membar_loadload() {
2909   __ z_acquire();
2910 }
2911 
2912 void LIR_Assembler::membar_storestore() {
2913   __ z_release();
2914 }
2915 
2916 void LIR_Assembler::membar_loadstore() {
2917   __ z_acquire();
2918 }
2919 
2920 void LIR_Assembler::membar_storeload() {
2921   __ z_fence();
2922 }
2923 
2924 void LIR_Assembler::on_spin_wait() {
2925   Unimplemented();
2926 }
2927 
2928 void LIR_Assembler::leal(LIR_Opr addr_opr, LIR_Opr dest, LIR_PatchCode patch_code, CodeEmitInfo* info) {
2929   assert(patch_code == lir_patch_none, "Patch code not supported");
2930   LIR_Address* addr = addr_opr->as_address_ptr();
2931   assert(addr->scale() == LIR_Address::times_1, "scaling unsupported");
2932   __ load_address(dest->as_pointer_register(), as_Address(addr));
2933 }
2934 
2935 void LIR_Assembler::get_thread(LIR_Opr result_reg) {
2936   ShouldNotCallThis(); // unused
2937 }
2938 
2939 #ifdef ASSERT
2940 // Emit run-time assertion.
2941 void LIR_Assembler::emit_assert(LIR_OpAssert* op) {
2942   Unimplemented();
2943 }
2944 #endif
2945 
2946 void LIR_Assembler::peephole(LIR_List*) {
2947   // Do nothing for now.
2948 }
2949 
2950 void LIR_Assembler::atomic_op(LIR_Code code, LIR_Opr src, LIR_Opr data, LIR_Opr dest, LIR_Opr tmp) {
2951   assert(code == lir_xadd, "lir_xchg not supported");
2952   Address src_addr = as_Address(src->as_address_ptr());
2953   Register base = src_addr.base();
2954   intptr_t disp = src_addr.disp();
2955   if (src_addr.index()->is_valid()) {
2956     // LAA and LAAG do not support index register.
2957     __ load_address(Z_R1_scratch, src_addr);
2958     base = Z_R1_scratch;
2959     disp = 0;
2960   }
2961   if (data->type() == T_INT) {
2962     __ z_laa(dest->as_register(), data->as_register(), disp, base);
2963   } else if (data->type() == T_LONG) {
2964     assert(data->as_register_lo() == data->as_register_hi(), "should be a single register");
2965     __ z_laag(dest->as_register_lo(), data->as_register_lo(), disp, base);
2966   } else {
2967     ShouldNotReachHere();
2968   }
2969 }
2970 
2971 void LIR_Assembler::emit_profile_type(LIR_OpProfileType* op) {
2972   Register obj = op->obj()->as_register();
2973   Register tmp1 = op->tmp()->as_pointer_register();
2974   Register tmp2 = Z_R1_scratch;
2975   Address mdo_addr = as_Address(op->mdp()->as_address_ptr());
2976   ciKlass* exact_klass = op->exact_klass();
2977   intptr_t current_klass = op->current_klass();
2978   bool not_null = op->not_null();
2979   bool no_conflict = op->no_conflict();
2980 
2981   Label update, next, none, null_seen, init_klass;
2982 
2983   bool do_null = !not_null;
2984   bool exact_klass_set = exact_klass != NULL && ciTypeEntries::valid_ciklass(current_klass) == exact_klass;
2985   bool do_update = !TypeEntries::is_type_unknown(current_klass) && !exact_klass_set;
2986 
2987   assert(do_null || do_update, "why are we here?");
2988   assert(!TypeEntries::was_null_seen(current_klass) || do_update, "why are we here?");
2989 
2990   __ verify_oop(obj);
2991 
2992   if (do_null || tmp1 != obj DEBUG_ONLY(|| true)) {
2993     __ z_ltgr(tmp1, obj);
2994   }
2995   if (do_null) {
2996     __ z_brnz(update);
2997     if (!TypeEntries::was_null_seen(current_klass)) {
2998       __ z_lg(tmp1, mdo_addr);
2999       __ z_oill(tmp1, TypeEntries::null_seen);
3000       __ z_stg(tmp1, mdo_addr);
3001     }
3002     if (do_update) {
3003       __ z_bru(next);
3004     }
3005   } else {
3006     __ asm_assert_ne("unexpect null obj", __LINE__);
3007   }
3008 
3009   __ bind(update);
3010 
3011   if (do_update) {
3012 #ifdef ASSERT
3013     if (exact_klass != NULL) {
3014       __ load_klass(tmp1, tmp1);
3015       metadata2reg(exact_klass->constant_encoding(), tmp2);
3016       __ z_cgr(tmp1, tmp2);
3017       __ asm_assert_eq("exact klass and actual klass differ", __LINE__);
3018     }
3019 #endif
3020 
3021     Label do_update;
3022     __ z_lg(tmp2, mdo_addr);
3023 
3024     if (!no_conflict) {
3025       if (exact_klass == NULL || TypeEntries::is_type_none(current_klass)) {
3026         if (exact_klass != NULL) {
3027           metadata2reg(exact_klass->constant_encoding(), tmp1);
3028         } else {
3029           __ load_klass(tmp1, tmp1);
3030         }
3031 
3032         // Klass seen before: nothing to do (regardless of unknown bit).
3033         __ z_lgr(Z_R0_scratch, tmp2);
3034         assert(Immediate::is_uimm(~TypeEntries::type_klass_mask, 16), "or change following instruction");
3035         __ z_nill(Z_R0_scratch, TypeEntries::type_klass_mask & 0xFFFF);
3036         __ compareU64_and_branch(Z_R0_scratch, tmp1, Assembler::bcondEqual, next);
3037 
3038         // Already unknown: Nothing to do anymore.
3039         __ z_tmll(tmp2, TypeEntries::type_unknown);
3040         __ z_brc(Assembler::bcondAllOne, next);
3041 
3042         if (TypeEntries::is_type_none(current_klass)) {
3043           __ z_lgr(Z_R0_scratch, tmp2);
3044           assert(Immediate::is_uimm(~TypeEntries::type_mask, 16), "or change following instruction");
3045           __ z_nill(Z_R0_scratch, TypeEntries::type_mask & 0xFFFF);
3046           __ compareU64_and_branch(Z_R0_scratch, (intptr_t)0, Assembler::bcondEqual, init_klass);
3047         }
3048       } else {
3049         assert(ciTypeEntries::valid_ciklass(current_klass) != NULL &&
3050                ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "conflict only");
3051 
3052         // Already unknown: Nothing to do anymore.
3053         __ z_tmll(tmp2, TypeEntries::type_unknown);
3054         __ z_brc(Assembler::bcondAllOne, next);
3055       }
3056 
3057       // Different than before. Cannot keep accurate profile.
3058       __ z_oill(tmp2, TypeEntries::type_unknown);
3059       __ z_bru(do_update);
3060     } else {
3061       // There's a single possible klass at this profile point.
3062       assert(exact_klass != NULL, "should be");
3063       if (TypeEntries::is_type_none(current_klass)) {
3064         metadata2reg(exact_klass->constant_encoding(), tmp1);
3065         __ z_lgr(Z_R0_scratch, tmp2);
3066         assert(Immediate::is_uimm(~TypeEntries::type_klass_mask, 16), "or change following instruction");
3067         __ z_nill(Z_R0_scratch, TypeEntries::type_klass_mask & 0xFFFF);
3068         __ compareU64_and_branch(Z_R0_scratch, tmp1, Assembler::bcondEqual, next);
3069 #ifdef ASSERT
3070         {
3071           Label ok;
3072           __ z_lgr(Z_R0_scratch, tmp2);
3073           assert(Immediate::is_uimm(~TypeEntries::type_mask, 16), "or change following instruction");
3074           __ z_nill(Z_R0_scratch, TypeEntries::type_mask & 0xFFFF);
3075           __ compareU64_and_branch(Z_R0_scratch, (intptr_t)0, Assembler::bcondEqual, ok);
3076           __ stop("unexpected profiling mismatch");
3077           __ bind(ok);
3078         }
3079 #endif
3080 
3081       } else {
3082         assert(ciTypeEntries::valid_ciklass(current_klass) != NULL &&
3083                ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "inconsistent");
3084 
3085         // Already unknown: Nothing to do anymore.
3086         __ z_tmll(tmp2, TypeEntries::type_unknown);
3087         __ z_brc(Assembler::bcondAllOne, next);
3088         __ z_oill(tmp2, TypeEntries::type_unknown);
3089         __ z_bru(do_update);
3090       }
3091     }
3092 
3093     __ bind(init_klass);
3094     // Combine klass and null_seen bit (only used if (tmp & type_mask)==0).
3095     __ z_ogr(tmp2, tmp1);
3096 
3097     __ bind(do_update);
3098     __ z_stg(tmp2, mdo_addr);
3099 
3100     __ bind(next);
3101   }
3102 }
3103 
3104 void LIR_Assembler::emit_updatecrc32(LIR_OpUpdateCRC32* op) {
3105   assert(op->crc()->is_single_cpu(), "crc must be register");
3106   assert(op->val()->is_single_cpu(), "byte value must be register");
3107   assert(op->result_opr()->is_single_cpu(), "result must be register");
3108   Register crc = op->crc()->as_register();
3109   Register val = op->val()->as_register();
3110   Register res = op->result_opr()->as_register();
3111 
3112   assert_different_registers(val, crc, res);
3113 
3114   __ load_const_optimized(res, StubRoutines::crc_table_addr());
3115   __ kernel_crc32_singleByteReg(crc, val, res, true);
3116   __ z_lgfr(res, crc);
3117 }
3118 
3119 #undef __
--- EOF ---