1 //
   2 // Copyright (c) 2008, 2018, Oracle and/or its affiliates. All rights reserved.
   3 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4 //
   5 // This code is free software; you can redistribute it and/or modify it
   6 // under the terms of the GNU General Public License version 2 only, as
   7 // published by the Free Software Foundation.
   8 //
   9 // This code is distributed in the hope that it will be useful, but WITHOUT
  10 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11 // FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12 // version 2 for more details (a copy is included in the LICENSE file that
  13 // accompanied this code).
  14 //
  15 // You should have received a copy of the GNU General Public License version
  16 // 2 along with this work; if not, write to the Free Software Foundation,
  17 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18 //
  19 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20 // or visit www.oracle.com if you need additional information or have any
  21 // questions.
  22 //
  23 
  24 // ARM Architecture Description File
  25 
  26 //----------DEFINITION BLOCK---------------------------------------------------
  27 // Define name --> value mappings to inform the ADLC of an integer valued name
  28 // Current support includes integer values in the range [0, 0x7FFFFFFF]
  29 // Format:
  30 //        int_def  <name>         ( <int_value>, <expression>);
  31 // Generated Code in ad_<arch>.hpp
  32 //        #define  <name>   (<expression>)
  33 //        // value == <int_value>
  34 // Generated code in ad_<arch>.cpp adlc_verification()
  35 //        assert( <name> == <int_value>, "Expect (<expression>) to equal <int_value>");
  36 //
  37 definitions %{
  38 // The default cost (of an ALU instruction).
  39   int_def DEFAULT_COST      (    100,     100);
  40   int_def HUGE_COST         (1000000, 1000000);
  41 
  42 // Memory refs are twice as expensive as run-of-the-mill.
  43   int_def MEMORY_REF_COST   (    200, DEFAULT_COST * 2);
  44 
  45 // Branches are even more expensive.
  46   int_def BRANCH_COST       (    300, DEFAULT_COST * 3);
  47   int_def CALL_COST         (    300, DEFAULT_COST * 3);
  48 %}
  49 
  50 
  51 //----------SOURCE BLOCK-------------------------------------------------------
  52 // This is a block of C++ code which provides values, functions, and
  53 // definitions necessary in the rest of the architecture description
  54 source_hpp %{
  55 // Header information of the source block.
  56 // Method declarations/definitions which are used outside
  57 // the ad-scope can conveniently be defined here.
  58 //
  59 // To keep related declarations/definitions/uses close together,
  60 // we switch between source %{ }% and source_hpp %{ }% freely as needed.
  61 
  62 // Does destination need to be loaded in a register then passed to a
  63 // branch instruction?
  64 extern bool maybe_far_call(const CallNode *n);
  65 extern bool maybe_far_call(const MachCallNode *n);
  66 static inline bool cache_reachable() {
  67   return MacroAssembler::_cache_fully_reachable();
  68 }
  69 
  70 #ifdef AARCH64
  71 #define ldr_32 ldr_w
  72 #define str_32 str_w
  73 #else
  74 #define ldr_32 ldr
  75 #define str_32 str
  76 #define tst_32 tst
  77 #define teq_32 teq
  78 #endif
  79 #if 1
  80 extern bool PrintOptoAssembly;
  81 #endif
  82 
  83 class c2 {
  84 public:
  85   static OptoRegPair return_value(int ideal_reg);
  86 };
  87 
  88 class CallStubImpl {
  89 
  90   //--------------------------------------------------------------
  91   //---<  Used for optimization in Compile::Shorten_branches  >---
  92   //--------------------------------------------------------------
  93 
  94  public:
  95   // Size of call trampoline stub.
  96   static uint size_call_trampoline() {
  97     return 0; // no call trampolines on this platform
  98   }
  99 
 100   // number of relocations needed by a call trampoline stub
 101   static uint reloc_call_trampoline() {
 102     return 0; // no call trampolines on this platform
 103   }
 104 };
 105 
 106 class HandlerImpl {
 107 
 108  public:
 109 
 110   static int emit_exception_handler(CodeBuffer &cbuf);
 111   static int emit_deopt_handler(CodeBuffer& cbuf);
 112 
 113   static uint size_exception_handler() {
 114 #ifdef AARCH64
 115     // ldr_literal; br; (pad); <literal>
 116     return 3 * Assembler::InstructionSize + wordSize;
 117 #else
 118     return ( 3 * 4 );
 119 #endif
 120   }
 121 
 122 
 123   static uint size_deopt_handler() {
 124     return ( 9 * 4 );
 125   }
 126 
 127 };
 128 
 129 %}
 130 
 131 source %{
 132 #define __ _masm.
 133 
 134 static FloatRegister reg_to_FloatRegister_object(int register_encoding);
 135 static Register reg_to_register_object(int register_encoding);
 136 
 137 
 138 // ****************************************************************************
 139 
 140 // REQUIRED FUNCTIONALITY
 141 
 142 // Indicate if the safepoint node needs the polling page as an input.
 143 // Since ARM does not have absolute addressing, it does.
 144 bool SafePointNode::needs_polling_address_input() {
 145   return true;
 146 }
 147 
 148 // emit an interrupt that is caught by the debugger (for debugging compiler)
 149 void emit_break(CodeBuffer &cbuf) {
 150   MacroAssembler _masm(&cbuf);
 151   __ breakpoint();
 152 }
 153 
 154 #ifndef PRODUCT
 155 void MachBreakpointNode::format( PhaseRegAlloc *, outputStream *st ) const {
 156   st->print("TA");
 157 }
 158 #endif
 159 
 160 void MachBreakpointNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 161   emit_break(cbuf);
 162 }
 163 
 164 uint MachBreakpointNode::size(PhaseRegAlloc *ra_) const {
 165   return MachNode::size(ra_);
 166 }
 167 
 168 
 169 void emit_nop(CodeBuffer &cbuf) {
 170   MacroAssembler _masm(&cbuf);
 171   __ nop();
 172 }
 173 
 174 
 175 void emit_call_reloc(CodeBuffer &cbuf, const MachCallNode *n, MachOper *m, RelocationHolder const& rspec) {
 176   int ret_addr_offset0 = n->as_MachCall()->ret_addr_offset();
 177   int call_site_offset = cbuf.insts()->mark_off();
 178   MacroAssembler _masm(&cbuf);
 179   __ set_inst_mark(); // needed in emit_to_interp_stub() to locate the call
 180   address target = (address)m->method();
 181   assert(n->as_MachCall()->entry_point() == target, "sanity");
 182   assert(maybe_far_call(n) == !__ reachable_from_cache(target), "sanity");
 183   assert(cache_reachable() == __ cache_fully_reachable(), "sanity");
 184 
 185   assert(target != NULL, "need real address");
 186 
 187   int ret_addr_offset = -1;
 188   if (rspec.type() == relocInfo::runtime_call_type) {
 189     __ call(target, rspec);
 190     ret_addr_offset = __ offset();
 191   } else {
 192     // scratches Rtemp
 193     ret_addr_offset = __ patchable_call(target, rspec, true);
 194   }
 195   assert(ret_addr_offset - call_site_offset == ret_addr_offset0, "fix ret_addr_offset()");
 196 }
 197 
 198 //=============================================================================
 199 // REQUIRED FUNCTIONALITY for encoding
 200 void emit_lo(CodeBuffer &cbuf, int val) {  }
 201 void emit_hi(CodeBuffer &cbuf, int val) {  }
 202 
 203 
 204 //=============================================================================
 205 const RegMask& MachConstantBaseNode::_out_RegMask = PTR_REG_mask();
 206 
 207 int Compile::ConstantTable::calculate_table_base_offset() const {
 208 #ifdef AARCH64
 209   return 0;
 210 #else
 211   int offset = -(size() / 2);
 212   // flds, fldd: 8-bit  offset multiplied by 4: +/- 1024
 213   // ldr, ldrb : 12-bit offset:                 +/- 4096
 214   if (!Assembler::is_simm10(offset)) {
 215     offset = Assembler::min_simm10();
 216   }
 217   return offset;
 218 #endif
 219 }
 220 
 221 bool MachConstantBaseNode::requires_postalloc_expand() const { return false; }
 222 void MachConstantBaseNode::postalloc_expand(GrowableArray <Node *> *nodes, PhaseRegAlloc *ra_) {
 223   ShouldNotReachHere();
 224 }
 225 
 226 void MachConstantBaseNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const {
 227   Compile* C = ra_->C;
 228   Compile::ConstantTable& constant_table = C->constant_table();
 229   MacroAssembler _masm(&cbuf);
 230 
 231   Register r = as_Register(ra_->get_encode(this));
 232   CodeSection* consts_section = __ code()->consts();
 233   int consts_size = consts_section->align_at_start(consts_section->size());
 234   assert(constant_table.size() == consts_size, "must be: %d == %d", constant_table.size(), consts_size);
 235 
 236   // Materialize the constant table base.
 237   address baseaddr = consts_section->start() + -(constant_table.table_base_offset());
 238   RelocationHolder rspec = internal_word_Relocation::spec(baseaddr);
 239   __ mov_address(r, baseaddr, rspec);
 240 }
 241 
 242 uint MachConstantBaseNode::size(PhaseRegAlloc*) const {
 243 #ifdef AARCH64
 244   return 5 * Assembler::InstructionSize;
 245 #else
 246   return 8;
 247 #endif
 248 }
 249 
 250 #ifndef PRODUCT
 251 void MachConstantBaseNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
 252   char reg[128];
 253   ra_->dump_register(this, reg);
 254   st->print("MOV_SLOW    &constanttable,%s\t! constant table base", reg);
 255 }
 256 #endif
 257 
 258 #ifndef PRODUCT
 259 void MachPrologNode::format( PhaseRegAlloc *ra_, outputStream *st ) const {
 260   Compile* C = ra_->C;
 261 
 262   for (int i = 0; i < OptoPrologueNops; i++) {
 263     st->print_cr("NOP"); st->print("\t");
 264   }
 265 #ifdef AARCH64
 266   if (OptoPrologueNops <= 0) {
 267     st->print_cr("NOP\t! required for safe patching");
 268     st->print("\t");
 269   }
 270 #endif
 271 
 272   size_t framesize = C->frame_size_in_bytes();
 273   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
 274   int bangsize = C->bang_size_in_bytes();
 275   // Remove two words for return addr and rbp,
 276   framesize -= 2*wordSize;
 277   bangsize -= 2*wordSize;
 278 
 279   // Calls to C2R adapters often do not accept exceptional returns.
 280   // We require that their callers must bang for them.  But be careful, because
 281   // some VM calls (such as call site linkage) can use several kilobytes of
 282   // stack.  But the stack safety zone should account for that.
 283   // See bugs 4446381, 4468289, 4497237.
 284   if (C->need_stack_bang(bangsize)) {
 285     st->print_cr("! stack bang (%d bytes)", bangsize); st->print("\t");
 286   }
 287   st->print_cr("PUSH   R_FP|R_LR_LR"); st->print("\t");
 288   if (framesize != 0) {
 289     st->print   ("SUB    R_SP, R_SP, " SIZE_FORMAT,framesize);
 290   }
 291 }
 292 #endif
 293 
 294 void MachPrologNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 295   Compile* C = ra_->C;
 296   MacroAssembler _masm(&cbuf);
 297 
 298   for (int i = 0; i < OptoPrologueNops; i++) {
 299     __ nop();
 300   }
 301 #ifdef AARCH64
 302   if (OptoPrologueNops <= 0) {
 303     __ nop(); // required for safe patching by patch_verified_entry()
 304   }
 305 #endif
 306 
 307   size_t framesize = C->frame_size_in_bytes();
 308   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
 309   int bangsize = C->bang_size_in_bytes();
 310   // Remove two words for return addr and fp,
 311   framesize -= 2*wordSize;
 312   bangsize -= 2*wordSize;
 313 
 314   // Calls to C2R adapters often do not accept exceptional returns.
 315   // We require that their callers must bang for them.  But be careful, because
 316   // some VM calls (such as call site linkage) can use several kilobytes of
 317   // stack.  But the stack safety zone should account for that.
 318   // See bugs 4446381, 4468289, 4497237.
 319   if (C->need_stack_bang(bangsize)) {
 320     __ arm_stack_overflow_check(bangsize, Rtemp);
 321   }
 322 
 323   __ raw_push(FP, LR);
 324   if (framesize != 0) {
 325     __ sub_slow(SP, SP, framesize);
 326   }
 327 
 328   // offset from scratch buffer is not valid
 329   if (strcmp(cbuf.name(), "Compile::Fill_buffer") == 0) {
 330     C->set_frame_complete( __ offset() );
 331   }
 332 
 333   if (C->has_mach_constant_base_node()) {
 334     // NOTE: We set the table base offset here because users might be
 335     // emitted before MachConstantBaseNode.
 336     Compile::ConstantTable& constant_table = C->constant_table();
 337     constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
 338   }
 339 }
 340 
 341 uint MachPrologNode::size(PhaseRegAlloc *ra_) const {
 342   return MachNode::size(ra_);
 343 }
 344 
 345 int MachPrologNode::reloc() const {
 346   return 10; // a large enough number
 347 }
 348 
 349 //=============================================================================
 350 #ifndef PRODUCT
 351 void MachEpilogNode::format( PhaseRegAlloc *ra_, outputStream *st ) const {
 352   Compile* C = ra_->C;
 353 
 354   size_t framesize = C->frame_size_in_bytes();
 355   framesize -= 2*wordSize;
 356 
 357   if (framesize != 0) {
 358     st->print("ADD    R_SP, R_SP, " SIZE_FORMAT "\n\t",framesize);
 359   }
 360   st->print("POP    R_FP|R_LR_LR");
 361 
 362   if (do_polling() && ra_->C->is_method_compilation()) {
 363     st->print("\n\t");
 364 #ifdef AARCH64
 365     if (MacroAssembler::page_reachable_from_cache(os::get_polling_page())) {
 366       st->print("ADRP     Rtemp, #PollAddr\t! Load Polling address\n\t");
 367       st->print("LDR      ZR,[Rtemp + #PollAddr & 0xfff]\t!Poll for Safepointing");
 368     } else {
 369       st->print("mov_slow Rtemp, #PollAddr\t! Load Polling address\n\t");
 370       st->print("LDR      ZR,[Rtemp]\t!Poll for Safepointing");
 371     }
 372 #else
 373     st->print("MOV    Rtemp, #PollAddr\t! Load Polling address\n\t");
 374     st->print("LDR    Rtemp,[Rtemp]\t!Poll for Safepointing");
 375 #endif
 376   }
 377 }
 378 #endif
 379 
 380 void MachEpilogNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 381   MacroAssembler _masm(&cbuf);
 382   Compile* C = ra_->C;
 383 
 384   size_t framesize = C->frame_size_in_bytes();
 385   framesize -= 2*wordSize;
 386   if (framesize != 0) {
 387     __ add_slow(SP, SP, framesize);
 388   }
 389   __ raw_pop(FP, LR);
 390 
 391   // If this does safepoint polling, then do it here
 392   if (do_polling() && ra_->C->is_method_compilation()) {
 393 #ifdef AARCH64
 394     if (false && MacroAssembler::page_reachable_from_cache(os::get_polling_page())) {
 395 /* FIXME: TODO
 396       __ relocate(relocInfo::xxx);
 397       __ adrp(Rtemp, (intptr_t)os::get_polling_page());
 398       __ relocate(relocInfo::poll_return_type);
 399       int offset = os::get_polling_page() & 0xfff;
 400       __ ldr(ZR, Address(Rtemp + offset));
 401 */
 402     } else {
 403       __ mov_address(Rtemp, (address)os::get_polling_page(), symbolic_Relocation::polling_page_reference);
 404       __ relocate(relocInfo::poll_return_type);
 405       __ ldr(ZR, Address(Rtemp));
 406     }
 407 #else
 408     // mov_slow here is usually one or two instruction
 409     __ mov_address(Rtemp, (address)os::get_polling_page(), symbolic_Relocation::polling_page_reference);
 410     __ relocate(relocInfo::poll_return_type);
 411     __ ldr(Rtemp, Address(Rtemp));
 412 #endif
 413   }
 414 }
 415 
 416 uint MachEpilogNode::size(PhaseRegAlloc *ra_) const {
 417 #ifdef AARCH64
 418   // allow for added alignment nop from mov_address bind_literal
 419   return MachNode::size(ra_) + 1 * Assembler::InstructionSize;
 420 #else
 421   return MachNode::size(ra_);
 422 #endif
 423 }
 424 
 425 int MachEpilogNode::reloc() const {
 426   return 16; // a large enough number
 427 }
 428 
 429 const Pipeline * MachEpilogNode::pipeline() const {
 430   return MachNode::pipeline_class();
 431 }
 432 
 433 int MachEpilogNode::safepoint_offset() const {
 434   assert( do_polling(), "no return for this epilog node");
 435   //  return MacroAssembler::size_of_sethi(os::get_polling_page());
 436   Unimplemented();
 437   return 0;
 438 }
 439 
 440 //=============================================================================
 441 
 442 // Figure out which register class each belongs in: rc_int, rc_float, rc_stack
 443 enum RC { rc_bad, rc_int, rc_float, rc_stack };
 444 static enum RC rc_class( OptoReg::Name reg ) {
 445   if (!OptoReg::is_valid(reg)) return rc_bad;
 446   if (OptoReg::is_stack(reg)) return rc_stack;
 447   VMReg r = OptoReg::as_VMReg(reg);
 448   if (r->is_Register()) return rc_int;
 449   assert(r->is_FloatRegister(), "must be");
 450   return rc_float;
 451 }
 452 
 453 static inline bool is_iRegLd_memhd(OptoReg::Name src_first, OptoReg::Name src_second, int offset) {
 454 #ifdef AARCH64
 455   return is_memoryHD(offset);
 456 #else
 457   int rlo = Matcher::_regEncode[src_first];
 458   int rhi = Matcher::_regEncode[src_second];
 459   if (!((rlo&1)==0 && (rlo+1 == rhi))) {
 460     tty->print_cr("CAUGHT BAD LDRD/STRD");
 461   }
 462   return (rlo&1)==0 && (rlo+1 == rhi) && is_memoryHD(offset);
 463 #endif
 464 }
 465 
 466 uint MachSpillCopyNode::implementation( CodeBuffer *cbuf,
 467                                         PhaseRegAlloc *ra_,
 468                                         bool do_size,
 469                                         outputStream* st ) const {
 470   // Get registers to move
 471   OptoReg::Name src_second = ra_->get_reg_second(in(1));
 472   OptoReg::Name src_first = ra_->get_reg_first(in(1));
 473   OptoReg::Name dst_second = ra_->get_reg_second(this );
 474   OptoReg::Name dst_first = ra_->get_reg_first(this );
 475 
 476   enum RC src_second_rc = rc_class(src_second);
 477   enum RC src_first_rc = rc_class(src_first);
 478   enum RC dst_second_rc = rc_class(dst_second);
 479   enum RC dst_first_rc = rc_class(dst_first);
 480 
 481   assert( OptoReg::is_valid(src_first) && OptoReg::is_valid(dst_first), "must move at least 1 register" );
 482 
 483   // Generate spill code!
 484   int size = 0;
 485 
 486   if (src_first == dst_first && src_second == dst_second)
 487     return size;            // Self copy, no move
 488 
 489 #ifdef TODO
 490   if (bottom_type()->isa_vect() != NULL) {
 491   }
 492 #endif
 493 
 494   // Shared code does not expect instruction set capability based bailouts here.
 495   // Handle offset unreachable bailout with minimal change in shared code.
 496   // Bailout only for real instruction emit.
 497   // This requires a single comment change in shared code. ( see output.cpp "Normal" instruction case )
 498 
 499   MacroAssembler _masm(cbuf);
 500 
 501   // --------------------------------------
 502   // Check for mem-mem move.  Load into unused float registers and fall into
 503   // the float-store case.
 504   if (src_first_rc == rc_stack && dst_first_rc == rc_stack) {
 505     int offset = ra_->reg2offset(src_first);
 506     if (cbuf && !is_memoryfp(offset)) {
 507       ra_->C->record_method_not_compilable("unable to handle large constant offsets");
 508       return 0;
 509     } else {
 510       if (src_second_rc != rc_bad) {
 511         assert((src_first&1)==0 && src_first+1 == src_second, "pair of registers must be aligned/contiguous");
 512         src_first     = OptoReg::Name(R_mem_copy_lo_num);
 513         src_second    = OptoReg::Name(R_mem_copy_hi_num);
 514         src_first_rc  = rc_float;
 515         src_second_rc = rc_float;
 516         if (cbuf) {
 517           __ ldr_double(Rmemcopy, Address(SP, offset));
 518         } else if (!do_size) {
 519           st->print(LDR_DOUBLE "   R_%s,[R_SP + #%d]\t! spill",OptoReg::regname(src_first),offset);
 520         }
 521       } else {
 522         src_first     = OptoReg::Name(R_mem_copy_lo_num);
 523         src_first_rc  = rc_float;
 524         if (cbuf) {
 525           __ ldr_float(Rmemcopy, Address(SP, offset));
 526         } else if (!do_size) {
 527           st->print(LDR_FLOAT "   R_%s,[R_SP + #%d]\t! spill",OptoReg::regname(src_first),offset);
 528         }
 529       }
 530       size += 4;
 531     }
 532   }
 533 
 534   if (src_second_rc == rc_stack && dst_second_rc == rc_stack) {
 535     Unimplemented();
 536   }
 537 
 538   // --------------------------------------
 539   // Check for integer reg-reg copy
 540   if (src_first_rc == rc_int && dst_first_rc == rc_int) {
 541     // Else normal reg-reg copy
 542     assert( src_second != dst_first, "smashed second before evacuating it" );
 543     if (cbuf) {
 544       __ mov(reg_to_register_object(Matcher::_regEncode[dst_first]), reg_to_register_object(Matcher::_regEncode[src_first]));
 545 #ifndef PRODUCT
 546     } else if (!do_size) {
 547       st->print("MOV    R_%s, R_%s\t# spill",
 548                 Matcher::regName[dst_first],
 549                 Matcher::regName[src_first]);
 550 #endif
 551     }
 552 #ifdef AARCH64
 553     if (src_first+1 == src_second && dst_first+1 == dst_second) {
 554       return size + 4;
 555     }
 556 #endif
 557     size += 4;
 558   }
 559 
 560   // Check for integer store
 561   if (src_first_rc == rc_int && dst_first_rc == rc_stack) {
 562     int offset = ra_->reg2offset(dst_first);
 563     if (cbuf && !is_memoryI(offset)) {
 564       ra_->C->record_method_not_compilable("unable to handle large constant offsets");
 565       return 0;
 566     } else {
 567       if (src_second_rc != rc_bad && is_iRegLd_memhd(src_first, src_second, offset)) {
 568         assert((src_first&1)==0 && src_first+1 == src_second, "pair of registers must be aligned/contiguous");
 569         if (cbuf) {
 570           __ str_64(reg_to_register_object(Matcher::_regEncode[src_first]), Address(SP, offset));
 571 #ifndef PRODUCT
 572         } else if (!do_size) {
 573           if (size != 0) st->print("\n\t");
 574           st->print(STR_64 "   R_%s,[R_SP + #%d]\t! spill",OptoReg::regname(src_first), offset);
 575 #endif
 576         }
 577         return size + 4;
 578       } else {
 579         if (cbuf) {
 580           __ str_32(reg_to_register_object(Matcher::_regEncode[src_first]), Address(SP, offset));
 581 #ifndef PRODUCT
 582         } else if (!do_size) {
 583           if (size != 0) st->print("\n\t");
 584           st->print(STR_32 "   R_%s,[R_SP + #%d]\t! spill",OptoReg::regname(src_first), offset);
 585 #endif
 586         }
 587       }
 588     }
 589     size += 4;
 590   }
 591 
 592   // Check for integer load
 593   if (dst_first_rc == rc_int && src_first_rc == rc_stack) {
 594     int offset = ra_->reg2offset(src_first);
 595     if (cbuf && !is_memoryI(offset)) {
 596       ra_->C->record_method_not_compilable("unable to handle large constant offsets");
 597       return 0;
 598     } else {
 599       if (src_second_rc != rc_bad && is_iRegLd_memhd(dst_first, dst_second, offset)) {
 600         assert((src_first&1)==0 && src_first+1 == src_second, "pair of registers must be aligned/contiguous");
 601         if (cbuf) {
 602           __ ldr_64(reg_to_register_object(Matcher::_regEncode[dst_first]), Address(SP, offset));
 603 #ifndef PRODUCT
 604         } else if (!do_size) {
 605           if (size != 0) st->print("\n\t");
 606           st->print(LDR_64 "   R_%s,[R_SP + #%d]\t! spill",OptoReg::regname(dst_first), offset);
 607 #endif
 608         }
 609         return size + 4;
 610       } else {
 611         if (cbuf) {
 612           __ ldr_32(reg_to_register_object(Matcher::_regEncode[dst_first]), Address(SP, offset));
 613 #ifndef PRODUCT
 614         } else if (!do_size) {
 615           if (size != 0) st->print("\n\t");
 616           st->print(LDR_32 "   R_%s,[R_SP + #%d]\t! spill",OptoReg::regname(dst_first), offset);
 617 #endif
 618         }
 619       }
 620     }
 621     size += 4;
 622   }
 623 
 624   // Check for float reg-reg copy
 625   if (src_first_rc == rc_float && dst_first_rc == rc_float) {
 626     if (src_second_rc != rc_bad) {
 627       assert((src_first&1)==0 && src_first+1 == src_second && (dst_first&1)==0 && dst_first+1 == dst_second, "pairs of registers must be aligned/contiguous");
 628       if (cbuf) {
 629       __ mov_double(reg_to_FloatRegister_object(Matcher::_regEncode[dst_first]), reg_to_FloatRegister_object(Matcher::_regEncode[src_first]));
 630 #ifndef PRODUCT
 631       } else if (!do_size) {
 632         st->print(MOV_DOUBLE "    R_%s, R_%s\t# spill",
 633                   Matcher::regName[dst_first],
 634                   Matcher::regName[src_first]);
 635 #endif
 636       }
 637       return 4;
 638     }
 639     if (cbuf) {
 640       __ mov_float(reg_to_FloatRegister_object(Matcher::_regEncode[dst_first]), reg_to_FloatRegister_object(Matcher::_regEncode[src_first]));
 641 #ifndef PRODUCT
 642     } else if (!do_size) {
 643       st->print(MOV_FLOAT "    R_%s, R_%s\t# spill",
 644                 Matcher::regName[dst_first],
 645                 Matcher::regName[src_first]);
 646 #endif
 647     }
 648     size = 4;
 649   }
 650 
 651   // Check for float store
 652   if (src_first_rc == rc_float && dst_first_rc == rc_stack) {
 653     int offset = ra_->reg2offset(dst_first);
 654     if (cbuf && !is_memoryfp(offset)) {
 655       ra_->C->record_method_not_compilable("unable to handle large constant offsets");
 656       return 0;
 657     } else {
 658       // Further check for aligned-adjacent pair, so we can use a double store
 659       if (src_second_rc != rc_bad) {
 660         assert((src_first&1)==0 && src_first+1 == src_second && (dst_first&1)==0 && dst_first+1 == dst_second, "pairs of registers and stack slots must be aligned/contiguous");
 661         if (cbuf) {
 662           __ str_double(reg_to_FloatRegister_object(Matcher::_regEncode[src_first]), Address(SP, offset));
 663 #ifndef PRODUCT
 664         } else if (!do_size) {
 665           if (size != 0) st->print("\n\t");
 666           st->print(STR_DOUBLE "   R_%s,[R_SP + #%d]\t! spill",OptoReg::regname(src_first),offset);
 667 #endif
 668         }
 669         return size + 4;
 670       } else {
 671         if (cbuf) {
 672           __ str_float(reg_to_FloatRegister_object(Matcher::_regEncode[src_first]), Address(SP, offset));
 673 #ifndef PRODUCT
 674         } else if (!do_size) {
 675           if (size != 0) st->print("\n\t");
 676           st->print(STR_FLOAT "   R_%s,[R_SP + #%d]\t! spill",OptoReg::regname(src_first),offset);
 677 #endif
 678         }
 679       }
 680     }
 681     size += 4;
 682   }
 683 
 684   // Check for float load
 685   if (dst_first_rc == rc_float && src_first_rc == rc_stack) {
 686     int offset = ra_->reg2offset(src_first);
 687     if (cbuf && !is_memoryfp(offset)) {
 688       ra_->C->record_method_not_compilable("unable to handle large constant offsets");
 689       return 0;
 690     } else {
 691       // Further check for aligned-adjacent pair, so we can use a double store
 692       if (src_second_rc != rc_bad) {
 693         assert((src_first&1)==0 && src_first+1 == src_second && (dst_first&1)==0 && dst_first+1 == dst_second, "pairs of registers and stack slots must be aligned/contiguous");
 694         if (cbuf) {
 695           __ ldr_double(reg_to_FloatRegister_object(Matcher::_regEncode[dst_first]), Address(SP, offset));
 696 #ifndef PRODUCT
 697         } else if (!do_size) {
 698           if (size != 0) st->print("\n\t");
 699           st->print(LDR_DOUBLE "   R_%s,[R_SP + #%d]\t! spill",OptoReg::regname(dst_first),offset);
 700 #endif
 701         }
 702         return size + 4;
 703       } else {
 704         if (cbuf) {
 705           __ ldr_float(reg_to_FloatRegister_object(Matcher::_regEncode[dst_first]), Address(SP, offset));
 706 #ifndef PRODUCT
 707         } else if (!do_size) {
 708           if (size != 0) st->print("\n\t");
 709           st->print(LDR_FLOAT "   R_%s,[R_SP + #%d]\t! spill",OptoReg::regname(dst_first),offset);
 710 #endif
 711         }
 712       }
 713     }
 714     size += 4;
 715   }
 716 
 717   // check for int reg -> float reg move
 718   if (src_first_rc == rc_int && dst_first_rc == rc_float) {
 719     // Further check for aligned-adjacent pair, so we can use a single instruction
 720     if (src_second_rc != rc_bad) {
 721       assert((dst_first&1)==0 && dst_first+1 == dst_second, "pairs of registers must be aligned/contiguous");
 722       assert((src_first&1)==0 && src_first+1 == src_second, "pairs of registers must be aligned/contiguous");
 723       assert(src_second_rc == rc_int && dst_second_rc == rc_float, "unsupported");
 724       if (cbuf) {
 725 #ifdef AARCH64
 726         __ fmov_dx(reg_to_FloatRegister_object(Matcher::_regEncode[dst_first]), reg_to_register_object(Matcher::_regEncode[src_first]));
 727 #else
 728         __ fmdrr(reg_to_FloatRegister_object(Matcher::_regEncode[dst_first]), reg_to_register_object(Matcher::_regEncode[src_first]), reg_to_register_object(Matcher::_regEncode[src_second]));
 729 #endif
 730 #ifndef PRODUCT
 731       } else if (!do_size) {
 732         if (size != 0) st->print("\n\t");
 733 #ifdef AARCH64
 734         st->print("FMOV_DX   R_%s, R_%s\t! spill",OptoReg::regname(dst_first), OptoReg::regname(src_first));
 735 #else
 736         st->print("FMDRR   R_%s, R_%s, R_%s\t! spill",OptoReg::regname(dst_first), OptoReg::regname(src_first), OptoReg::regname(src_second));
 737 #endif
 738 #endif
 739       }
 740       return size + 4;
 741     } else {
 742       if (cbuf) {
 743         __ fmsr(reg_to_FloatRegister_object(Matcher::_regEncode[dst_first]), reg_to_register_object(Matcher::_regEncode[src_first]));
 744 #ifndef PRODUCT
 745       } else if (!do_size) {
 746         if (size != 0) st->print("\n\t");
 747         st->print(FMSR "   R_%s, R_%s\t! spill",OptoReg::regname(dst_first), OptoReg::regname(src_first));
 748 #endif
 749       }
 750       size += 4;
 751     }
 752   }
 753 
 754   // check for float reg -> int reg move
 755   if (src_first_rc == rc_float && dst_first_rc == rc_int) {
 756     // Further check for aligned-adjacent pair, so we can use a single instruction
 757     if (src_second_rc != rc_bad) {
 758       assert((src_first&1)==0 && src_first+1 == src_second, "pairs of registers must be aligned/contiguous");
 759       assert((dst_first&1)==0 && dst_first+1 == dst_second, "pairs of registers must be aligned/contiguous");
 760       assert(src_second_rc == rc_float && dst_second_rc == rc_int, "unsupported");
 761       if (cbuf) {
 762 #ifdef AARCH64
 763         __ fmov_xd(reg_to_register_object(Matcher::_regEncode[dst_first]), reg_to_FloatRegister_object(Matcher::_regEncode[src_first]));
 764 #else
 765         __ fmrrd(reg_to_register_object(Matcher::_regEncode[dst_first]), reg_to_register_object(Matcher::_regEncode[dst_second]), reg_to_FloatRegister_object(Matcher::_regEncode[src_first]));
 766 #endif
 767 #ifndef PRODUCT
 768       } else if (!do_size) {
 769         if (size != 0) st->print("\n\t");
 770 #ifdef AARCH64
 771         st->print("FMOV_XD R_%s, R_%s\t! spill",OptoReg::regname(dst_first), OptoReg::regname(src_first));
 772 #else
 773         st->print("FMRRD   R_%s, R_%s, R_%s\t! spill",OptoReg::regname(dst_first), OptoReg::regname(dst_second), OptoReg::regname(src_first));
 774 #endif
 775 #endif
 776       }
 777       return size + 4;
 778     } else {
 779       if (cbuf) {
 780         __ fmrs(reg_to_register_object(Matcher::_regEncode[dst_first]), reg_to_FloatRegister_object(Matcher::_regEncode[src_first]));
 781 #ifndef PRODUCT
 782       } else if (!do_size) {
 783         if (size != 0) st->print("\n\t");
 784         st->print(FMRS "   R_%s, R_%s\t! spill",OptoReg::regname(dst_first), OptoReg::regname(src_first));
 785 #endif
 786       }
 787       size += 4;
 788     }
 789   }
 790 
 791   // --------------------------------------------------------------------
 792   // Check for hi bits still needing moving.  Only happens for misaligned
 793   // arguments to native calls.
 794   if (src_second == dst_second)
 795     return size;               // Self copy; no move
 796   assert( src_second_rc != rc_bad && dst_second_rc != rc_bad, "src_second & dst_second cannot be Bad" );
 797 
 798 #ifndef AARCH64
 799   // Check for integer reg-reg copy.  Hi bits are stuck up in the top
 800   // 32-bits of a 64-bit register, but are needed in low bits of another
 801   // register (else it's a hi-bits-to-hi-bits copy which should have
 802   // happened already as part of a 64-bit move)
 803   if (src_second_rc == rc_int && dst_second_rc == rc_int) {
 804     if (cbuf) {
 805       __ mov(reg_to_register_object(Matcher::_regEncode[dst_second]), reg_to_register_object(Matcher::_regEncode[src_second]));
 806 #ifndef PRODUCT
 807     } else if (!do_size) {
 808       if (size != 0) st->print("\n\t");
 809       st->print("MOV    R_%s, R_%s\t# spill high",
 810                 Matcher::regName[dst_second],
 811                 Matcher::regName[src_second]);
 812 #endif
 813     }
 814     return size+4;
 815   }
 816 
 817   // Check for high word integer store
 818   if (src_second_rc == rc_int && dst_second_rc == rc_stack) {
 819     int offset = ra_->reg2offset(dst_second);
 820 
 821     if (cbuf && !is_memoryP(offset)) {
 822       ra_->C->record_method_not_compilable("unable to handle large constant offsets");
 823       return 0;
 824     } else {
 825       if (cbuf) {
 826         __ str(reg_to_register_object(Matcher::_regEncode[src_second]), Address(SP, offset));
 827 #ifndef PRODUCT
 828       } else if (!do_size) {
 829         if (size != 0) st->print("\n\t");
 830         st->print("STR   R_%s,[R_SP + #%d]\t! spill",OptoReg::regname(src_second), offset);
 831 #endif
 832       }
 833     }
 834     return size + 4;
 835   }
 836 
 837   // Check for high word integer load
 838   if (dst_second_rc == rc_int && src_second_rc == rc_stack) {
 839     int offset = ra_->reg2offset(src_second);
 840     if (cbuf && !is_memoryP(offset)) {
 841       ra_->C->record_method_not_compilable("unable to handle large constant offsets");
 842       return 0;
 843     } else {
 844       if (cbuf) {
 845         __ ldr(reg_to_register_object(Matcher::_regEncode[dst_second]), Address(SP, offset));
 846 #ifndef PRODUCT
 847       } else if (!do_size) {
 848         if (size != 0) st->print("\n\t");
 849         st->print("LDR   R_%s,[R_SP + #%d]\t! spill",OptoReg::regname(dst_second), offset);
 850 #endif
 851       }
 852     }
 853     return size + 4;
 854   }
 855 #endif
 856 
 857   Unimplemented();
 858   return 0; // Mute compiler
 859 }
 860 
 861 #ifndef PRODUCT
 862 void MachSpillCopyNode::format( PhaseRegAlloc *ra_, outputStream *st ) const {
 863   implementation( NULL, ra_, false, st );
 864 }
 865 #endif
 866 
 867 void MachSpillCopyNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 868   implementation( &cbuf, ra_, false, NULL );
 869 }
 870 
 871 uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
 872   return implementation( NULL, ra_, true, NULL );
 873 }
 874 
 875 //=============================================================================
 876 #ifndef PRODUCT
 877 void MachNopNode::format( PhaseRegAlloc *, outputStream *st ) const {
 878   st->print("NOP \t# %d bytes pad for loops and calls", 4 * _count);
 879 }
 880 #endif
 881 
 882 void MachNopNode::emit(CodeBuffer &cbuf, PhaseRegAlloc * ) const {
 883   MacroAssembler _masm(&cbuf);
 884   for(int i = 0; i < _count; i += 1) {
 885     __ nop();
 886   }
 887 }
 888 
 889 uint MachNopNode::size(PhaseRegAlloc *ra_) const {
 890   return 4 * _count;
 891 }
 892 
 893 
 894 //=============================================================================
 895 #ifndef PRODUCT
 896 void BoxLockNode::format( PhaseRegAlloc *ra_, outputStream *st ) const {
 897   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
 898   int reg = ra_->get_reg_first(this);
 899   st->print("ADD    %s,R_SP+#%d",Matcher::regName[reg], offset);
 900 }
 901 #endif
 902 
 903 void BoxLockNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 904   MacroAssembler _masm(&cbuf);
 905   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
 906   int reg = ra_->get_encode(this);
 907   Register dst = reg_to_register_object(reg);
 908 
 909   if (is_aimm(offset)) {
 910     __ add(dst, SP, offset);
 911   } else {
 912     __ mov_slow(dst, offset);
 913 #ifdef AARCH64
 914     __ add(dst, SP, dst, ex_lsl);
 915 #else
 916     __ add(dst, SP, dst);
 917 #endif
 918   }
 919 }
 920 
 921 uint BoxLockNode::size(PhaseRegAlloc *ra_) const {
 922   // BoxLockNode is not a MachNode, so we can't just call MachNode::size(ra_)
 923   assert(ra_ == ra_->C->regalloc(), "sanity");
 924   return ra_->C->scratch_emit_size(this);
 925 }
 926 
 927 //=============================================================================
 928 #ifndef PRODUCT
 929 #ifdef AARCH64
 930 #define R_RTEMP "R_R16"
 931 #else
 932 #define R_RTEMP "R_R12"
 933 #endif
 934 void MachUEPNode::format( PhaseRegAlloc *ra_, outputStream *st ) const {
 935   st->print_cr("\nUEP:");
 936   if (UseCompressedClassPointers) {
 937     st->print_cr("\tLDR_w " R_RTEMP ",[R_R0 + oopDesc::klass_offset_in_bytes]\t! Inline cache check");
 938     st->print_cr("\tdecode_klass " R_RTEMP);
 939   } else {
 940     st->print_cr("\tLDR   " R_RTEMP ",[R_R0 + oopDesc::klass_offset_in_bytes]\t! Inline cache check");
 941   }
 942   st->print_cr("\tCMP   " R_RTEMP ",R_R8" );
 943   st->print   ("\tB.NE  SharedRuntime::handle_ic_miss_stub");
 944 }
 945 #endif
 946 
 947 void MachUEPNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 948   MacroAssembler _masm(&cbuf);
 949   Register iCache  = reg_to_register_object(Matcher::inline_cache_reg_encode());
 950   assert(iCache == Ricklass, "should be");
 951   Register receiver = R0;
 952 
 953   __ load_klass(Rtemp, receiver);
 954   __ cmp(Rtemp, iCache);
 955 #ifdef AARCH64
 956   Label match;
 957   __ b(match, eq);
 958   __ jump(SharedRuntime::get_ic_miss_stub(), relocInfo::runtime_call_type, Rtemp);
 959   __ bind(match);
 960 #else
 961   __ jump(SharedRuntime::get_ic_miss_stub(), relocInfo::runtime_call_type, noreg, ne);
 962 #endif
 963 }
 964 
 965 uint MachUEPNode::size(PhaseRegAlloc *ra_) const {
 966   return MachNode::size(ra_);
 967 }
 968 
 969 
 970 //=============================================================================
 971 
 972 // Emit exception handler code.
 973 int HandlerImpl::emit_exception_handler(CodeBuffer& cbuf) {
 974   MacroAssembler _masm(&cbuf);
 975 
 976   address base = __ start_a_stub(size_exception_handler());
 977   if (base == NULL) {
 978     ciEnv::current()->record_failure("CodeCache is full");
 979     return 0;  // CodeBuffer::expand failed
 980   }
 981 
 982   int offset = __ offset();
 983 
 984   // OK to trash LR, because exception blob will kill it
 985   __ jump(OptoRuntime::exception_blob()->entry_point(), relocInfo::runtime_call_type, LR_tmp);
 986 
 987   assert(__ offset() - offset <= (int) size_exception_handler(), "overflow");
 988 
 989   __ end_a_stub();
 990 
 991   return offset;
 992 }
 993 
 994 int HandlerImpl::emit_deopt_handler(CodeBuffer& cbuf) {
 995   // Can't use any of the current frame's registers as we may have deopted
 996   // at a poll and everything can be live.
 997   MacroAssembler _masm(&cbuf);
 998 
 999   address base = __ start_a_stub(size_deopt_handler());
1000   if (base == NULL) {
1001     ciEnv::current()->record_failure("CodeCache is full");
1002     return 0;  // CodeBuffer::expand failed
1003   }
1004 
1005   int offset = __ offset();
1006   address deopt_pc = __ pc();
1007 
1008 #ifdef AARCH64
1009   // See LR saved by caller in sharedRuntime_arm.cpp
1010   // see also hse1 ws
1011   // see also LIR_Assembler::emit_deopt_handler
1012 
1013   __ raw_push(LR, LR); // preserve LR in both slots
1014   __ mov_relative_address(LR, deopt_pc);
1015   __ str(LR, Address(SP, 1 * wordSize)); // save deopt PC
1016   // OK to kill LR, because deopt blob will restore it from SP[0]
1017   __ jump(SharedRuntime::deopt_blob()->unpack(), relocInfo::runtime_call_type, LR_tmp);
1018 #else
1019   __ sub(SP, SP, wordSize); // make room for saved PC
1020   __ push(LR); // save LR that may be live when we get here
1021   __ mov_relative_address(LR, deopt_pc);
1022   __ str(LR, Address(SP, wordSize)); // save deopt PC
1023   __ pop(LR); // restore LR
1024   __ jump(SharedRuntime::deopt_blob()->unpack(), relocInfo::runtime_call_type, noreg);
1025 #endif
1026 
1027   assert(__ offset() - offset <= (int) size_deopt_handler(), "overflow");
1028 
1029   __ end_a_stub();
1030   return offset;
1031 }
1032 
1033 const bool Matcher::match_rule_supported(int opcode) {
1034   if (!has_match_rule(opcode))
1035     return false;
1036 
1037   switch (opcode) {
1038   case Op_PopCountI:
1039   case Op_PopCountL:
1040     if (!UsePopCountInstruction)
1041       return false;
1042     break;
1043   case Op_LShiftCntV:
1044   case Op_RShiftCntV:
1045   case Op_AddVB:
1046   case Op_AddVS:
1047   case Op_AddVI:
1048   case Op_AddVL:
1049   case Op_SubVB:
1050   case Op_SubVS:
1051   case Op_SubVI:
1052   case Op_SubVL:
1053   case Op_MulVS:
1054   case Op_MulVI:
1055   case Op_LShiftVB:
1056   case Op_LShiftVS:
1057   case Op_LShiftVI:
1058   case Op_LShiftVL:
1059   case Op_RShiftVB:
1060   case Op_RShiftVS:
1061   case Op_RShiftVI:
1062   case Op_RShiftVL:
1063   case Op_URShiftVB:
1064   case Op_URShiftVS:
1065   case Op_URShiftVI:
1066   case Op_URShiftVL:
1067   case Op_AndV:
1068   case Op_OrV:
1069   case Op_XorV:
1070     return VM_Version::has_simd();
1071   case Op_LoadVector:
1072   case Op_StoreVector:
1073   case Op_AddVF:
1074   case Op_SubVF:
1075   case Op_MulVF:
1076 #ifdef AARCH64
1077     return VM_Version::has_simd();
1078 #else
1079     return VM_Version::has_vfp() || VM_Version::has_simd();
1080 #endif
1081   case Op_AddVD:
1082   case Op_SubVD:
1083   case Op_MulVD:
1084   case Op_DivVF:
1085   case Op_DivVD:
1086 #ifdef AARCH64
1087     return VM_Version::has_simd();
1088 #else
1089     return VM_Version::has_vfp();
1090 #endif
1091   }
1092 
1093   return true;  // Per default match rules are supported.
1094 }
1095 
1096 const bool Matcher::match_rule_supported_vector(int opcode, int vlen) {
1097 
1098   // TODO
1099   // identify extra cases that we might want to provide match rules for
1100   // e.g. Op_ vector nodes and other intrinsics while guarding with vlen
1101   bool ret_value = match_rule_supported(opcode);
1102   // Add rules here.
1103 
1104   return ret_value;  // Per default match rules are supported.
1105 }
1106 
1107 const bool Matcher::has_predicated_vectors(void) {
1108   return false;
1109 }
1110 
1111 const int Matcher::float_pressure(int default_pressure_threshold) {
1112   return default_pressure_threshold;
1113 }
1114 
1115 int Matcher::regnum_to_fpu_offset(int regnum) {
1116   return regnum - 32; // The FP registers are in the second chunk
1117 }
1118 
1119 // Vector width in bytes
1120 const int Matcher::vector_width_in_bytes(BasicType bt) {
1121   return MaxVectorSize;
1122 }
1123 
1124 // Vector ideal reg corresponding to specified size in bytes
1125 const uint Matcher::vector_ideal_reg(int size) {
1126   assert(MaxVectorSize >= size, "");
1127   switch(size) {
1128     case  8: return Op_VecD;
1129     case 16: return Op_VecX;
1130   }
1131   ShouldNotReachHere();
1132   return 0;
1133 }
1134 
1135 const uint Matcher::vector_shift_count_ideal_reg(int size) {
1136   return vector_ideal_reg(size);
1137 }
1138 
1139 // Limits on vector size (number of elements) loaded into vector.
1140 const int Matcher::max_vector_size(const BasicType bt) {
1141   assert(is_java_primitive(bt), "only primitive type vectors");
1142   return vector_width_in_bytes(bt)/type2aelembytes(bt);
1143 }
1144 
1145 const int Matcher::min_vector_size(const BasicType bt) {
1146   assert(is_java_primitive(bt), "only primitive type vectors");
1147   return 8/type2aelembytes(bt);
1148 }
1149 
1150 // ARM doesn't support misaligned vectors store/load.
1151 const bool Matcher::misaligned_vectors_ok() {
1152   return false;
1153 }
1154 
1155 // ARM doesn't support AES intrinsics
1156 const bool Matcher::pass_original_key_for_aes() {
1157   return false;
1158 }
1159 
1160 const bool Matcher::convL2FSupported(void) {
1161 #ifdef AARCH64
1162   return true;
1163 #else
1164   return false;
1165 #endif
1166 }
1167 
1168 // Is this branch offset short enough that a short branch can be used?
1169 //
1170 // NOTE: If the platform does not provide any short branch variants, then
1171 //       this method should return false for offset 0.
1172 bool Matcher::is_short_branch_offset(int rule, int br_size, int offset) {
1173   // The passed offset is relative to address of the branch.
1174   // On ARM a branch displacement is calculated relative to address
1175   // of the branch + 8.
1176   //
1177   // offset -= 8;
1178   // return (Assembler::is_simm24(offset));
1179   return false;
1180 }
1181 
1182 const bool Matcher::isSimpleConstant64(jlong value) {
1183   // Will one (StoreL ConL) be cheaper than two (StoreI ConI)?.
1184 #ifdef AARCH64
1185   return (value == 0);
1186 #else
1187   return false;
1188 #endif
1189 }
1190 
1191 // No scaling for the parameter the ClearArray node.
1192 const bool Matcher::init_array_count_is_in_bytes = true;
1193 
1194 #ifdef AARCH64
1195 const int Matcher::long_cmove_cost() { return 1; }
1196 #else
1197 // Needs 2 CMOV's for longs.
1198 const int Matcher::long_cmove_cost() { return 2; }
1199 #endif
1200 
1201 #ifdef AARCH64
1202 const int Matcher::float_cmove_cost() { return 1; }
1203 #else
1204 // CMOVF/CMOVD are expensive on ARM.
1205 const int Matcher::float_cmove_cost() { return ConditionalMoveLimit; }
1206 #endif
1207 
1208 // Does the CPU require late expand (see block.cpp for description of late expand)?
1209 const bool Matcher::require_postalloc_expand = false;
1210 
1211 // Do we need to mask the count passed to shift instructions or does
1212 // the cpu only look at the lower 5/6 bits anyway?
1213 // FIXME: does this handle vector shifts as well?
1214 #ifdef AARCH64
1215 const bool Matcher::need_masked_shift_count = false;
1216 #else
1217 const bool Matcher::need_masked_shift_count = true;
1218 #endif
1219 
1220 const bool Matcher::convi2l_type_required = true;
1221 
1222 // Should the Matcher clone shifts on addressing modes, expecting them
1223 // to be subsumed into complex addressing expressions or compute them
1224 // into registers?
1225 bool Matcher::clone_address_expressions(AddPNode* m, Matcher::MStack& mstack, VectorSet& address_visited) {
1226   return clone_base_plus_offset_address(m, mstack, address_visited);
1227 }
1228 
1229 void Compile::reshape_address(AddPNode* addp) {
1230 }
1231 
1232 bool Matcher::narrow_oop_use_complex_address() {
1233   NOT_LP64(ShouldNotCallThis());
1234   assert(UseCompressedOops, "only for compressed oops code");
1235   return false;
1236 }
1237 
1238 bool Matcher::narrow_klass_use_complex_address() {
1239   NOT_LP64(ShouldNotCallThis());
1240   assert(UseCompressedClassPointers, "only for compressed klass code");
1241   return false;
1242 }
1243 
1244 bool Matcher::const_oop_prefer_decode() {
1245   NOT_LP64(ShouldNotCallThis());
1246   return true;
1247 }
1248 
1249 bool Matcher::const_klass_prefer_decode() {
1250   NOT_LP64(ShouldNotCallThis());
1251   return true;
1252 }
1253 
1254 // Is it better to copy float constants, or load them directly from memory?
1255 // Intel can load a float constant from a direct address, requiring no
1256 // extra registers.  Most RISCs will have to materialize an address into a
1257 // register first, so they would do better to copy the constant from stack.
1258 const bool Matcher::rematerialize_float_constants = false;
1259 
1260 // If CPU can load and store mis-aligned doubles directly then no fixup is
1261 // needed.  Else we split the double into 2 integer pieces and move it
1262 // piece-by-piece.  Only happens when passing doubles into C code as the
1263 // Java calling convention forces doubles to be aligned.
1264 #ifdef AARCH64
1265 // On stack replacement support:
1266 // We don't need Load[DL]_unaligned support, because interpreter stack
1267 // has correct alignment
1268 const bool Matcher::misaligned_doubles_ok = true;
1269 #else
1270 const bool Matcher::misaligned_doubles_ok = false;
1271 #endif
1272 
1273 // No-op on ARM.
1274 void Matcher::pd_implicit_null_fixup(MachNode *node, uint idx) {
1275 }
1276 
1277 // Advertise here if the CPU requires explicit rounding operations
1278 // to implement the UseStrictFP mode.
1279 const bool Matcher::strict_fp_requires_explicit_rounding = false;
1280 
1281 // Are floats converted to double when stored to stack during deoptimization?
1282 // ARM does not handle callee-save floats.
1283 bool Matcher::float_in_double() {
1284   return false;
1285 }
1286 
1287 // Do ints take an entire long register or just half?
1288 // Note that we if-def off of _LP64.
1289 // The relevant question is how the int is callee-saved.  In _LP64
1290 // the whole long is written but de-opt'ing will have to extract
1291 // the relevant 32 bits, in not-_LP64 only the low 32 bits is written.
1292 #ifdef _LP64
1293 const bool Matcher::int_in_long = true;
1294 #else
1295 const bool Matcher::int_in_long = false;
1296 #endif
1297 
1298 // Return whether or not this register is ever used as an argument.  This
1299 // function is used on startup to build the trampoline stubs in generateOptoStub.
1300 // Registers not mentioned will be killed by the VM call in the trampoline, and
1301 // arguments in those registers not be available to the callee.
1302 bool Matcher::can_be_java_arg( int reg ) {
1303 #ifdef AARCH64
1304   if (reg >= R_R0_num && reg < R_R8_num) return true;
1305   if (reg >= R_V0_num && reg <= R_V7b_num && ((reg & 3) < 2)) return true;
1306 #else
1307   if (reg == R_R0_num ||
1308       reg == R_R1_num ||
1309       reg == R_R2_num ||
1310       reg == R_R3_num) return true;
1311 
1312   if (reg >= R_S0_num &&
1313       reg <= R_S13_num) return true;
1314 #endif
1315   return false;
1316 }
1317 
1318 bool Matcher::is_spillable_arg( int reg ) {
1319   return can_be_java_arg(reg);
1320 }
1321 
1322 bool Matcher::use_asm_for_ldiv_by_con( jlong divisor ) {
1323   return false;
1324 }
1325 
1326 // Register for DIVI projection of divmodI
1327 RegMask Matcher::divI_proj_mask() {
1328   ShouldNotReachHere();
1329   return RegMask();
1330 }
1331 
1332 // Register for MODI projection of divmodI
1333 RegMask Matcher::modI_proj_mask() {
1334   ShouldNotReachHere();
1335   return RegMask();
1336 }
1337 
1338 // Register for DIVL projection of divmodL
1339 RegMask Matcher::divL_proj_mask() {
1340   ShouldNotReachHere();
1341   return RegMask();
1342 }
1343 
1344 // Register for MODL projection of divmodL
1345 RegMask Matcher::modL_proj_mask() {
1346   ShouldNotReachHere();
1347   return RegMask();
1348 }
1349 
1350 const RegMask Matcher::method_handle_invoke_SP_save_mask() {
1351   return FP_REGP_mask();
1352 }
1353 
1354 bool maybe_far_call(const CallNode *n) {
1355   return !MacroAssembler::_reachable_from_cache(n->as_Call()->entry_point());
1356 }
1357 
1358 bool maybe_far_call(const MachCallNode *n) {
1359   return !MacroAssembler::_reachable_from_cache(n->as_MachCall()->entry_point());
1360 }
1361 
1362 %}
1363 
1364 //----------ENCODING BLOCK-----------------------------------------------------
1365 // This block specifies the encoding classes used by the compiler to output
1366 // byte streams.  Encoding classes are parameterized macros used by
1367 // Machine Instruction Nodes in order to generate the bit encoding of the
1368 // instruction.  Operands specify their base encoding interface with the
1369 // interface keyword.  There are currently supported four interfaces,
1370 // REG_INTER, CONST_INTER, MEMORY_INTER, & COND_INTER.  REG_INTER causes an
1371 // operand to generate a function which returns its register number when
1372 // queried.   CONST_INTER causes an operand to generate a function which
1373 // returns the value of the constant when queried.  MEMORY_INTER causes an
1374 // operand to generate four functions which return the Base Register, the
1375 // Index Register, the Scale Value, and the Offset Value of the operand when
1376 // queried.  COND_INTER causes an operand to generate six functions which
1377 // return the encoding code (ie - encoding bits for the instruction)
1378 // associated with each basic boolean condition for a conditional instruction.
1379 //
1380 // Instructions specify two basic values for encoding.  Again, a function
1381 // is available to check if the constant displacement is an oop. They use the
1382 // ins_encode keyword to specify their encoding classes (which must be
1383 // a sequence of enc_class names, and their parameters, specified in
1384 // the encoding block), and they use the
1385 // opcode keyword to specify, in order, their primary, secondary, and
1386 // tertiary opcode.  Only the opcode sections which a particular instruction
1387 // needs for encoding need to be specified.
1388 encode %{
1389   enc_class call_epilog %{
1390     // nothing
1391   %}
1392 
1393   enc_class Java_To_Runtime (method meth) %{
1394     // CALL directly to the runtime
1395     emit_call_reloc(cbuf, as_MachCall(), $meth, runtime_call_Relocation::spec());
1396   %}
1397 
1398   enc_class Java_Static_Call (method meth) %{
1399     // CALL to fixup routine.  Fixup routine uses ScopeDesc info to determine
1400     // who we intended to call.
1401 
1402     if ( !_method) {
1403       emit_call_reloc(cbuf, as_MachCall(), $meth, runtime_call_Relocation::spec());
1404     } else {
1405       int method_index = resolved_method_index(cbuf);
1406       RelocationHolder rspec = _optimized_virtual ? opt_virtual_call_Relocation::spec(method_index)
1407                                                   : static_call_Relocation::spec(method_index);
1408       emit_call_reloc(cbuf, as_MachCall(), $meth, rspec);
1409 
1410       // Emit stubs for static call.
1411       address stub = CompiledStaticCall::emit_to_interp_stub(cbuf);
1412       if (stub == NULL) {
1413         ciEnv::current()->record_failure("CodeCache is full");
1414         return;
1415       }
1416     }
1417   %}
1418 
1419   enc_class save_last_PC %{
1420     // preserve mark
1421     address mark = cbuf.insts()->mark();
1422     debug_only(int off0 = cbuf.insts_size());
1423     MacroAssembler _masm(&cbuf);
1424     int ret_addr_offset = as_MachCall()->ret_addr_offset();
1425     __ adr(LR, mark + ret_addr_offset);
1426     __ str(LR, Address(Rthread, JavaThread::last_Java_pc_offset()));
1427     debug_only(int off1 = cbuf.insts_size());
1428     assert(off1 - off0 == 2 * Assembler::InstructionSize, "correct size prediction");
1429     // restore mark
1430     cbuf.insts()->set_mark(mark);
1431   %}
1432 
1433   enc_class preserve_SP %{
1434     // preserve mark
1435     address mark = cbuf.insts()->mark();
1436     debug_only(int off0 = cbuf.insts_size());
1437     MacroAssembler _masm(&cbuf);
1438     // FP is preserved across all calls, even compiled calls.
1439     // Use it to preserve SP in places where the callee might change the SP.
1440     __ mov(Rmh_SP_save, SP);
1441     debug_only(int off1 = cbuf.insts_size());
1442     assert(off1 - off0 == 4, "correct size prediction");
1443     // restore mark
1444     cbuf.insts()->set_mark(mark);
1445   %}
1446 
1447   enc_class restore_SP %{
1448     MacroAssembler _masm(&cbuf);
1449     __ mov(SP, Rmh_SP_save);
1450   %}
1451 
1452   enc_class Java_Dynamic_Call (method meth) %{
1453     MacroAssembler _masm(&cbuf);
1454     Register R8_ic_reg = reg_to_register_object(Matcher::inline_cache_reg_encode());
1455     assert(R8_ic_reg == Ricklass, "should be");
1456     __ set_inst_mark();
1457 #ifdef AARCH64
1458 // TODO: see C1 LIR_Assembler::ic_call()
1459     InlinedAddress oop_literal((address)Universe::non_oop_word());
1460     int offset = __ offset();
1461     int fixed_size = mov_oop_size * 4;
1462     if (VM_Version::prefer_moves_over_load_literal()) {
1463       uintptr_t val = (uintptr_t)Universe::non_oop_word();
1464       __ movz(R8_ic_reg, (val >>  0) & 0xffff,  0);
1465       __ movk(R8_ic_reg, (val >> 16) & 0xffff, 16);
1466       __ movk(R8_ic_reg, (val >> 32) & 0xffff, 32);
1467       __ movk(R8_ic_reg, (val >> 48) & 0xffff, 48);
1468     } else {
1469       __ ldr_literal(R8_ic_reg, oop_literal);
1470     }
1471     assert(__ offset() - offset == fixed_size, "bad mov_oop size");
1472 #else
1473     __ movw(R8_ic_reg, ((unsigned int)Universe::non_oop_word()) & 0xffff);
1474     __ movt(R8_ic_reg, ((unsigned int)Universe::non_oop_word()) >> 16);
1475 #endif
1476     address  virtual_call_oop_addr = __ inst_mark();
1477     // CALL to fixup routine.  Fixup routine uses ScopeDesc info to determine
1478     // who we intended to call.
1479     int method_index = resolved_method_index(cbuf);
1480     __ relocate(virtual_call_Relocation::spec(virtual_call_oop_addr, method_index));
1481     emit_call_reloc(cbuf, as_MachCall(), $meth, RelocationHolder::none);
1482 #ifdef AARCH64
1483     if (!VM_Version::prefer_moves_over_load_literal()) {
1484       Label skip_literal;
1485       __ b(skip_literal);
1486       int off2 = __ offset();
1487       __ bind_literal(oop_literal);
1488       if (__ offset() - off2 == wordSize) {
1489         // no padding, so insert nop for worst-case sizing
1490         __ nop();
1491       }
1492       __ bind(skip_literal);
1493     }
1494 #endif
1495   %}
1496 
1497   enc_class LdReplImmI(immI src, regD dst, iRegI tmp, int cnt, int wth) %{
1498     // FIXME: load from constant table?
1499     // Load a constant replicated "count" times with width "width"
1500     int count = $cnt$$constant;
1501     int width = $wth$$constant;
1502     assert(count*width == 4, "sanity");
1503     int val = $src$$constant;
1504     if (width < 4) {
1505       int bit_width = width * 8;
1506       val &= (((int)1) << bit_width) - 1; // mask off sign bits
1507       for (int i = 0; i < count - 1; i++) {
1508         val |= (val << bit_width);
1509       }
1510     }
1511     MacroAssembler _masm(&cbuf);
1512 
1513     if (val == -1) {
1514       __ mvn($tmp$$Register, 0);
1515     } else if (val == 0) {
1516       __ mov($tmp$$Register, 0);
1517     } else {
1518       __ movw($tmp$$Register, val & 0xffff);
1519       __ movt($tmp$$Register, (unsigned int)val >> 16);
1520     }
1521     __ fmdrr($dst$$FloatRegister, $tmp$$Register, $tmp$$Register);
1522   %}
1523 
1524   enc_class LdReplImmF(immF src, regD dst, iRegI tmp) %{
1525     // Replicate float con 2 times and pack into vector (8 bytes) in regD.
1526     float fval = $src$$constant;
1527     int val = *((int*)&fval);
1528     MacroAssembler _masm(&cbuf);
1529 
1530     if (val == -1) {
1531       __ mvn($tmp$$Register, 0);
1532     } else if (val == 0) {
1533       __ mov($tmp$$Register, 0);
1534     } else {
1535       __ movw($tmp$$Register, val & 0xffff);
1536       __ movt($tmp$$Register, (unsigned int)val >> 16);
1537     }
1538     __ fmdrr($dst$$FloatRegister, $tmp$$Register, $tmp$$Register);
1539   %}
1540 
1541   enc_class enc_String_Compare(R0RegP str1, R1RegP str2, R2RegI cnt1, R3RegI cnt2, iRegI result, iRegI tmp1, iRegI tmp2) %{
1542     Label Ldone, Lloop;
1543     MacroAssembler _masm(&cbuf);
1544 
1545     Register   str1_reg = $str1$$Register;
1546     Register   str2_reg = $str2$$Register;
1547     Register   cnt1_reg = $cnt1$$Register; // int
1548     Register   cnt2_reg = $cnt2$$Register; // int
1549     Register   tmp1_reg = $tmp1$$Register;
1550     Register   tmp2_reg = $tmp2$$Register;
1551     Register result_reg = $result$$Register;
1552 
1553     assert_different_registers(str1_reg, str2_reg, cnt1_reg, cnt2_reg, tmp1_reg, tmp2_reg);
1554 
1555     // Compute the minimum of the string lengths(str1_reg) and the
1556     // difference of the string lengths (stack)
1557 
1558     // See if the lengths are different, and calculate min in str1_reg.
1559     // Stash diff in tmp2 in case we need it for a tie-breaker.
1560     __ subs_32(tmp2_reg, cnt1_reg, cnt2_reg);
1561 #ifdef AARCH64
1562     Label Lskip;
1563     __ _lsl_w(cnt1_reg, cnt1_reg, exact_log2(sizeof(jchar))); // scale the limit
1564     __ b(Lskip, mi);
1565     __ _lsl_w(cnt1_reg, cnt2_reg, exact_log2(sizeof(jchar))); // scale the limit
1566     __ bind(Lskip);
1567 #else
1568     __ mov(cnt1_reg, AsmOperand(cnt1_reg, lsl, exact_log2(sizeof(jchar)))); // scale the limit
1569     __ mov(cnt1_reg, AsmOperand(cnt2_reg, lsl, exact_log2(sizeof(jchar))), pl); // scale the limit
1570 #endif
1571 
1572     // reallocate cnt1_reg, cnt2_reg, result_reg
1573     // Note:  limit_reg holds the string length pre-scaled by 2
1574     Register limit_reg = cnt1_reg;
1575     Register  chr2_reg = cnt2_reg;
1576     Register  chr1_reg = tmp1_reg;
1577     // str{12} are the base pointers
1578 
1579     // Is the minimum length zero?
1580     __ cmp_32(limit_reg, 0);
1581     if (result_reg != tmp2_reg) {
1582       __ mov(result_reg, tmp2_reg, eq);
1583     }
1584     __ b(Ldone, eq);
1585 
1586     // Load first characters
1587     __ ldrh(chr1_reg, Address(str1_reg, 0));
1588     __ ldrh(chr2_reg, Address(str2_reg, 0));
1589 
1590     // Compare first characters
1591     __ subs(chr1_reg, chr1_reg, chr2_reg);
1592     if (result_reg != chr1_reg) {
1593       __ mov(result_reg, chr1_reg, ne);
1594     }
1595     __ b(Ldone, ne);
1596 
1597     {
1598       // Check after comparing first character to see if strings are equivalent
1599       // Check if the strings start at same location
1600       __ cmp(str1_reg, str2_reg);
1601       // Check if the length difference is zero
1602       __ cond_cmp(tmp2_reg, 0, eq);
1603       __ mov(result_reg, 0, eq); // result is zero
1604       __ b(Ldone, eq);
1605       // Strings might not be equal
1606     }
1607 
1608     __ subs(chr1_reg, limit_reg, 1 * sizeof(jchar));
1609     if (result_reg != tmp2_reg) {
1610       __ mov(result_reg, tmp2_reg, eq);
1611     }
1612     __ b(Ldone, eq);
1613 
1614     // Shift str1_reg and str2_reg to the end of the arrays, negate limit
1615     __ add(str1_reg, str1_reg, limit_reg);
1616     __ add(str2_reg, str2_reg, limit_reg);
1617     __ neg(limit_reg, chr1_reg);  // limit = -(limit-2)
1618 
1619     // Compare the rest of the characters
1620     __ bind(Lloop);
1621     __ ldrh(chr1_reg, Address(str1_reg, limit_reg));
1622     __ ldrh(chr2_reg, Address(str2_reg, limit_reg));
1623     __ subs(chr1_reg, chr1_reg, chr2_reg);
1624     if (result_reg != chr1_reg) {
1625       __ mov(result_reg, chr1_reg, ne);
1626     }
1627     __ b(Ldone, ne);
1628 
1629     __ adds(limit_reg, limit_reg, sizeof(jchar));
1630     __ b(Lloop, ne);
1631 
1632     // If strings are equal up to min length, return the length difference.
1633     if (result_reg != tmp2_reg) {
1634       __ mov(result_reg, tmp2_reg);
1635     }
1636 
1637     // Otherwise, return the difference between the first mismatched chars.
1638     __ bind(Ldone);
1639   %}
1640 
1641   enc_class enc_String_Equals(R0RegP str1, R1RegP str2, R2RegI cnt, iRegI result, iRegI tmp1, iRegI tmp2) %{
1642     Label Lchar, Lchar_loop, Ldone, Lequal;
1643     MacroAssembler _masm(&cbuf);
1644 
1645     Register   str1_reg = $str1$$Register;
1646     Register   str2_reg = $str2$$Register;
1647     Register    cnt_reg = $cnt$$Register; // int
1648     Register   tmp1_reg = $tmp1$$Register;
1649     Register   tmp2_reg = $tmp2$$Register;
1650     Register result_reg = $result$$Register;
1651 
1652     assert_different_registers(str1_reg, str2_reg, cnt_reg, tmp1_reg, tmp2_reg, result_reg);
1653 
1654     __ cmp(str1_reg, str2_reg); //same char[] ?
1655     __ b(Lequal, eq);
1656 
1657     __ cbz_32(cnt_reg, Lequal); // count == 0
1658 
1659     //rename registers
1660     Register limit_reg = cnt_reg;
1661     Register  chr1_reg = tmp1_reg;
1662     Register  chr2_reg = tmp2_reg;
1663 
1664     __ logical_shift_left(limit_reg, limit_reg, exact_log2(sizeof(jchar)));
1665 
1666     //check for alignment and position the pointers to the ends
1667     __ orr(chr1_reg, str1_reg, str2_reg);
1668     __ tst(chr1_reg, 0x3);
1669 
1670     // notZero means at least one not 4-byte aligned.
1671     // We could optimize the case when both arrays are not aligned
1672     // but it is not frequent case and it requires additional checks.
1673     __ b(Lchar, ne);
1674 
1675     // Compare char[] arrays aligned to 4 bytes.
1676     __ char_arrays_equals(str1_reg, str2_reg, limit_reg, result_reg,
1677                           chr1_reg, chr2_reg, Ldone);
1678 
1679     __ b(Lequal); // equal
1680 
1681     // char by char compare
1682     __ bind(Lchar);
1683     __ mov(result_reg, 0);
1684     __ add(str1_reg, limit_reg, str1_reg);
1685     __ add(str2_reg, limit_reg, str2_reg);
1686     __ neg(limit_reg, limit_reg); //negate count
1687 
1688     // Lchar_loop
1689     __ bind(Lchar_loop);
1690     __ ldrh(chr1_reg, Address(str1_reg, limit_reg));
1691     __ ldrh(chr2_reg, Address(str2_reg, limit_reg));
1692     __ cmp(chr1_reg, chr2_reg);
1693     __ b(Ldone, ne);
1694     __ adds(limit_reg, limit_reg, sizeof(jchar));
1695     __ b(Lchar_loop, ne);
1696 
1697     __ bind(Lequal);
1698     __ mov(result_reg, 1);  //equal
1699 
1700     __ bind(Ldone);
1701   %}
1702 
1703   enc_class enc_Array_Equals(R0RegP ary1, R1RegP ary2, iRegI tmp1, iRegI tmp2, iRegI tmp3, iRegI result) %{
1704     Label Ldone, Lloop, Lequal;
1705     MacroAssembler _masm(&cbuf);
1706 
1707     Register   ary1_reg = $ary1$$Register;
1708     Register   ary2_reg = $ary2$$Register;
1709     Register   tmp1_reg = $tmp1$$Register;
1710     Register   tmp2_reg = $tmp2$$Register;
1711     Register   tmp3_reg = $tmp3$$Register;
1712     Register result_reg = $result$$Register;
1713 
1714     assert_different_registers(ary1_reg, ary2_reg, tmp1_reg, tmp2_reg, tmp3_reg, result_reg);
1715 
1716     int length_offset  = arrayOopDesc::length_offset_in_bytes();
1717     int base_offset    = arrayOopDesc::base_offset_in_bytes(T_CHAR);
1718 
1719     // return true if the same array
1720 #ifdef AARCH64
1721     __ cmp(ary1_reg, ary2_reg);
1722     __ b(Lequal, eq);
1723 
1724     __ mov(result_reg, 0);
1725 
1726     __ cbz(ary1_reg, Ldone); // not equal
1727 
1728     __ cbz(ary2_reg, Ldone); // not equal
1729 #else
1730     __ teq(ary1_reg, ary2_reg);
1731     __ mov(result_reg, 1, eq);
1732     __ b(Ldone, eq); // equal
1733 
1734     __ tst(ary1_reg, ary1_reg);
1735     __ mov(result_reg, 0, eq);
1736     __ b(Ldone, eq);    // not equal
1737 
1738     __ tst(ary2_reg, ary2_reg);
1739     __ mov(result_reg, 0, eq);
1740     __ b(Ldone, eq);    // not equal
1741 #endif
1742 
1743     //load the lengths of arrays
1744     __ ldr_s32(tmp1_reg, Address(ary1_reg, length_offset)); // int
1745     __ ldr_s32(tmp2_reg, Address(ary2_reg, length_offset)); // int
1746 
1747     // return false if the two arrays are not equal length
1748 #ifdef AARCH64
1749     __ cmp_w(tmp1_reg, tmp2_reg);
1750     __ b(Ldone, ne);    // not equal
1751 
1752     __ cbz_w(tmp1_reg, Lequal); // zero-length arrays are equal
1753 #else
1754     __ teq_32(tmp1_reg, tmp2_reg);
1755     __ mov(result_reg, 0, ne);
1756     __ b(Ldone, ne);    // not equal
1757 
1758     __ tst(tmp1_reg, tmp1_reg);
1759     __ mov(result_reg, 1, eq);
1760     __ b(Ldone, eq);    // zero-length arrays are equal
1761 #endif
1762 
1763     // load array addresses
1764     __ add(ary1_reg, ary1_reg, base_offset);
1765     __ add(ary2_reg, ary2_reg, base_offset);
1766 
1767     // renaming registers
1768     Register chr1_reg  =  tmp3_reg;   // for characters in ary1
1769     Register chr2_reg  =  tmp2_reg;   // for characters in ary2
1770     Register limit_reg =  tmp1_reg;   // length
1771 
1772     // set byte count
1773     __ logical_shift_left_32(limit_reg, limit_reg, exact_log2(sizeof(jchar)));
1774 
1775     // Compare char[] arrays aligned to 4 bytes.
1776     __ char_arrays_equals(ary1_reg, ary2_reg, limit_reg, result_reg,
1777                           chr1_reg, chr2_reg, Ldone);
1778     __ bind(Lequal);
1779     __ mov(result_reg, 1);  //equal
1780 
1781     __ bind(Ldone);
1782     %}
1783 %}
1784 
1785 //----------FRAME--------------------------------------------------------------
1786 // Definition of frame structure and management information.
1787 //
1788 //  S T A C K   L A Y O U T    Allocators stack-slot number
1789 //                             |   (to get allocators register number
1790 //  G  Owned by    |        |  v    add VMRegImpl::stack0)
1791 //  r   CALLER     |        |
1792 //  o     |        +--------+      pad to even-align allocators stack-slot
1793 //  w     V        |  pad0  |        numbers; owned by CALLER
1794 //  t   -----------+--------+----> Matcher::_in_arg_limit, unaligned
1795 //  h     ^        |   in   |  5
1796 //        |        |  args  |  4   Holes in incoming args owned by SELF
1797 //  |     |        |        |  3
1798 //  |     |        +--------+
1799 //  V     |        | old out|      Empty on Intel, window on Sparc
1800 //        |    old |preserve|      Must be even aligned.
1801 //        |     SP-+--------+----> Matcher::_old_SP, 8 (or 16 in LP64)-byte aligned
1802 //        |        |   in   |  3   area for Intel ret address
1803 //     Owned by    |preserve|      Empty on Sparc.
1804 //       SELF      +--------+
1805 //        |        |  pad2  |  2   pad to align old SP
1806 //        |        +--------+  1
1807 //        |        | locks  |  0
1808 //        |        +--------+----> VMRegImpl::stack0, 8 (or 16 in LP64)-byte aligned
1809 //        |        |  pad1  | 11   pad to align new SP
1810 //        |        +--------+
1811 //        |        |        | 10
1812 //        |        | spills |  9   spills
1813 //        V        |        |  8   (pad0 slot for callee)
1814 //      -----------+--------+----> Matcher::_out_arg_limit, unaligned
1815 //        ^        |  out   |  7
1816 //        |        |  args  |  6   Holes in outgoing args owned by CALLEE
1817 //     Owned by    +--------+
1818 //      CALLEE     | new out|  6   Empty on Intel, window on Sparc
1819 //        |    new |preserve|      Must be even-aligned.
1820 //        |     SP-+--------+----> Matcher::_new_SP, even aligned
1821 //        |        |        |
1822 //
1823 // Note 1: Only region 8-11 is determined by the allocator.  Region 0-5 is
1824 //         known from SELF's arguments and the Java calling convention.
1825 //         Region 6-7 is determined per call site.
1826 // Note 2: If the calling convention leaves holes in the incoming argument
1827 //         area, those holes are owned by SELF.  Holes in the outgoing area
1828 //         are owned by the CALLEE.  Holes should not be nessecary in the
1829 //         incoming area, as the Java calling convention is completely under
1830 //         the control of the AD file.  Doubles can be sorted and packed to
1831 //         avoid holes.  Holes in the outgoing arguments may be nessecary for
1832 //         varargs C calling conventions.
1833 // Note 3: Region 0-3 is even aligned, with pad2 as needed.  Region 3-5 is
1834 //         even aligned with pad0 as needed.
1835 //         Region 6 is even aligned.  Region 6-7 is NOT even aligned;
1836 //         region 6-11 is even aligned; it may be padded out more so that
1837 //         the region from SP to FP meets the minimum stack alignment.
1838 
1839 frame %{
1840   // What direction does stack grow in (assumed to be same for native & Java)
1841   stack_direction(TOWARDS_LOW);
1842 
1843   // These two registers define part of the calling convention
1844   // between compiled code and the interpreter.
1845   inline_cache_reg(R_Ricklass);          // Inline Cache Register or Method* for I2C
1846   interpreter_method_oop_reg(R_Rmethod); // Method Oop Register when calling interpreter
1847 
1848   // Optional: name the operand used by cisc-spilling to access [stack_pointer + offset]
1849   cisc_spilling_operand_name(indOffset);
1850 
1851   // Number of stack slots consumed by a Monitor enter
1852   sync_stack_slots(1 * VMRegImpl::slots_per_word);
1853 
1854   // Compiled code's Frame Pointer
1855 #ifdef AARCH64
1856   frame_pointer(R_SP);
1857 #else
1858   frame_pointer(R_R13);
1859 #endif
1860 
1861   // Stack alignment requirement
1862   stack_alignment(StackAlignmentInBytes);
1863   //  LP64: Alignment size in bytes (128-bit -> 16 bytes)
1864   // !LP64: Alignment size in bytes (64-bit  ->  8 bytes)
1865 
1866   // Number of stack slots between incoming argument block and the start of
1867   // a new frame.  The PROLOG must add this many slots to the stack.  The
1868   // EPILOG must remove this many slots.
1869   // FP + LR
1870   in_preserve_stack_slots(2 * VMRegImpl::slots_per_word);
1871 
1872   // Number of outgoing stack slots killed above the out_preserve_stack_slots
1873   // for calls to C.  Supports the var-args backing area for register parms.
1874   // ADLC doesn't support parsing expressions, so I folded the math by hand.
1875   varargs_C_out_slots_killed( 0);
1876 
1877   // The after-PROLOG location of the return address.  Location of
1878   // return address specifies a type (REG or STACK) and a number
1879   // representing the register number (i.e. - use a register name) or
1880   // stack slot.
1881   // Ret Addr is on stack in slot 0 if no locks or verification or alignment.
1882   // Otherwise, it is above the locks and verification slot and alignment word
1883   return_addr(STACK - 1*VMRegImpl::slots_per_word +
1884               align_up((Compile::current()->in_preserve_stack_slots() +
1885                         Compile::current()->fixed_slots()),
1886                        stack_alignment_in_slots()));
1887 
1888   // Body of function which returns an OptoRegs array locating
1889   // arguments either in registers or in stack slots for calling
1890   // java
1891   calling_convention %{
1892     (void) SharedRuntime::java_calling_convention(sig_bt, regs, length, is_outgoing);
1893 
1894   %}
1895 
1896   // Body of function which returns an OptoRegs array locating
1897   // arguments either in registers or in stack slots for callin
1898   // C.
1899   c_calling_convention %{
1900     // This is obviously always outgoing
1901     (void) SharedRuntime::c_calling_convention(sig_bt, regs, /*regs2=*/NULL, length);
1902   %}
1903 
1904   // Location of compiled Java return values.  Same as C
1905   return_value %{
1906     return c2::return_value(ideal_reg);
1907   %}
1908 
1909 %}
1910 
1911 //----------ATTRIBUTES---------------------------------------------------------
1912 //----------Instruction Attributes---------------------------------------------
1913 ins_attrib ins_cost(DEFAULT_COST); // Required cost attribute
1914 ins_attrib ins_size(32);           // Required size attribute (in bits)
1915 ins_attrib ins_short_branch(0);    // Required flag: is this instruction a
1916                                    // non-matching short branch variant of some
1917                                                             // long branch?
1918 
1919 //----------OPERANDS-----------------------------------------------------------
1920 // Operand definitions must precede instruction definitions for correct parsing
1921 // in the ADLC because operands constitute user defined types which are used in
1922 // instruction definitions.
1923 
1924 //----------Simple Operands----------------------------------------------------
1925 // Immediate Operands
1926 // Integer Immediate: 32-bit
1927 operand immI() %{
1928   match(ConI);
1929 
1930   op_cost(0);
1931   // formats are generated automatically for constants and base registers
1932   format %{ %}
1933   interface(CONST_INTER);
1934 %}
1935 
1936 // Integer Immediate: 8-bit unsigned - for VMOV
1937 operand immU8() %{
1938   predicate(0 <= n->get_int() && (n->get_int() <= 255));
1939   match(ConI);
1940   op_cost(0);
1941 
1942   format %{ %}
1943   interface(CONST_INTER);
1944 %}
1945 
1946 // Integer Immediate: 16-bit
1947 operand immI16() %{
1948   predicate((n->get_int() >> 16) == 0 && VM_Version::supports_movw());
1949   match(ConI);
1950   op_cost(0);
1951 
1952   format %{ %}
1953   interface(CONST_INTER);
1954 %}
1955 
1956 #ifndef AARCH64
1957 // Integer Immediate: offset for half and double word loads and stores
1958 operand immIHD() %{
1959   predicate(is_memoryHD(n->get_int()));
1960   match(ConI);
1961   op_cost(0);
1962   format %{ %}
1963   interface(CONST_INTER);
1964 %}
1965 
1966 // Integer Immediate: offset for fp loads and stores
1967 operand immIFP() %{
1968   predicate(is_memoryfp(n->get_int()) && ((n->get_int() & 3) == 0));
1969   match(ConI);
1970   op_cost(0);
1971 
1972   format %{ %}
1973   interface(CONST_INTER);
1974 %}
1975 #endif
1976 
1977 // Valid scale values for addressing modes and shifts
1978 operand immU5() %{
1979   predicate(0 <= n->get_int() && (n->get_int() <= 31));
1980   match(ConI);
1981   op_cost(0);
1982 
1983   format %{ %}
1984   interface(CONST_INTER);
1985 %}
1986 
1987 // Integer Immediate: 6-bit
1988 operand immU6Big() %{
1989   predicate(n->get_int() >= 32 && n->get_int() <= 63);
1990   match(ConI);
1991   op_cost(0);
1992   format %{ %}
1993   interface(CONST_INTER);
1994 %}
1995 
1996 // Integer Immediate: 0-bit
1997 operand immI0() %{
1998   predicate(n->get_int() == 0);
1999   match(ConI);
2000   op_cost(0);
2001 
2002   format %{ %}
2003   interface(CONST_INTER);
2004 %}
2005 
2006 // Integer Immediate: the value 1
2007 operand immI_1() %{
2008   predicate(n->get_int() == 1);
2009   match(ConI);
2010   op_cost(0);
2011 
2012   format %{ %}
2013   interface(CONST_INTER);
2014 %}
2015 
2016 // Integer Immediate: the value 2
2017 operand immI_2() %{
2018   predicate(n->get_int() == 2);
2019   match(ConI);
2020   op_cost(0);
2021 
2022   format %{ %}
2023   interface(CONST_INTER);
2024 %}
2025 
2026 // Integer Immediate: the value 3
2027 operand immI_3() %{
2028   predicate(n->get_int() == 3);
2029   match(ConI);
2030   op_cost(0);
2031 
2032   format %{ %}
2033   interface(CONST_INTER);
2034 %}
2035 
2036 // Integer Immediate: the value 4
2037 operand immI_4() %{
2038   predicate(n->get_int() == 4);
2039   match(ConI);
2040   op_cost(0);
2041 
2042   format %{ %}
2043   interface(CONST_INTER);
2044 %}
2045 
2046 // Integer Immediate: the value 8
2047 operand immI_8() %{
2048   predicate(n->get_int() == 8);
2049   match(ConI);
2050   op_cost(0);
2051 
2052   format %{ %}
2053   interface(CONST_INTER);
2054 %}
2055 
2056 // Int Immediate non-negative
2057 operand immU31()
2058 %{
2059   predicate(n->get_int() >= 0);
2060   match(ConI);
2061 
2062   op_cost(0);
2063   format %{ %}
2064   interface(CONST_INTER);
2065 %}
2066 
2067 // Integer Immediate: the values 32-63
2068 operand immI_32_63() %{
2069   predicate(n->get_int() >= 32 && n->get_int() <= 63);
2070   match(ConI);
2071   op_cost(0);
2072 
2073   format %{ %}
2074   interface(CONST_INTER);
2075 %}
2076 
2077 // Immediates for special shifts (sign extend)
2078 
2079 // Integer Immediate: the value 16
2080 operand immI_16() %{
2081   predicate(n->get_int() == 16);
2082   match(ConI);
2083   op_cost(0);
2084 
2085   format %{ %}
2086   interface(CONST_INTER);
2087 %}
2088 
2089 // Integer Immediate: the value 24
2090 operand immI_24() %{
2091   predicate(n->get_int() == 24);
2092   match(ConI);
2093   op_cost(0);
2094 
2095   format %{ %}
2096   interface(CONST_INTER);
2097 %}
2098 
2099 // Integer Immediate: the value 255
2100 operand immI_255() %{
2101   predicate( n->get_int() == 255 );
2102   match(ConI);
2103   op_cost(0);
2104 
2105   format %{ %}
2106   interface(CONST_INTER);
2107 %}
2108 
2109 // Integer Immediate: the value 65535
2110 operand immI_65535() %{
2111   predicate(n->get_int() == 65535);
2112   match(ConI);
2113   op_cost(0);
2114 
2115   format %{ %}
2116   interface(CONST_INTER);
2117 %}
2118 
2119 // Integer Immediates for arithmetic instructions
2120 
2121 operand aimmI() %{
2122   predicate(is_aimm(n->get_int()));
2123   match(ConI);
2124   op_cost(0);
2125 
2126   format %{ %}
2127   interface(CONST_INTER);
2128 %}
2129 
2130 operand aimmIneg() %{
2131   predicate(is_aimm(-n->get_int()));
2132   match(ConI);
2133   op_cost(0);
2134 
2135   format %{ %}
2136   interface(CONST_INTER);
2137 %}
2138 
2139 operand aimmU31() %{
2140   predicate((0 <= n->get_int()) && is_aimm(n->get_int()));
2141   match(ConI);
2142   op_cost(0);
2143 
2144   format %{ %}
2145   interface(CONST_INTER);
2146 %}
2147 
2148 // Integer Immediates for logical instructions
2149 
2150 operand limmI() %{
2151   predicate(is_limmI(n->get_int()));
2152   match(ConI);
2153   op_cost(0);
2154 
2155   format %{ %}
2156   interface(CONST_INTER);
2157 %}
2158 
2159 operand limmIlow8() %{
2160   predicate(is_limmI_low(n->get_int(), 8));
2161   match(ConI);
2162   op_cost(0);
2163 
2164   format %{ %}
2165   interface(CONST_INTER);
2166 %}
2167 
2168 operand limmU31() %{
2169   predicate(0 <= n->get_int() && is_limmI(n->get_int()));
2170   match(ConI);
2171   op_cost(0);
2172 
2173   format %{ %}
2174   interface(CONST_INTER);
2175 %}
2176 
2177 operand limmIn() %{
2178   predicate(is_limmI(~n->get_int()));
2179   match(ConI);
2180   op_cost(0);
2181 
2182   format %{ %}
2183   interface(CONST_INTER);
2184 %}
2185 
2186 #ifdef AARCH64
2187 // Long Immediate: for logical instruction
2188 operand limmL() %{
2189   predicate(is_limmL(n->get_long()));
2190   match(ConL);
2191   op_cost(0);
2192 
2193   format %{ %}
2194   interface(CONST_INTER);
2195 %}
2196 
2197 operand limmLn() %{
2198   predicate(is_limmL(~n->get_long()));
2199   match(ConL);
2200   op_cost(0);
2201 
2202   format %{ %}
2203   interface(CONST_INTER);
2204 %}
2205 
2206 // Long Immediate: for arithmetic instruction
2207 operand aimmL() %{
2208   predicate(is_aimm(n->get_long()));
2209   match(ConL);
2210   op_cost(0);
2211 
2212   format %{ %}
2213   interface(CONST_INTER);
2214 %}
2215 
2216 operand aimmLneg() %{
2217   predicate(is_aimm(-n->get_long()));
2218   match(ConL);
2219   op_cost(0);
2220 
2221   format %{ %}
2222   interface(CONST_INTER);
2223 %}
2224 #endif // AARCH64
2225 
2226 // Long Immediate: the value FF
2227 operand immL_FF() %{
2228   predicate( n->get_long() == 0xFFL );
2229   match(ConL);
2230   op_cost(0);
2231 
2232   format %{ %}
2233   interface(CONST_INTER);
2234 %}
2235 
2236 // Long Immediate: the value FFFF
2237 operand immL_FFFF() %{
2238   predicate( n->get_long() == 0xFFFFL );
2239   match(ConL);
2240   op_cost(0);
2241 
2242   format %{ %}
2243   interface(CONST_INTER);
2244 %}
2245 
2246 // Pointer Immediate: 32 or 64-bit
2247 operand immP() %{
2248   match(ConP);
2249 
2250   op_cost(5);
2251   // formats are generated automatically for constants and base registers
2252   format %{ %}
2253   interface(CONST_INTER);
2254 %}
2255 
2256 operand immP0() %{
2257   predicate(n->get_ptr() == 0);
2258   match(ConP);
2259   op_cost(0);
2260 
2261   format %{ %}
2262   interface(CONST_INTER);
2263 %}
2264 
2265 operand immP_poll() %{
2266   predicate(n->get_ptr() != 0 && n->get_ptr() == (intptr_t)os::get_polling_page());
2267   match(ConP);
2268 
2269   // formats are generated automatically for constants and base registers
2270   format %{ %}
2271   interface(CONST_INTER);
2272 %}
2273 
2274 // Pointer Immediate
2275 operand immN()
2276 %{
2277   match(ConN);
2278 
2279   op_cost(10);
2280   format %{ %}
2281   interface(CONST_INTER);
2282 %}
2283 
2284 operand immNKlass()
2285 %{
2286   match(ConNKlass);
2287 
2288   op_cost(10);
2289   format %{ %}
2290   interface(CONST_INTER);
2291 %}
2292 
2293 // NULL Pointer Immediate
2294 operand immN0()
2295 %{
2296   predicate(n->get_narrowcon() == 0);
2297   match(ConN);
2298 
2299   op_cost(0);
2300   format %{ %}
2301   interface(CONST_INTER);
2302 %}
2303 
2304 operand immL() %{
2305   match(ConL);
2306   op_cost(40);
2307   // formats are generated automatically for constants and base registers
2308   format %{ %}
2309   interface(CONST_INTER);
2310 %}
2311 
2312 operand immL0() %{
2313   predicate(n->get_long() == 0L);
2314   match(ConL);
2315   op_cost(0);
2316   // formats are generated automatically for constants and base registers
2317   format %{ %}
2318   interface(CONST_INTER);
2319 %}
2320 
2321 // Long Immediate: 16-bit
2322 operand immL16() %{
2323   predicate(n->get_long() >= 0 && n->get_long() < (1<<16)  && VM_Version::supports_movw());
2324   match(ConL);
2325   op_cost(0);
2326 
2327   format %{ %}
2328   interface(CONST_INTER);
2329 %}
2330 
2331 // Long Immediate: low 32-bit mask
2332 operand immL_32bits() %{
2333   predicate(n->get_long() == 0xFFFFFFFFL);
2334   match(ConL);
2335   op_cost(0);
2336 
2337   format %{ %}
2338   interface(CONST_INTER);
2339 %}
2340 
2341 // Double Immediate
2342 operand immD() %{
2343   match(ConD);
2344 
2345   op_cost(40);
2346   format %{ %}
2347   interface(CONST_INTER);
2348 %}
2349 
2350 // Double Immediate: +0.0d.
2351 operand immD0() %{
2352   predicate(jlong_cast(n->getd()) == 0);
2353 
2354   match(ConD);
2355   op_cost(0);
2356   format %{ %}
2357   interface(CONST_INTER);
2358 %}
2359 
2360 operand imm8D() %{
2361   predicate(Assembler::double_num(n->getd()).can_be_imm8());
2362   match(ConD);
2363 
2364   op_cost(0);
2365   format %{ %}
2366   interface(CONST_INTER);
2367 %}
2368 
2369 // Float Immediate
2370 operand immF() %{
2371   match(ConF);
2372 
2373   op_cost(20);
2374   format %{ %}
2375   interface(CONST_INTER);
2376 %}
2377 
2378 // Float Immediate: +0.0f
2379 operand immF0() %{
2380   predicate(jint_cast(n->getf()) == 0);
2381   match(ConF);
2382 
2383   op_cost(0);
2384   format %{ %}
2385   interface(CONST_INTER);
2386 %}
2387 
2388 // Float Immediate: encoded as 8 bits
2389 operand imm8F() %{
2390   predicate(Assembler::float_num(n->getf()).can_be_imm8());
2391   match(ConF);
2392 
2393   op_cost(0);
2394   format %{ %}
2395   interface(CONST_INTER);
2396 %}
2397 
2398 // Integer Register Operands
2399 // Integer Register
2400 operand iRegI() %{
2401   constraint(ALLOC_IN_RC(int_reg));
2402   match(RegI);
2403   match(R0RegI);
2404   match(R1RegI);
2405   match(R2RegI);
2406   match(R3RegI);
2407 #ifdef AARCH64
2408   match(ZRRegI);
2409 #else
2410   match(R12RegI);
2411 #endif
2412 
2413   format %{ %}
2414   interface(REG_INTER);
2415 %}
2416 
2417 // Pointer Register
2418 operand iRegP() %{
2419   constraint(ALLOC_IN_RC(ptr_reg));
2420   match(RegP);
2421   match(R0RegP);
2422   match(R1RegP);
2423   match(R2RegP);
2424   match(RExceptionRegP);
2425   match(R8RegP);
2426   match(R9RegP);
2427   match(RthreadRegP); // FIXME: move to sp_ptr_RegP?
2428   match(R12RegP);
2429   match(LRRegP);
2430 
2431   match(sp_ptr_RegP);
2432   match(store_ptr_RegP);
2433 
2434   format %{ %}
2435   interface(REG_INTER);
2436 %}
2437 
2438 // GPRs + Rthread + SP
2439 operand sp_ptr_RegP() %{
2440   constraint(ALLOC_IN_RC(sp_ptr_reg));
2441   match(RegP);
2442   match(iRegP);
2443   match(SPRegP); // FIXME: check cost
2444 
2445   format %{ %}
2446   interface(REG_INTER);
2447 %}
2448 
2449 #ifdef AARCH64
2450 // Like sp_ptr_reg, but exclude regs (Aarch64 SP) that can't be
2451 // stored directly.  Includes ZR, so can't be used as a destination.
2452 operand store_ptr_RegP() %{
2453   constraint(ALLOC_IN_RC(store_ptr_reg));
2454   match(RegP);
2455   match(iRegP);
2456   match(ZRRegP);
2457 
2458   format %{ %}
2459   interface(REG_INTER);
2460 %}
2461 
2462 operand store_RegI() %{
2463   constraint(ALLOC_IN_RC(store_reg));
2464   match(RegI);
2465   match(iRegI);
2466   match(ZRRegI);
2467 
2468   format %{ %}
2469   interface(REG_INTER);
2470 %}
2471 
2472 operand store_RegL() %{
2473   constraint(ALLOC_IN_RC(store_ptr_reg));
2474   match(RegL);
2475   match(iRegL);
2476   match(ZRRegL);
2477 
2478   format %{ %}
2479   interface(REG_INTER);
2480 %}
2481 
2482 operand store_RegN() %{
2483   constraint(ALLOC_IN_RC(store_reg));
2484   match(RegN);
2485   match(iRegN);
2486   match(ZRRegN);
2487 
2488   format %{ %}
2489   interface(REG_INTER);
2490 %}
2491 #endif
2492 
2493 operand R0RegP() %{
2494   constraint(ALLOC_IN_RC(R0_regP));
2495   match(iRegP);
2496 
2497   format %{ %}
2498   interface(REG_INTER);
2499 %}
2500 
2501 operand R1RegP() %{
2502   constraint(ALLOC_IN_RC(R1_regP));
2503   match(iRegP);
2504 
2505   format %{ %}
2506   interface(REG_INTER);
2507 %}
2508 
2509 operand R2RegP() %{
2510   constraint(ALLOC_IN_RC(R2_regP));
2511   match(iRegP);
2512 
2513   format %{ %}
2514   interface(REG_INTER);
2515 %}
2516 
2517 operand RExceptionRegP() %{
2518   constraint(ALLOC_IN_RC(Rexception_regP));
2519   match(iRegP);
2520 
2521   format %{ %}
2522   interface(REG_INTER);
2523 %}
2524 
2525 operand RthreadRegP() %{
2526   constraint(ALLOC_IN_RC(Rthread_regP));
2527   match(iRegP);
2528 
2529   format %{ %}
2530   interface(REG_INTER);
2531 %}
2532 
2533 operand IPRegP() %{
2534   constraint(ALLOC_IN_RC(IP_regP));
2535   match(iRegP);
2536 
2537   format %{ %}
2538   interface(REG_INTER);
2539 %}
2540 
2541 operand LRRegP() %{
2542   constraint(ALLOC_IN_RC(LR_regP));
2543   match(iRegP);
2544 
2545   format %{ %}
2546   interface(REG_INTER);
2547 %}
2548 
2549 operand R0RegI() %{
2550   constraint(ALLOC_IN_RC(R0_regI));
2551   match(iRegI);
2552 
2553   format %{ %}
2554   interface(REG_INTER);
2555 %}
2556 
2557 operand R1RegI() %{
2558   constraint(ALLOC_IN_RC(R1_regI));
2559   match(iRegI);
2560 
2561   format %{ %}
2562   interface(REG_INTER);
2563 %}
2564 
2565 operand R2RegI() %{
2566   constraint(ALLOC_IN_RC(R2_regI));
2567   match(iRegI);
2568 
2569   format %{ %}
2570   interface(REG_INTER);
2571 %}
2572 
2573 operand R3RegI() %{
2574   constraint(ALLOC_IN_RC(R3_regI));
2575   match(iRegI);
2576 
2577   format %{ %}
2578   interface(REG_INTER);
2579 %}
2580 
2581 #ifndef AARCH64
2582 operand R12RegI() %{
2583   constraint(ALLOC_IN_RC(R12_regI));
2584   match(iRegI);
2585 
2586   format %{ %}
2587   interface(REG_INTER);
2588 %}
2589 #endif
2590 
2591 // Long Register
2592 operand iRegL() %{
2593   constraint(ALLOC_IN_RC(long_reg));
2594   match(RegL);
2595 #ifdef AARCH64
2596   match(iRegLd);
2597 #else
2598   match(R0R1RegL);
2599   match(R2R3RegL);
2600 #endif
2601 //match(iRegLex);
2602 
2603   format %{ %}
2604   interface(REG_INTER);
2605 %}
2606 
2607 operand iRegLd() %{
2608   constraint(ALLOC_IN_RC(long_reg_align));
2609   match(iRegL); // FIXME: allows unaligned R11/R12?
2610 
2611   format %{ %}
2612   interface(REG_INTER);
2613 %}
2614 
2615 #ifndef AARCH64
2616 // first long arg, or return value
2617 operand R0R1RegL() %{
2618   constraint(ALLOC_IN_RC(R0R1_regL));
2619   match(iRegL);
2620 
2621   format %{ %}
2622   interface(REG_INTER);
2623 %}
2624 
2625 operand R2R3RegL() %{
2626   constraint(ALLOC_IN_RC(R2R3_regL));
2627   match(iRegL);
2628 
2629   format %{ %}
2630   interface(REG_INTER);
2631 %}
2632 #endif
2633 
2634 // Condition Code Flag Register
2635 operand flagsReg() %{
2636   constraint(ALLOC_IN_RC(int_flags));
2637   match(RegFlags);
2638 
2639   format %{ "apsr" %}
2640   interface(REG_INTER);
2641 %}
2642 
2643 // Result of compare to 0 (TST)
2644 operand flagsReg_EQNELTGE() %{
2645   constraint(ALLOC_IN_RC(int_flags));
2646   match(RegFlags);
2647 
2648   format %{ "apsr_EQNELTGE" %}
2649   interface(REG_INTER);
2650 %}
2651 
2652 // Condition Code Register, unsigned comparisons.
2653 operand flagsRegU() %{
2654   constraint(ALLOC_IN_RC(int_flags));
2655   match(RegFlags);
2656 #ifdef TODO
2657   match(RegFlagsP);
2658 #endif
2659 
2660   format %{ "apsr_U" %}
2661   interface(REG_INTER);
2662 %}
2663 
2664 // Condition Code Register, pointer comparisons.
2665 operand flagsRegP() %{
2666   constraint(ALLOC_IN_RC(int_flags));
2667   match(RegFlags);
2668 
2669   format %{ "apsr_P" %}
2670   interface(REG_INTER);
2671 %}
2672 
2673 // Condition Code Register, long comparisons.
2674 #ifndef AARCH64
2675 operand flagsRegL_LTGE() %{
2676   constraint(ALLOC_IN_RC(int_flags));
2677   match(RegFlags);
2678 
2679   format %{ "apsr_L_LTGE" %}
2680   interface(REG_INTER);
2681 %}
2682 
2683 operand flagsRegL_EQNE() %{
2684   constraint(ALLOC_IN_RC(int_flags));
2685   match(RegFlags);
2686 
2687   format %{ "apsr_L_EQNE" %}
2688   interface(REG_INTER);
2689 %}
2690 
2691 operand flagsRegL_LEGT() %{
2692   constraint(ALLOC_IN_RC(int_flags));
2693   match(RegFlags);
2694 
2695   format %{ "apsr_L_LEGT" %}
2696   interface(REG_INTER);
2697 %}
2698 
2699 operand flagsRegUL_LTGE() %{
2700   constraint(ALLOC_IN_RC(int_flags));
2701   match(RegFlags);
2702 
2703   format %{ "apsr_UL_LTGE" %}
2704   interface(REG_INTER);
2705 %}
2706 
2707 operand flagsRegUL_EQNE() %{
2708   constraint(ALLOC_IN_RC(int_flags));
2709   match(RegFlags);
2710 
2711   format %{ "apsr_UL_EQNE" %}
2712   interface(REG_INTER);
2713 %}
2714 
2715 operand flagsRegUL_LEGT() %{
2716   constraint(ALLOC_IN_RC(int_flags));
2717   match(RegFlags);
2718 
2719   format %{ "apsr_UL_LEGT" %}
2720   interface(REG_INTER);
2721 %}
2722 #endif
2723 
2724 // Condition Code Register, floating comparisons, unordered same as "less".
2725 operand flagsRegF() %{
2726   constraint(ALLOC_IN_RC(float_flags));
2727   match(RegFlags);
2728 
2729   format %{ "fpscr_F" %}
2730   interface(REG_INTER);
2731 %}
2732 
2733 // Vectors
2734 operand vecD() %{
2735   constraint(ALLOC_IN_RC(actual_dflt_reg));
2736   match(VecD);
2737 
2738   format %{ %}
2739   interface(REG_INTER);
2740 %}
2741 
2742 operand vecX() %{
2743   constraint(ALLOC_IN_RC(vectorx_reg));
2744   match(VecX);
2745 
2746   format %{ %}
2747   interface(REG_INTER);
2748 %}
2749 
2750 operand regD() %{
2751   constraint(ALLOC_IN_RC(actual_dflt_reg));
2752   match(RegD);
2753   match(regD_low);
2754 
2755   format %{ %}
2756   interface(REG_INTER);
2757 %}
2758 
2759 operand regF() %{
2760   constraint(ALLOC_IN_RC(sflt_reg));
2761   match(RegF);
2762 
2763   format %{ %}
2764   interface(REG_INTER);
2765 %}
2766 
2767 operand regD_low() %{
2768   constraint(ALLOC_IN_RC(dflt_low_reg));
2769   match(RegD);
2770 
2771   format %{ %}
2772   interface(REG_INTER);
2773 %}
2774 
2775 // Special Registers
2776 
2777 // Method Register
2778 operand inline_cache_regP(iRegP reg) %{
2779   constraint(ALLOC_IN_RC(Ricklass_regP));
2780   match(reg);
2781   format %{ %}
2782   interface(REG_INTER);
2783 %}
2784 
2785 operand interpreter_method_oop_regP(iRegP reg) %{
2786   constraint(ALLOC_IN_RC(Rmethod_regP));
2787   match(reg);
2788   format %{ %}
2789   interface(REG_INTER);
2790 %}
2791 
2792 
2793 //----------Complex Operands---------------------------------------------------
2794 // Indirect Memory Reference
2795 operand indirect(sp_ptr_RegP reg) %{
2796   constraint(ALLOC_IN_RC(sp_ptr_reg));
2797   match(reg);
2798 
2799   op_cost(100);
2800   format %{ "[$reg]" %}
2801   interface(MEMORY_INTER) %{
2802     base($reg);
2803 #ifdef AARCH64
2804     index(0xff); // 0xff => no index
2805 #else
2806     index(0xf); // PC => no index
2807 #endif
2808     scale(0x0);
2809     disp(0x0);
2810   %}
2811 %}
2812 
2813 #ifdef AARCH64
2814 // Indirect with scaled*1 uimm12 offset
2815 operand indOffsetU12ScaleB(sp_ptr_RegP reg, immUL12 offset) %{
2816   constraint(ALLOC_IN_RC(sp_ptr_reg));
2817   match(AddP reg offset);
2818 
2819   op_cost(100);
2820   format %{ "[$reg + $offset]" %}
2821   interface(MEMORY_INTER) %{
2822     base($reg);
2823 #ifdef AARCH64
2824     index(0xff); // 0xff => no index
2825 #else
2826     index(0xf); // PC => no index
2827 #endif
2828     scale(0x0);
2829     disp($offset);
2830   %}
2831 %}
2832 
2833 // Indirect with scaled*2 uimm12 offset
2834 operand indOffsetU12ScaleS(sp_ptr_RegP reg, immUL12x2 offset) %{
2835   constraint(ALLOC_IN_RC(sp_ptr_reg));
2836   match(AddP reg offset);
2837 
2838   op_cost(100);
2839   format %{ "[$reg + $offset]" %}
2840   interface(MEMORY_INTER) %{
2841     base($reg);
2842 #ifdef AARCH64
2843     index(0xff); // 0xff => no index
2844 #else
2845     index(0xf); // PC => no index
2846 #endif
2847     scale(0x0);
2848     disp($offset);
2849   %}
2850 %}
2851 
2852 // Indirect with scaled*4 uimm12 offset
2853 operand indOffsetU12ScaleI(sp_ptr_RegP reg, immUL12x4 offset) %{
2854   constraint(ALLOC_IN_RC(sp_ptr_reg));
2855   match(AddP reg offset);
2856 
2857   op_cost(100);
2858   format %{ "[$reg + $offset]" %}
2859   interface(MEMORY_INTER) %{
2860     base($reg);
2861 #ifdef AARCH64
2862     index(0xff); // 0xff => no index
2863 #else
2864     index(0xf); // PC => no index
2865 #endif
2866     scale(0x0);
2867     disp($offset);
2868   %}
2869 %}
2870 
2871 // Indirect with scaled*8 uimm12 offset
2872 operand indOffsetU12ScaleL(sp_ptr_RegP reg, immUL12x8 offset) %{
2873   constraint(ALLOC_IN_RC(sp_ptr_reg));
2874   match(AddP reg offset);
2875 
2876   op_cost(100);
2877   format %{ "[$reg + $offset]" %}
2878   interface(MEMORY_INTER) %{
2879     base($reg);
2880 #ifdef AARCH64
2881     index(0xff); // 0xff => no index
2882 #else
2883     index(0xf); // PC => no index
2884 #endif
2885     scale(0x0);
2886     disp($offset);
2887   %}
2888 %}
2889 
2890 // Indirect with scaled*16 uimm12 offset
2891 operand indOffsetU12ScaleQ(sp_ptr_RegP reg, immUL12x16 offset) %{
2892   constraint(ALLOC_IN_RC(sp_ptr_reg));
2893   match(AddP reg offset);
2894 
2895   op_cost(100);
2896   format %{ "[$reg + $offset]" %}
2897   interface(MEMORY_INTER) %{
2898     base($reg);
2899 #ifdef AARCH64
2900     index(0xff); // 0xff => no index
2901 #else
2902     index(0xf); // PC => no index
2903 #endif
2904     scale(0x0);
2905     disp($offset);
2906   %}
2907 %}
2908 
2909 #else // ! AARCH64
2910 
2911 // Indirect with Offset in ]-4096, 4096[
2912 operand indOffset12(sp_ptr_RegP reg, immI12 offset) %{
2913   constraint(ALLOC_IN_RC(sp_ptr_reg));
2914   match(AddP reg offset);
2915 
2916   op_cost(100);
2917   format %{ "[$reg + $offset]" %}
2918   interface(MEMORY_INTER) %{
2919     base($reg);
2920 #ifdef AARCH64
2921     index(0xff); // 0xff => no index
2922 #else
2923     index(0xf); // PC => no index
2924 #endif
2925     scale(0x0);
2926     disp($offset);
2927   %}
2928 %}
2929 
2930 // Indirect with offset for float load/store
2931 operand indOffsetFP(sp_ptr_RegP reg, immIFP offset) %{
2932   constraint(ALLOC_IN_RC(sp_ptr_reg));
2933   match(AddP reg offset);
2934 
2935   op_cost(100);
2936   format %{ "[$reg + $offset]" %}
2937   interface(MEMORY_INTER) %{
2938     base($reg);
2939 #ifdef AARCH64
2940     index(0xff); // 0xff => no index
2941 #else
2942     index(0xf); // PC => no index
2943 #endif
2944     scale(0x0);
2945     disp($offset);
2946   %}
2947 %}
2948 
2949 // Indirect with Offset for half and double words
2950 operand indOffsetHD(sp_ptr_RegP reg, immIHD offset) %{
2951   constraint(ALLOC_IN_RC(sp_ptr_reg));
2952   match(AddP reg offset);
2953 
2954   op_cost(100);
2955   format %{ "[$reg + $offset]" %}
2956   interface(MEMORY_INTER) %{
2957     base($reg);
2958 #ifdef AARCH64
2959     index(0xff); // 0xff => no index
2960 #else
2961     index(0xf); // PC => no index
2962 #endif
2963     scale(0x0);
2964     disp($offset);
2965   %}
2966 %}
2967 
2968 // Indirect with Offset and Offset+4 in ]-1024, 1024[
2969 operand indOffsetFPx2(sp_ptr_RegP reg, immX10x2 offset) %{
2970   constraint(ALLOC_IN_RC(sp_ptr_reg));
2971   match(AddP reg offset);
2972 
2973   op_cost(100);
2974   format %{ "[$reg + $offset]" %}
2975   interface(MEMORY_INTER) %{
2976     base($reg);
2977 #ifdef AARCH64
2978     index(0xff); // 0xff => no index
2979 #else
2980     index(0xf); // PC => no index
2981 #endif
2982     scale(0x0);
2983     disp($offset);
2984   %}
2985 %}
2986 
2987 // Indirect with Offset and Offset+4 in ]-4096, 4096[
2988 operand indOffset12x2(sp_ptr_RegP reg, immI12x2 offset) %{
2989   constraint(ALLOC_IN_RC(sp_ptr_reg));
2990   match(AddP reg offset);
2991 
2992   op_cost(100);
2993   format %{ "[$reg + $offset]" %}
2994   interface(MEMORY_INTER) %{
2995     base($reg);
2996 #ifdef AARCH64
2997     index(0xff); // 0xff => no index
2998 #else
2999     index(0xf); // PC => no index
3000 #endif
3001     scale(0x0);
3002     disp($offset);
3003   %}
3004 %}
3005 #endif // !AARCH64
3006 
3007 // Indirect with Register Index
3008 operand indIndex(iRegP addr, iRegX index) %{
3009   constraint(ALLOC_IN_RC(ptr_reg));
3010   match(AddP addr index);
3011 
3012   op_cost(100);
3013   format %{ "[$addr + $index]" %}
3014   interface(MEMORY_INTER) %{
3015     base($addr);
3016     index($index);
3017     scale(0x0);
3018     disp(0x0);
3019   %}
3020 %}
3021 
3022 #ifdef AARCH64
3023 // Indirect Memory Times Scale Plus Index Register
3024 operand indIndexScaleS(iRegP addr, iRegX index, immI_1 scale) %{
3025   constraint(ALLOC_IN_RC(ptr_reg));
3026   match(AddP addr (LShiftX index scale));
3027 
3028   op_cost(100);
3029   format %{"[$addr + $index << $scale]" %}
3030   interface(MEMORY_INTER) %{
3031     base($addr);
3032     index($index);
3033     scale($scale);
3034     disp(0x0);
3035   %}
3036 %}
3037 
3038 // Indirect Memory Times Scale Plus 32-bit Index Register
3039 operand indIndexIScaleS(iRegP addr, iRegI index, immI_1 scale) %{
3040   constraint(ALLOC_IN_RC(ptr_reg));
3041   match(AddP addr (LShiftX (ConvI2L index) scale));
3042 
3043   op_cost(100);
3044   format %{"[$addr + $index.w << $scale]" %}
3045   interface(MEMORY_INTER) %{
3046     base($addr);
3047     index($index);
3048     scale($scale);
3049     disp(0x7fffffff); // sxtw
3050   %}
3051 %}
3052 
3053 // Indirect Memory Times Scale Plus Index Register
3054 operand indIndexScaleI(iRegP addr, iRegX index, immI_2 scale) %{
3055   constraint(ALLOC_IN_RC(ptr_reg));
3056   match(AddP addr (LShiftX index scale));
3057 
3058   op_cost(100);
3059   format %{"[$addr + $index << $scale]" %}
3060   interface(MEMORY_INTER) %{
3061     base($addr);
3062     index($index);
3063     scale($scale);
3064     disp(0x0);
3065   %}
3066 %}
3067 
3068 // Indirect Memory Times Scale Plus 32-bit Index Register
3069 operand indIndexIScaleI(iRegP addr, iRegI index, immI_2 scale) %{
3070   constraint(ALLOC_IN_RC(ptr_reg));
3071   match(AddP addr (LShiftX (ConvI2L index) scale));
3072 
3073   op_cost(100);
3074   format %{"[$addr + $index.w << $scale]" %}
3075   interface(MEMORY_INTER) %{
3076     base($addr);
3077     index($index);
3078     scale($scale);
3079     disp(0x7fffffff); // sxtw
3080   %}
3081 %}
3082 
3083 // Indirect Memory Times Scale Plus Index Register
3084 operand indIndexScaleL(iRegP addr, iRegX index, immI_3 scale) %{
3085   constraint(ALLOC_IN_RC(ptr_reg));
3086   match(AddP addr (LShiftX index scale));
3087 
3088   op_cost(100);
3089   format %{"[$addr + $index << $scale]" %}
3090   interface(MEMORY_INTER) %{
3091     base($addr);
3092     index($index);
3093     scale($scale);
3094     disp(0x0);
3095   %}
3096 %}
3097 
3098 // Indirect Memory Times Scale Plus 32-bit Index Register
3099 operand indIndexIScaleL(iRegP addr, iRegI index, immI_3 scale) %{
3100   constraint(ALLOC_IN_RC(ptr_reg));
3101   match(AddP addr (LShiftX (ConvI2L index) scale));
3102 
3103   op_cost(100);
3104   format %{"[$addr + $index.w << $scale]" %}
3105   interface(MEMORY_INTER) %{
3106     base($addr);
3107     index($index);
3108     scale($scale);
3109     disp(0x7fffffff); // sxtw
3110   %}
3111 %}
3112 
3113 // Indirect Memory Times Scale Plus Index Register
3114 operand indIndexScaleQ(iRegP addr, iRegX index, immI_4 scale) %{
3115   constraint(ALLOC_IN_RC(ptr_reg));
3116   match(AddP addr (LShiftX index scale));
3117 
3118   op_cost(100);
3119   format %{"[$addr + $index << $scale]" %}
3120   interface(MEMORY_INTER) %{
3121     base($addr);
3122     index($index);
3123     scale($scale);
3124     disp(0x0);
3125   %}
3126 %}
3127 
3128 // Indirect Memory Times Scale Plus 32-bit Index Register
3129 operand indIndexIScaleQ(iRegP addr, iRegI index, immI_4 scale) %{
3130   constraint(ALLOC_IN_RC(ptr_reg));
3131   match(AddP addr (LShiftX (ConvI2L index) scale));
3132 
3133   op_cost(100);
3134   format %{"[$addr + $index.w << $scale]" %}
3135   interface(MEMORY_INTER) %{
3136     base($addr);
3137     index($index);
3138     scale($scale);
3139     disp(0x7fffffff); // sxtw
3140   %}
3141 %}
3142 #else
3143 // Indirect Memory Times Scale Plus Index Register
3144 operand indIndexScale(iRegP addr, iRegX index, immU5 scale) %{
3145   constraint(ALLOC_IN_RC(ptr_reg));
3146   match(AddP addr (LShiftX index scale));
3147 
3148   op_cost(100);
3149   format %{"[$addr + $index << $scale]" %}
3150   interface(MEMORY_INTER) %{
3151     base($addr);
3152     index($index);
3153     scale($scale);
3154     disp(0x0);
3155   %}
3156 %}
3157 #endif
3158 
3159 // Operands for expressing Control Flow
3160 // NOTE:  Label is a predefined operand which should not be redefined in
3161 //        the AD file.  It is generically handled within the ADLC.
3162 
3163 //----------Conditional Branch Operands----------------------------------------
3164 // Comparison Op  - This is the operation of the comparison, and is limited to
3165 //                  the following set of codes:
3166 //                  L (<), LE (<=), G (>), GE (>=), E (==), NE (!=)
3167 //
3168 // Other attributes of the comparison, such as unsignedness, are specified
3169 // by the comparison instruction that sets a condition code flags register.
3170 // That result is represented by a flags operand whose subtype is appropriate
3171 // to the unsignedness (etc.) of the comparison.
3172 //
3173 // Later, the instruction which matches both the Comparison Op (a Bool) and
3174 // the flags (produced by the Cmp) specifies the coding of the comparison op
3175 // by matching a specific subtype of Bool operand below, such as cmpOpU.
3176 
3177 operand cmpOp() %{
3178   match(Bool);
3179 
3180   format %{ "" %}
3181   interface(COND_INTER) %{
3182     equal(0x0);
3183     not_equal(0x1);
3184     less(0xb);
3185     greater_equal(0xa);
3186     less_equal(0xd);
3187     greater(0xc);
3188     overflow(0x0); // unsupported/unimplemented
3189     no_overflow(0x0); // unsupported/unimplemented
3190   %}
3191 %}
3192 
3193 // integer comparison with 0, signed
3194 operand cmpOp0() %{
3195   match(Bool);
3196 
3197   format %{ "" %}
3198   interface(COND_INTER) %{
3199     equal(0x0);
3200     not_equal(0x1);
3201     less(0x4);
3202     greater_equal(0x5);
3203     less_equal(0xd); // unsupported
3204     greater(0xc); // unsupported
3205     overflow(0x0); // unsupported/unimplemented
3206     no_overflow(0x0); // unsupported/unimplemented
3207   %}
3208 %}
3209 
3210 // Comparison Op, unsigned
3211 operand cmpOpU() %{
3212   match(Bool);
3213 
3214   format %{ "u" %}
3215   interface(COND_INTER) %{
3216     equal(0x0);
3217     not_equal(0x1);
3218     less(0x3);
3219     greater_equal(0x2);
3220     less_equal(0x9);
3221     greater(0x8);
3222     overflow(0x0); // unsupported/unimplemented
3223     no_overflow(0x0); // unsupported/unimplemented
3224   %}
3225 %}
3226 
3227 // Comparison Op, pointer (same as unsigned)
3228 operand cmpOpP() %{
3229   match(Bool);
3230 
3231   format %{ "p" %}
3232   interface(COND_INTER) %{
3233     equal(0x0);
3234     not_equal(0x1);
3235     less(0x3);
3236     greater_equal(0x2);
3237     less_equal(0x9);
3238     greater(0x8);
3239     overflow(0x0); // unsupported/unimplemented
3240     no_overflow(0x0); // unsupported/unimplemented
3241   %}
3242 %}
3243 
3244 operand cmpOpL() %{
3245   match(Bool);
3246 
3247   format %{ "L" %}
3248   interface(COND_INTER) %{
3249     equal(0x0);
3250     not_equal(0x1);
3251     less(0xb);
3252     greater_equal(0xa);
3253     less_equal(0xd);
3254     greater(0xc);
3255     overflow(0x0); // unsupported/unimplemented
3256     no_overflow(0x0); // unsupported/unimplemented
3257   %}
3258 %}
3259 
3260 operand cmpOpL_commute() %{
3261   match(Bool);
3262 
3263   format %{ "L" %}
3264   interface(COND_INTER) %{
3265     equal(0x0);
3266     not_equal(0x1);
3267     less(0xc);
3268     greater_equal(0xd);
3269     less_equal(0xa);
3270     greater(0xb);
3271     overflow(0x0); // unsupported/unimplemented
3272     no_overflow(0x0); // unsupported/unimplemented
3273   %}
3274 %}
3275 
3276 operand cmpOpUL() %{
3277   match(Bool);
3278 
3279   format %{ "UL" %}
3280   interface(COND_INTER) %{
3281     equal(0x0);
3282     not_equal(0x1);
3283     less(0x3);
3284     greater_equal(0x2);
3285     less_equal(0x9);
3286     greater(0x8);
3287     overflow(0x0); // unsupported/unimplemented
3288     no_overflow(0x0); // unsupported/unimplemented
3289   %}
3290 %}
3291 
3292 operand cmpOpUL_commute() %{
3293   match(Bool);
3294 
3295   format %{ "UL" %}
3296   interface(COND_INTER) %{
3297     equal(0x0);
3298     not_equal(0x1);
3299     less(0x8);
3300     greater_equal(0x9);
3301     less_equal(0x2);
3302     greater(0x3);
3303     overflow(0x0); // unsupported/unimplemented
3304     no_overflow(0x0); // unsupported/unimplemented
3305   %}
3306 %}
3307 
3308 
3309 //----------OPERAND CLASSES----------------------------------------------------
3310 // Operand Classes are groups of operands that are used to simplify
3311 // instruction definitions by not requiring the AD writer to specify separate
3312 // instructions for every form of operand when the instruction accepts
3313 // multiple operand types with the same basic encoding and format.  The classic
3314 // case of this is memory operands.
3315 #ifdef AARCH64
3316 opclass memoryB(indirect, indIndex, indOffsetU12ScaleB);
3317 opclass memoryS(indirect, indIndex, indIndexScaleS, indIndexIScaleS, indOffsetU12ScaleS);
3318 opclass memoryI(indirect, indIndex, indIndexScaleI, indIndexIScaleI, indOffsetU12ScaleI);
3319 opclass memoryL(indirect, indIndex, indIndexScaleL, indIndexIScaleL, indOffsetU12ScaleL);
3320 opclass memoryP(indirect, indIndex, indIndexScaleL, indIndexIScaleL, indOffsetU12ScaleL);
3321 opclass memoryQ(indirect, indIndex, indIndexScaleQ, indIndexIScaleQ, indOffsetU12ScaleQ);
3322 opclass memoryF(indirect, indIndex, indIndexScaleI, indIndexIScaleI, indOffsetU12ScaleI);
3323 opclass memoryD(indirect, indIndex, indIndexScaleL, indIndexIScaleL, indOffsetU12ScaleL);
3324 
3325 opclass memoryScaledS(indIndexScaleS, indIndexIScaleS);
3326 opclass memoryScaledI(indIndexScaleI, indIndexIScaleI);
3327 opclass memoryScaledL(indIndexScaleL, indIndexIScaleL);
3328 opclass memoryScaledP(indIndexScaleL, indIndexIScaleL);
3329 opclass memoryScaledQ(indIndexScaleQ, indIndexIScaleQ);
3330 opclass memoryScaledF(indIndexScaleI, indIndexIScaleI);
3331 opclass memoryScaledD(indIndexScaleL, indIndexIScaleL);
3332 // when ldrex/strex is used:
3333 opclass memoryex ( indirect );
3334 opclass indIndexMemory( indIndex );
3335 opclass memoryvld ( indirect /* , write back mode not implemented */ );
3336 
3337 #else
3338 
3339 opclass memoryI ( indirect, indOffset12, indIndex, indIndexScale );
3340 opclass memoryP ( indirect, indOffset12, indIndex, indIndexScale );
3341 opclass memoryF ( indirect, indOffsetFP );
3342 opclass memoryF2 ( indirect, indOffsetFPx2 );
3343 opclass memoryD ( indirect, indOffsetFP );
3344 opclass memoryfp( indirect, indOffsetFP );
3345 opclass memoryB ( indirect, indIndex, indOffsetHD );
3346 opclass memoryS ( indirect, indIndex, indOffsetHD );
3347 opclass memoryL ( indirect, indIndex, indOffsetHD );
3348 
3349 opclass memoryScaledI(indIndexScale);
3350 opclass memoryScaledP(indIndexScale);
3351 
3352 // when ldrex/strex is used:
3353 opclass memoryex ( indirect );
3354 opclass indIndexMemory( indIndex );
3355 opclass memorylong ( indirect, indOffset12x2 );
3356 opclass memoryvld ( indirect /* , write back mode not implemented */ );
3357 #endif
3358 
3359 //----------PIPELINE-----------------------------------------------------------
3360 pipeline %{
3361 
3362 //----------ATTRIBUTES---------------------------------------------------------
3363 attributes %{
3364   fixed_size_instructions;           // Fixed size instructions
3365   max_instructions_per_bundle = 4;   // Up to 4 instructions per bundle
3366   instruction_unit_size = 4;         // An instruction is 4 bytes long
3367   instruction_fetch_unit_size = 16;  // The processor fetches one line
3368   instruction_fetch_units = 1;       // of 16 bytes
3369 
3370   // List of nop instructions
3371   nops( Nop_A0, Nop_A1, Nop_MS, Nop_FA, Nop_BR );
3372 %}
3373 
3374 //----------RESOURCES----------------------------------------------------------
3375 // Resources are the functional units available to the machine
3376 resources(A0, A1, MS, BR, FA, FM, IDIV, FDIV, IALU = A0 | A1);
3377 
3378 //----------PIPELINE DESCRIPTION-----------------------------------------------
3379 // Pipeline Description specifies the stages in the machine's pipeline
3380 
3381 pipe_desc(A, P, F, B, I, J, S, R, E, C, M, W, X, T, D);
3382 
3383 //----------PIPELINE CLASSES---------------------------------------------------
3384 // Pipeline Classes describe the stages in which input and output are
3385 // referenced by the hardware pipeline.
3386 
3387 // Integer ALU reg-reg operation
3388 pipe_class ialu_reg_reg(iRegI dst, iRegI src1, iRegI src2) %{
3389     single_instruction;
3390     dst   : E(write);
3391     src1  : R(read);
3392     src2  : R(read);
3393     IALU  : R;
3394 %}
3395 
3396 // Integer ALU reg-reg long operation
3397 pipe_class ialu_reg_reg_2(iRegL dst, iRegL src1, iRegL src2) %{
3398     instruction_count(2);
3399     dst   : E(write);
3400     src1  : R(read);
3401     src2  : R(read);
3402     IALU  : R;
3403     IALU  : R;
3404 %}
3405 
3406 // Integer ALU reg-reg long dependent operation
3407 pipe_class ialu_reg_reg_2_dep(iRegL dst, iRegL src1, iRegL src2, flagsReg cr) %{
3408     instruction_count(1); multiple_bundles;
3409     dst   : E(write);
3410     src1  : R(read);
3411     src2  : R(read);
3412     cr    : E(write);
3413     IALU  : R(2);
3414 %}
3415 
3416 // Integer ALU reg-imm operaion
3417 pipe_class ialu_reg_imm(iRegI dst, iRegI src1) %{
3418     single_instruction;
3419     dst   : E(write);
3420     src1  : R(read);
3421     IALU  : R;
3422 %}
3423 
3424 // Integer ALU reg-reg operation with condition code
3425 pipe_class ialu_cc_reg_reg(iRegI dst, iRegI src1, iRegI src2, flagsReg cr) %{
3426     single_instruction;
3427     dst   : E(write);
3428     cr    : E(write);
3429     src1  : R(read);
3430     src2  : R(read);
3431     IALU  : R;
3432 %}
3433 
3434 // Integer ALU zero-reg operation
3435 pipe_class ialu_zero_reg(iRegI dst, immI0 zero, iRegI src2) %{
3436     single_instruction;
3437     dst   : E(write);
3438     src2  : R(read);
3439     IALU  : R;
3440 %}
3441 
3442 // Integer ALU zero-reg operation with condition code only
3443 pipe_class ialu_cconly_zero_reg(flagsReg cr, iRegI src) %{
3444     single_instruction;
3445     cr    : E(write);
3446     src   : R(read);
3447     IALU  : R;
3448 %}
3449 
3450 // Integer ALU reg-reg operation with condition code only
3451 pipe_class ialu_cconly_reg_reg(flagsReg cr, iRegI src1, iRegI src2) %{
3452     single_instruction;
3453     cr    : E(write);
3454     src1  : R(read);
3455     src2  : R(read);
3456     IALU  : R;
3457 %}
3458 
3459 // Integer ALU reg-imm operation with condition code only
3460 pipe_class ialu_cconly_reg_imm(flagsReg cr, iRegI src1) %{
3461     single_instruction;
3462     cr    : E(write);
3463     src1  : R(read);
3464     IALU  : R;
3465 %}
3466 
3467 // Integer ALU reg-reg-zero operation with condition code only
3468 pipe_class ialu_cconly_reg_reg_zero(flagsReg cr, iRegI src1, iRegI src2, immI0 zero) %{
3469     single_instruction;
3470     cr    : E(write);
3471     src1  : R(read);
3472     src2  : R(read);
3473     IALU  : R;
3474 %}
3475 
3476 // Integer ALU reg-imm-zero operation with condition code only
3477 pipe_class ialu_cconly_reg_imm_zero(flagsReg cr, iRegI src1, immI0 zero) %{
3478     single_instruction;
3479     cr    : E(write);
3480     src1  : R(read);
3481     IALU  : R;
3482 %}
3483 
3484 // Integer ALU reg-reg operation with condition code, src1 modified
3485 pipe_class ialu_cc_rwreg_reg(flagsReg cr, iRegI src1, iRegI src2) %{
3486     single_instruction;
3487     cr    : E(write);
3488     src1  : E(write);
3489     src1  : R(read);
3490     src2  : R(read);
3491     IALU  : R;
3492 %}
3493 
3494 pipe_class cmpL_reg(iRegI dst, iRegL src1, iRegL src2, flagsReg cr ) %{
3495     multiple_bundles;
3496     dst   : E(write)+4;
3497     cr    : E(write);
3498     src1  : R(read);
3499     src2  : R(read);
3500     IALU  : R(3);
3501     BR    : R(2);
3502 %}
3503 
3504 // Integer ALU operation
3505 pipe_class ialu_none(iRegI dst) %{
3506     single_instruction;
3507     dst   : E(write);
3508     IALU  : R;
3509 %}
3510 
3511 // Integer ALU reg operation
3512 pipe_class ialu_reg(iRegI dst, iRegI src) %{
3513     single_instruction; may_have_no_code;
3514     dst   : E(write);
3515     src   : R(read);
3516     IALU  : R;
3517 %}
3518 
3519 // Integer ALU reg conditional operation
3520 // This instruction has a 1 cycle stall, and cannot execute
3521 // in the same cycle as the instruction setting the condition
3522 // code. We kludge this by pretending to read the condition code
3523 // 1 cycle earlier, and by marking the functional units as busy
3524 // for 2 cycles with the result available 1 cycle later than
3525 // is really the case.
3526 pipe_class ialu_reg_flags( iRegI op2_out, iRegI op2_in, iRegI op1, flagsReg cr ) %{
3527     single_instruction;
3528     op2_out : C(write);
3529     op1     : R(read);
3530     cr      : R(read);       // This is really E, with a 1 cycle stall
3531     BR      : R(2);
3532     MS      : R(2);
3533 %}
3534 
3535 // Integer ALU reg operation
3536 pipe_class ialu_move_reg_L_to_I(iRegI dst, iRegL src) %{
3537     single_instruction; may_have_no_code;
3538     dst   : E(write);
3539     src   : R(read);
3540     IALU  : R;
3541 %}
3542 pipe_class ialu_move_reg_I_to_L(iRegL dst, iRegI src) %{
3543     single_instruction; may_have_no_code;
3544     dst   : E(write);
3545     src   : R(read);
3546     IALU  : R;
3547 %}
3548 
3549 // Two integer ALU reg operations
3550 pipe_class ialu_reg_2(iRegL dst, iRegL src) %{
3551     instruction_count(2);
3552     dst   : E(write);
3553     src   : R(read);
3554     A0    : R;
3555     A1    : R;
3556 %}
3557 
3558 // Two integer ALU reg operations
3559 pipe_class ialu_move_reg_L_to_L(iRegL dst, iRegL src) %{
3560     instruction_count(2); may_have_no_code;
3561     dst   : E(write);
3562     src   : R(read);
3563     A0    : R;
3564     A1    : R;
3565 %}
3566 
3567 // Integer ALU imm operation
3568 pipe_class ialu_imm(iRegI dst) %{
3569     single_instruction;
3570     dst   : E(write);
3571     IALU  : R;
3572 %}
3573 
3574 pipe_class ialu_imm_n(iRegI dst) %{
3575     single_instruction;
3576     dst   : E(write);
3577     IALU  : R;
3578 %}
3579 
3580 // Integer ALU reg-reg with carry operation
3581 pipe_class ialu_reg_reg_cy(iRegI dst, iRegI src1, iRegI src2, iRegI cy) %{
3582     single_instruction;
3583     dst   : E(write);
3584     src1  : R(read);
3585     src2  : R(read);
3586     IALU  : R;
3587 %}
3588 
3589 // Integer ALU cc operation
3590 pipe_class ialu_cc(iRegI dst, flagsReg cc) %{
3591     single_instruction;
3592     dst   : E(write);
3593     cc    : R(read);
3594     IALU  : R;
3595 %}
3596 
3597 // Integer ALU cc / second IALU operation
3598 pipe_class ialu_reg_ialu( iRegI dst, iRegI src ) %{
3599     instruction_count(1); multiple_bundles;
3600     dst   : E(write)+1;
3601     src   : R(read);
3602     IALU  : R;
3603 %}
3604 
3605 // Integer ALU cc / second IALU operation
3606 pipe_class ialu_reg_reg_ialu( iRegI dst, iRegI p, iRegI q ) %{
3607     instruction_count(1); multiple_bundles;
3608     dst   : E(write)+1;
3609     p     : R(read);
3610     q     : R(read);
3611     IALU  : R;
3612 %}
3613 
3614 // Integer ALU hi-lo-reg operation
3615 pipe_class ialu_hi_lo_reg(iRegI dst, immI src) %{
3616     instruction_count(1); multiple_bundles;
3617     dst   : E(write)+1;
3618     IALU  : R(2);
3619 %}
3620 
3621 // Long Constant
3622 pipe_class loadConL( iRegL dst, immL src ) %{
3623     instruction_count(2); multiple_bundles;
3624     dst   : E(write)+1;
3625     IALU  : R(2);
3626     IALU  : R(2);
3627 %}
3628 
3629 // Pointer Constant
3630 pipe_class loadConP( iRegP dst, immP src ) %{
3631     instruction_count(0); multiple_bundles;
3632     fixed_latency(6);
3633 %}
3634 
3635 // Polling Address
3636 pipe_class loadConP_poll( iRegP dst, immP_poll src ) %{
3637     dst   : E(write);
3638     IALU  : R;
3639 %}
3640 
3641 // Long Constant small
3642 pipe_class loadConLlo( iRegL dst, immL src ) %{
3643     instruction_count(2);
3644     dst   : E(write);
3645     IALU  : R;
3646     IALU  : R;
3647 %}
3648 
3649 // [PHH] This is wrong for 64-bit.  See LdImmF/D.
3650 pipe_class loadConFD(regF dst, immF src, iRegP tmp) %{
3651     instruction_count(1); multiple_bundles;
3652     src   : R(read);
3653     dst   : M(write)+1;
3654     IALU  : R;
3655     MS    : E;
3656 %}
3657 
3658 // Integer ALU nop operation
3659 pipe_class ialu_nop() %{
3660     single_instruction;
3661     IALU  : R;
3662 %}
3663 
3664 // Integer ALU nop operation
3665 pipe_class ialu_nop_A0() %{
3666     single_instruction;
3667     A0    : R;
3668 %}
3669 
3670 // Integer ALU nop operation
3671 pipe_class ialu_nop_A1() %{
3672     single_instruction;
3673     A1    : R;
3674 %}
3675 
3676 // Integer Multiply reg-reg operation
3677 pipe_class imul_reg_reg(iRegI dst, iRegI src1, iRegI src2) %{
3678     single_instruction;
3679     dst   : E(write);
3680     src1  : R(read);
3681     src2  : R(read);
3682     MS    : R(5);
3683 %}
3684 
3685 pipe_class mulL_reg_reg(iRegL dst, iRegL src1, iRegL src2) %{
3686     single_instruction;
3687     dst   : E(write)+4;
3688     src1  : R(read);
3689     src2  : R(read);
3690     MS    : R(6);
3691 %}
3692 
3693 // Integer Divide reg-reg
3694 pipe_class sdiv_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI temp, flagsReg cr) %{
3695     instruction_count(1); multiple_bundles;
3696     dst   : E(write);
3697     temp  : E(write);
3698     src1  : R(read);
3699     src2  : R(read);
3700     temp  : R(read);
3701     MS    : R(38);
3702 %}
3703 
3704 // Long Divide
3705 pipe_class divL_reg_reg(iRegL dst, iRegL src1, iRegL src2) %{
3706     dst  : E(write)+71;
3707     src1 : R(read);
3708     src2 : R(read)+1;
3709     MS   : R(70);
3710 %}
3711 
3712 // Floating Point Add Float
3713 pipe_class faddF_reg_reg(regF dst, regF src1, regF src2) %{
3714     single_instruction;
3715     dst   : X(write);
3716     src1  : E(read);
3717     src2  : E(read);
3718     FA    : R;
3719 %}
3720 
3721 // Floating Point Add Double
3722 pipe_class faddD_reg_reg(regD dst, regD src1, regD src2) %{
3723     single_instruction;
3724     dst   : X(write);
3725     src1  : E(read);
3726     src2  : E(read);
3727     FA    : R;
3728 %}
3729 
3730 // Floating Point Conditional Move based on integer flags
3731 pipe_class int_conditional_float_move (cmpOp cmp, flagsReg cr, regF dst, regF src) %{
3732     single_instruction;
3733     dst   : X(write);
3734     src   : E(read);
3735     cr    : R(read);
3736     FA    : R(2);
3737     BR    : R(2);
3738 %}
3739 
3740 // Floating Point Conditional Move based on integer flags
3741 pipe_class int_conditional_double_move (cmpOp cmp, flagsReg cr, regD dst, regD src) %{
3742     single_instruction;
3743     dst   : X(write);
3744     src   : E(read);
3745     cr    : R(read);
3746     FA    : R(2);
3747     BR    : R(2);
3748 %}
3749 
3750 // Floating Point Multiply Float
3751 pipe_class fmulF_reg_reg(regF dst, regF src1, regF src2) %{
3752     single_instruction;
3753     dst   : X(write);
3754     src1  : E(read);
3755     src2  : E(read);
3756     FM    : R;
3757 %}
3758 
3759 // Floating Point Multiply Double
3760 pipe_class fmulD_reg_reg(regD dst, regD src1, regD src2) %{
3761     single_instruction;
3762     dst   : X(write);
3763     src1  : E(read);
3764     src2  : E(read);
3765     FM    : R;
3766 %}
3767 
3768 // Floating Point Divide Float
3769 pipe_class fdivF_reg_reg(regF dst, regF src1, regF src2) %{
3770     single_instruction;
3771     dst   : X(write);
3772     src1  : E(read);
3773     src2  : E(read);
3774     FM    : R;
3775     FDIV  : C(14);
3776 %}
3777 
3778 // Floating Point Divide Double
3779 pipe_class fdivD_reg_reg(regD dst, regD src1, regD src2) %{
3780     single_instruction;
3781     dst   : X(write);
3782     src1  : E(read);
3783     src2  : E(read);
3784     FM    : R;
3785     FDIV  : C(17);
3786 %}
3787 
3788 // Floating Point Move/Negate/Abs Float
3789 pipe_class faddF_reg(regF dst, regF src) %{
3790     single_instruction;
3791     dst   : W(write);
3792     src   : E(read);
3793     FA    : R(1);
3794 %}
3795 
3796 // Floating Point Move/Negate/Abs Double
3797 pipe_class faddD_reg(regD dst, regD src) %{
3798     single_instruction;
3799     dst   : W(write);
3800     src   : E(read);
3801     FA    : R;
3802 %}
3803 
3804 // Floating Point Convert F->D
3805 pipe_class fcvtF2D(regD dst, regF src) %{
3806     single_instruction;
3807     dst   : X(write);
3808     src   : E(read);
3809     FA    : R;
3810 %}
3811 
3812 // Floating Point Convert I->D
3813 pipe_class fcvtI2D(regD dst, regF src) %{
3814     single_instruction;
3815     dst   : X(write);
3816     src   : E(read);
3817     FA    : R;
3818 %}
3819 
3820 // Floating Point Convert LHi->D
3821 pipe_class fcvtLHi2D(regD dst, regD src) %{
3822     single_instruction;
3823     dst   : X(write);
3824     src   : E(read);
3825     FA    : R;
3826 %}
3827 
3828 // Floating Point Convert L->D
3829 pipe_class fcvtL2D(regD dst, iRegL src) %{
3830     single_instruction;
3831     dst   : X(write);
3832     src   : E(read);
3833     FA    : R;
3834 %}
3835 
3836 // Floating Point Convert L->F
3837 pipe_class fcvtL2F(regF dst, iRegL src) %{
3838     single_instruction;
3839     dst   : X(write);
3840     src   : E(read);
3841     FA    : R;
3842 %}
3843 
3844 // Floating Point Convert D->F
3845 pipe_class fcvtD2F(regD dst, regF src) %{
3846     single_instruction;
3847     dst   : X(write);
3848     src   : E(read);
3849     FA    : R;
3850 %}
3851 
3852 // Floating Point Convert I->L
3853 pipe_class fcvtI2L(regD dst, regF src) %{
3854     single_instruction;
3855     dst   : X(write);
3856     src   : E(read);
3857     FA    : R;
3858 %}
3859 
3860 // Floating Point Convert D->F
3861 pipe_class fcvtD2I(iRegI dst, regD src, flagsReg cr) %{
3862     instruction_count(1); multiple_bundles;
3863     dst   : X(write)+6;
3864     src   : E(read);
3865     FA    : R;
3866 %}
3867 
3868 // Floating Point Convert D->L
3869 pipe_class fcvtD2L(regD dst, regD src, flagsReg cr) %{
3870     instruction_count(1); multiple_bundles;
3871     dst   : X(write)+6;
3872     src   : E(read);
3873     FA    : R;
3874 %}
3875 
3876 // Floating Point Convert F->I
3877 pipe_class fcvtF2I(regF dst, regF src, flagsReg cr) %{
3878     instruction_count(1); multiple_bundles;
3879     dst   : X(write)+6;
3880     src   : E(read);
3881     FA    : R;
3882 %}
3883 
3884 // Floating Point Convert F->L
3885 pipe_class fcvtF2L(regD dst, regF src, flagsReg cr) %{
3886     instruction_count(1); multiple_bundles;
3887     dst   : X(write)+6;
3888     src   : E(read);
3889     FA    : R;
3890 %}
3891 
3892 // Floating Point Convert I->F
3893 pipe_class fcvtI2F(regF dst, regF src) %{
3894     single_instruction;
3895     dst   : X(write);
3896     src   : E(read);
3897     FA    : R;
3898 %}
3899 
3900 // Floating Point Compare
3901 pipe_class faddF_fcc_reg_reg_zero(flagsRegF cr, regF src1, regF src2, immI0 zero) %{
3902     single_instruction;
3903     cr    : X(write);
3904     src1  : E(read);
3905     src2  : E(read);
3906     FA    : R;
3907 %}
3908 
3909 // Floating Point Compare
3910 pipe_class faddD_fcc_reg_reg_zero(flagsRegF cr, regD src1, regD src2, immI0 zero) %{
3911     single_instruction;
3912     cr    : X(write);
3913     src1  : E(read);
3914     src2  : E(read);
3915     FA    : R;
3916 %}
3917 
3918 // Floating Add Nop
3919 pipe_class fadd_nop() %{
3920     single_instruction;
3921     FA  : R;
3922 %}
3923 
3924 // Integer Store to Memory
3925 pipe_class istore_mem_reg(memoryI mem, iRegI src) %{
3926     single_instruction;
3927     mem   : R(read);
3928     src   : C(read);
3929     MS    : R;
3930 %}
3931 
3932 // Integer Store to Memory
3933 pipe_class istore_mem_spORreg(memoryI mem, sp_ptr_RegP src) %{
3934     single_instruction;
3935     mem   : R(read);
3936     src   : C(read);
3937     MS    : R;
3938 %}
3939 
3940 // Float Store
3941 pipe_class fstoreF_mem_reg(memoryF mem, RegF src) %{
3942     single_instruction;
3943     mem : R(read);
3944     src : C(read);
3945     MS  : R;
3946 %}
3947 
3948 // Float Store
3949 pipe_class fstoreF_mem_zero(memoryF mem, immF0 src) %{
3950     single_instruction;
3951     mem : R(read);
3952     MS  : R;
3953 %}
3954 
3955 // Double Store
3956 pipe_class fstoreD_mem_reg(memoryD mem, RegD src) %{
3957     instruction_count(1);
3958     mem : R(read);
3959     src : C(read);
3960     MS  : R;
3961 %}
3962 
3963 // Double Store
3964 pipe_class fstoreD_mem_zero(memoryD mem, immD0 src) %{
3965     single_instruction;
3966     mem : R(read);
3967     MS  : R;
3968 %}
3969 
3970 // Integer Load (when sign bit propagation not needed)
3971 pipe_class iload_mem(iRegI dst, memoryI mem) %{
3972     single_instruction;
3973     mem : R(read);
3974     dst : C(write);
3975     MS  : R;
3976 %}
3977 
3978 // Integer Load (when sign bit propagation or masking is needed)
3979 pipe_class iload_mask_mem(iRegI dst, memoryI mem) %{
3980     single_instruction;
3981     mem : R(read);
3982     dst : M(write);
3983     MS  : R;
3984 %}
3985 
3986 // Float Load
3987 pipe_class floadF_mem(regF dst, memoryF mem) %{
3988     single_instruction;
3989     mem : R(read);
3990     dst : M(write);
3991     MS  : R;
3992 %}
3993 
3994 // Float Load
3995 pipe_class floadD_mem(regD dst, memoryD mem) %{
3996     instruction_count(1); multiple_bundles; // Again, unaligned argument is only multiple case
3997     mem : R(read);
3998     dst : M(write);
3999     MS  : R;
4000 %}
4001 
4002 // Memory Nop
4003 pipe_class mem_nop() %{
4004     single_instruction;
4005     MS  : R;
4006 %}
4007 
4008 pipe_class sethi(iRegP dst, immI src) %{
4009     single_instruction;
4010     dst  : E(write);
4011     IALU : R;
4012 %}
4013 
4014 pipe_class loadPollP(iRegP poll) %{
4015     single_instruction;
4016     poll : R(read);
4017     MS   : R;
4018 %}
4019 
4020 pipe_class br(Universe br, label labl) %{
4021     single_instruction_with_delay_slot;
4022     BR  : R;
4023 %}
4024 
4025 pipe_class br_cc(Universe br, cmpOp cmp, flagsReg cr, label labl) %{
4026     single_instruction_with_delay_slot;
4027     cr    : E(read);
4028     BR    : R;
4029 %}
4030 
4031 pipe_class br_reg(Universe br, cmpOp cmp, iRegI op1, label labl) %{
4032     single_instruction_with_delay_slot;
4033     op1 : E(read);
4034     BR  : R;
4035     MS  : R;
4036 %}
4037 
4038 pipe_class br_nop() %{
4039     single_instruction;
4040     BR  : R;
4041 %}
4042 
4043 pipe_class simple_call(method meth) %{
4044     instruction_count(2); multiple_bundles; force_serialization;
4045     fixed_latency(100);
4046     BR  : R(1);
4047     MS  : R(1);
4048     A0  : R(1);
4049 %}
4050 
4051 pipe_class compiled_call(method meth) %{
4052     instruction_count(1); multiple_bundles; force_serialization;
4053     fixed_latency(100);
4054     MS  : R(1);
4055 %}
4056 
4057 pipe_class call(method meth) %{
4058     instruction_count(0); multiple_bundles; force_serialization;
4059     fixed_latency(100);
4060 %}
4061 
4062 pipe_class tail_call(Universe ignore, label labl) %{
4063     single_instruction; has_delay_slot;
4064     fixed_latency(100);
4065     BR  : R(1);
4066     MS  : R(1);
4067 %}
4068 
4069 pipe_class ret(Universe ignore) %{
4070     single_instruction; has_delay_slot;
4071     BR  : R(1);
4072     MS  : R(1);
4073 %}
4074 
4075 // The real do-nothing guy
4076 pipe_class empty( ) %{
4077     instruction_count(0);
4078 %}
4079 
4080 pipe_class long_memory_op() %{
4081     instruction_count(0); multiple_bundles; force_serialization;
4082     fixed_latency(25);
4083     MS  : R(1);
4084 %}
4085 
4086 // Check-cast
4087 pipe_class partial_subtype_check_pipe(Universe ignore, iRegP array, iRegP match ) %{
4088     array : R(read);
4089     match  : R(read);
4090     IALU   : R(2);
4091     BR     : R(2);
4092     MS     : R;
4093 %}
4094 
4095 // Convert FPU flags into +1,0,-1
4096 pipe_class floating_cmp( iRegI dst, regF src1, regF src2 ) %{
4097     src1  : E(read);
4098     src2  : E(read);
4099     dst   : E(write);
4100     FA    : R;
4101     MS    : R(2);
4102     BR    : R(2);
4103 %}
4104 
4105 // Compare for p < q, and conditionally add y
4106 pipe_class cadd_cmpltmask( iRegI p, iRegI q, iRegI y ) %{
4107     p     : E(read);
4108     q     : E(read);
4109     y     : E(read);
4110     IALU  : R(3)
4111 %}
4112 
4113 // Perform a compare, then move conditionally in a branch delay slot.
4114 pipe_class min_max( iRegI src2, iRegI srcdst ) %{
4115     src2   : E(read);
4116     srcdst : E(read);
4117     IALU   : R;
4118     BR     : R;
4119 %}
4120 
4121 // Define the class for the Nop node
4122 define %{
4123    MachNop = ialu_nop;
4124 %}
4125 
4126 %}
4127 
4128 //----------INSTRUCTIONS-------------------------------------------------------
4129 
4130 //------------Special Nop instructions for bundling - no match rules-----------
4131 // Nop using the A0 functional unit
4132 instruct Nop_A0() %{
4133   ins_pipe(ialu_nop_A0);
4134 %}
4135 
4136 // Nop using the A1 functional unit
4137 instruct Nop_A1( ) %{
4138   ins_pipe(ialu_nop_A1);
4139 %}
4140 
4141 // Nop using the memory functional unit
4142 instruct Nop_MS( ) %{
4143   ins_pipe(mem_nop);
4144 %}
4145 
4146 // Nop using the floating add functional unit
4147 instruct Nop_FA( ) %{
4148   ins_pipe(fadd_nop);
4149 %}
4150 
4151 // Nop using the branch functional unit
4152 instruct Nop_BR( ) %{
4153   ins_pipe(br_nop);
4154 %}
4155 
4156 //----------Load/Store/Move Instructions---------------------------------------
4157 //----------Load Instructions--------------------------------------------------
4158 // Load Byte (8bit signed)
4159 instruct loadB(iRegI dst, memoryB mem) %{
4160   match(Set dst (LoadB mem));
4161   ins_cost(MEMORY_REF_COST);
4162 
4163   size(4);
4164   format %{ "LDRSB   $dst,$mem\t! byte -> int" %}
4165   ins_encode %{
4166     // High 32 bits are harmlessly set on Aarch64
4167     __ ldrsb($dst$$Register, $mem$$Address);
4168   %}
4169   ins_pipe(iload_mask_mem);
4170 %}
4171 
4172 // Load Byte (8bit signed) into a Long Register
4173 instruct loadB2L(iRegL dst, memoryB mem) %{
4174   match(Set dst (ConvI2L (LoadB mem)));
4175   ins_cost(MEMORY_REF_COST);
4176 
4177 #ifdef AARCH64
4178   size(4);
4179   format %{ "LDRSB $dst,$mem\t! byte -> long"  %}
4180   ins_encode %{
4181     __ ldrsb($dst$$Register, $mem$$Address);
4182   %}
4183 #else
4184   size(8);
4185   format %{ "LDRSB $dst.lo,$mem\t! byte -> long\n\t"
4186             "ASR   $dst.hi,$dst.lo,31" %}
4187   ins_encode %{
4188     __ ldrsb($dst$$Register, $mem$$Address);
4189     __ mov($dst$$Register->successor(), AsmOperand($dst$$Register, asr, 31));
4190   %}
4191 #endif
4192   ins_pipe(iload_mask_mem);
4193 %}
4194 
4195 // Load Unsigned Byte (8bit UNsigned) into an int reg
4196 instruct loadUB(iRegI dst, memoryB mem) %{
4197   match(Set dst (LoadUB mem));
4198   ins_cost(MEMORY_REF_COST);
4199 
4200   size(4);
4201   format %{ "LDRB   $dst,$mem\t! ubyte -> int" %}
4202   ins_encode %{
4203     __ ldrb($dst$$Register, $mem$$Address);
4204   %}
4205   ins_pipe(iload_mem);
4206 %}
4207 
4208 // Load Unsigned Byte (8bit UNsigned) into a Long Register
4209 instruct loadUB2L(iRegL dst, memoryB mem) %{
4210   match(Set dst (ConvI2L (LoadUB mem)));
4211   ins_cost(MEMORY_REF_COST);
4212 
4213 #ifdef AARCH64
4214   size(4);
4215   format %{ "LDRB  $dst,$mem\t! ubyte -> long"  %}
4216   ins_encode %{
4217     __ ldrb($dst$$Register, $mem$$Address);
4218   %}
4219 #else
4220   size(8);
4221   format %{ "LDRB  $dst.lo,$mem\t! ubyte -> long\n\t"
4222             "MOV   $dst.hi,0" %}
4223   ins_encode %{
4224     __ ldrb($dst$$Register, $mem$$Address);
4225     __ mov($dst$$Register->successor(), 0);
4226   %}
4227 #endif
4228   ins_pipe(iload_mem);
4229 %}
4230 
4231 // Load Unsigned Byte (8 bit UNsigned) with immediate mask into Long Register
4232 instruct loadUB2L_limmI(iRegL dst, memoryB mem, limmIlow8 mask) %{
4233   match(Set dst (ConvI2L (AndI (LoadUB mem) mask)));
4234 
4235 #ifdef AARCH64
4236   ins_cost(MEMORY_REF_COST + DEFAULT_COST);
4237   size(8);
4238   format %{ "LDRB  $dst,$mem\t! ubyte -> long\n\t"
4239             "AND  $dst,$dst,$mask" %}
4240   ins_encode %{
4241     __ ldrb($dst$$Register, $mem$$Address);
4242     __ andr($dst$$Register, $dst$$Register, limmI_low($mask$$constant, 8));
4243   %}
4244 #else
4245   ins_cost(MEMORY_REF_COST + 2*DEFAULT_COST);
4246   size(12);
4247   format %{ "LDRB  $dst.lo,$mem\t! ubyte -> long\n\t"
4248             "MOV   $dst.hi,0\n\t"
4249             "AND  $dst.lo,$dst.lo,$mask" %}
4250   ins_encode %{
4251     __ ldrb($dst$$Register, $mem$$Address);
4252     __ mov($dst$$Register->successor(), 0);
4253     __ andr($dst$$Register, $dst$$Register, limmI_low($mask$$constant, 8));
4254   %}
4255 #endif
4256   ins_pipe(iload_mem);
4257 %}
4258 
4259 // Load Short (16bit signed)
4260 #ifdef AARCH64
4261 // XXX This variant shouldn't be necessary if 6217251 is implemented
4262 instruct loadSoff(iRegI dst, memoryScaledS mem, aimmX off, iRegP tmp) %{
4263   match(Set dst (LoadS (AddP mem off)));
4264   ins_cost(MEMORY_REF_COST + DEFAULT_COST); // assume shift/sign-extend is free
4265   effect(TEMP tmp);
4266   size(4 * 2);
4267 
4268   format %{ "LDRSH   $dst,$mem+$off\t! short temp=$tmp" %}
4269   ins_encode %{
4270     Register base = reg_to_register_object($mem$$base);
4271     __ add($tmp$$Register, base, $off$$constant);
4272     Address nmem = Address::make_raw($tmp$$reg, $mem$$index, $mem$$scale, $mem$$disp, relocInfo::none);
4273     __ ldrsh($dst$$Register, nmem);
4274   %}
4275   ins_pipe(iload_mask_mem);
4276 %}
4277 #endif
4278 
4279 instruct loadS(iRegI dst, memoryS mem) %{
4280   match(Set dst (LoadS mem));
4281   ins_cost(MEMORY_REF_COST);
4282 
4283   size(4);
4284   format %{ "LDRSH   $dst,$mem\t! short" %}
4285   ins_encode %{
4286     __ ldrsh($dst$$Register, $mem$$Address);
4287   %}
4288   ins_pipe(iload_mask_mem);
4289 %}
4290 
4291 // Load Short (16 bit signed) to Byte (8 bit signed)
4292 instruct loadS2B(iRegI dst, memoryS mem, immI_24 twentyfour) %{
4293   match(Set dst (RShiftI (LShiftI (LoadS mem) twentyfour) twentyfour));
4294   ins_cost(MEMORY_REF_COST);
4295 
4296   size(4);
4297 
4298   format %{ "LDRSB   $dst,$mem\t! short -> byte" %}
4299   ins_encode %{
4300     // High 32 bits are harmlessly set on Aarch64
4301     __ ldrsb($dst$$Register, $mem$$Address);
4302   %}
4303   ins_pipe(iload_mask_mem);
4304 %}
4305 
4306 // Load Short (16bit signed) into a Long Register
4307 instruct loadS2L(iRegL dst, memoryS mem) %{
4308   match(Set dst (ConvI2L (LoadS mem)));
4309   ins_cost(MEMORY_REF_COST);
4310 
4311 #ifdef AARCH64
4312   size(4);
4313   format %{ "LDRSH $dst,$mem\t! short -> long"  %}
4314   ins_encode %{
4315     __ ldrsh($dst$$Register, $mem$$Address);
4316   %}
4317 #else
4318   size(8);
4319   format %{ "LDRSH $dst.lo,$mem\t! short -> long\n\t"
4320             "ASR   $dst.hi,$dst.lo,31" %}
4321   ins_encode %{
4322     __ ldrsh($dst$$Register, $mem$$Address);
4323     __ mov($dst$$Register->successor(), AsmOperand($dst$$Register, asr, 31));
4324   %}
4325 #endif
4326   ins_pipe(iload_mask_mem);
4327 %}
4328 
4329 // Load Unsigned Short/Char (16bit UNsigned)
4330 
4331 #ifdef AARCH64
4332 // XXX This variant shouldn't be necessary if 6217251 is implemented
4333 instruct loadUSoff(iRegI dst, memoryScaledS mem, aimmX off, iRegP tmp) %{
4334   match(Set dst (LoadUS (AddP mem off)));
4335   ins_cost(MEMORY_REF_COST + DEFAULT_COST); // assume shift/sign-extend is free
4336   effect(TEMP tmp);
4337   size(4 * 2);
4338 
4339   format %{ "LDRH   $dst,$mem+$off\t! ushort/char temp=$tmp" %}
4340   ins_encode %{
4341     Register base = reg_to_register_object($mem$$base);
4342     __ add($tmp$$Register, base, $off$$constant);
4343     Address nmem = Address::make_raw($tmp$$reg, $mem$$index, $mem$$scale, $mem$$disp, relocInfo::none);
4344     __ ldrh($dst$$Register, nmem);
4345   %}
4346   ins_pipe(iload_mem);
4347 %}
4348 #endif
4349 
4350 instruct loadUS(iRegI dst, memoryS mem) %{
4351   match(Set dst (LoadUS mem));
4352   ins_cost(MEMORY_REF_COST);
4353 
4354   size(4);
4355   format %{ "LDRH   $dst,$mem\t! ushort/char" %}
4356   ins_encode %{
4357     __ ldrh($dst$$Register, $mem$$Address);
4358   %}
4359   ins_pipe(iload_mem);
4360 %}
4361 
4362 // Load Unsigned Short/Char (16 bit UNsigned) to Byte (8 bit signed)
4363 instruct loadUS2B(iRegI dst, memoryB mem, immI_24 twentyfour) %{
4364   match(Set dst (RShiftI (LShiftI (LoadUS mem) twentyfour) twentyfour));
4365   ins_cost(MEMORY_REF_COST);
4366 
4367   size(4);
4368   format %{ "LDRSB   $dst,$mem\t! ushort -> byte" %}
4369   ins_encode %{
4370     __ ldrsb($dst$$Register, $mem$$Address);
4371   %}
4372   ins_pipe(iload_mask_mem);
4373 %}
4374 
4375 // Load Unsigned Short/Char (16bit UNsigned) into a Long Register
4376 instruct loadUS2L(iRegL dst, memoryS mem) %{
4377   match(Set dst (ConvI2L (LoadUS mem)));
4378   ins_cost(MEMORY_REF_COST);
4379 
4380 #ifdef AARCH64
4381   size(4);
4382   format %{ "LDRH  $dst,$mem\t! short -> long"  %}
4383   ins_encode %{
4384     __ ldrh($dst$$Register, $mem$$Address);
4385   %}
4386 #else
4387   size(8);
4388   format %{ "LDRH  $dst.lo,$mem\t! short -> long\n\t"
4389             "MOV   $dst.hi, 0" %}
4390   ins_encode %{
4391     __ ldrh($dst$$Register, $mem$$Address);
4392     __ mov($dst$$Register->successor(), 0);
4393   %}
4394 #endif
4395   ins_pipe(iload_mem);
4396 %}
4397 
4398 // Load Unsigned Short/Char (16bit UNsigned) with mask 0xFF into a Long Register
4399 instruct loadUS2L_immI_255(iRegL dst, memoryB mem, immI_255 mask) %{
4400   match(Set dst (ConvI2L (AndI (LoadUS mem) mask)));
4401   ins_cost(MEMORY_REF_COST);
4402 
4403 #ifdef AARCH64
4404   size(4);
4405   format %{ "LDRB  $dst,$mem"  %}
4406   ins_encode %{
4407     __ ldrb($dst$$Register, $mem$$Address);
4408   %}
4409 #else
4410   size(8);
4411   format %{ "LDRB  $dst.lo,$mem\t! \n\t"
4412             "MOV   $dst.hi, 0" %}
4413   ins_encode %{
4414     __ ldrb($dst$$Register, $mem$$Address);
4415     __ mov($dst$$Register->successor(), 0);
4416   %}
4417 #endif
4418   ins_pipe(iload_mem);
4419 %}
4420 
4421 // Load Unsigned Short/Char (16bit UNsigned) with a immediate mask into a Long Register
4422 instruct loadUS2L_limmI(iRegL dst, memoryS mem, limmI mask) %{
4423   match(Set dst (ConvI2L (AndI (LoadUS mem) mask)));
4424 #ifdef AARCH64
4425   ins_cost(MEMORY_REF_COST + 1*DEFAULT_COST);
4426 
4427   size(8);
4428   format %{ "LDRH   $dst,$mem\t! ushort/char & mask -> long\n\t"
4429             "AND    $dst,$dst,$mask" %}
4430   ins_encode %{
4431     __ ldrh($dst$$Register, $mem$$Address);
4432     __ andr($dst$$Register, $dst$$Register, (uintx)$mask$$constant);
4433   %}
4434 #else
4435   ins_cost(MEMORY_REF_COST + 2*DEFAULT_COST);
4436 
4437   size(12);
4438   format %{ "LDRH   $dst,$mem\t! ushort/char & mask -> long\n\t"
4439             "MOV    $dst.hi, 0\n\t"
4440             "AND    $dst,$dst,$mask" %}
4441   ins_encode %{
4442     __ ldrh($dst$$Register, $mem$$Address);
4443     __ mov($dst$$Register->successor(), 0);
4444     __ andr($dst$$Register, $dst$$Register, $mask$$constant);
4445   %}
4446 #endif
4447   ins_pipe(iload_mem);
4448 %}
4449 
4450 // Load Integer
4451 
4452 #ifdef AARCH64
4453 // XXX This variant shouldn't be necessary if 6217251 is implemented
4454 instruct loadIoff(iRegI dst, memoryScaledI mem, aimmX off, iRegP tmp) %{
4455   match(Set dst (LoadI (AddP mem off)));
4456   ins_cost(MEMORY_REF_COST + DEFAULT_COST); // assume shift/sign-extend is free
4457   effect(TEMP tmp);
4458   size(4 * 2);
4459 
4460   format %{ "ldr_s32 $dst,$mem+$off\t! int temp=$tmp" %}
4461   ins_encode %{
4462     Register base = reg_to_register_object($mem$$base);
4463     __ add($tmp$$Register, base, $off$$constant);
4464     Address nmem = Address::make_raw($tmp$$reg, $mem$$index, $mem$$scale, $mem$$disp, relocInfo::none);
4465     __ ldr_s32($dst$$Register, nmem);
4466   %}
4467   ins_pipe(iload_mem);
4468 %}
4469 #endif
4470 
4471 instruct loadI(iRegI dst, memoryI mem) %{
4472   match(Set dst (LoadI mem));
4473   ins_cost(MEMORY_REF_COST);
4474 
4475   size(4);
4476   format %{ "ldr_s32 $dst,$mem\t! int" %}
4477   ins_encode %{
4478     __ ldr_s32($dst$$Register, $mem$$Address);
4479   %}
4480   ins_pipe(iload_mem);
4481 %}
4482 
4483 // Load Integer to Byte (8 bit signed)
4484 instruct loadI2B(iRegI dst, memoryS mem, immI_24 twentyfour) %{
4485   match(Set dst (RShiftI (LShiftI (LoadI mem) twentyfour) twentyfour));
4486   ins_cost(MEMORY_REF_COST);
4487 
4488   size(4);
4489 
4490   format %{ "LDRSB   $dst,$mem\t! int -> byte" %}
4491   ins_encode %{
4492     __ ldrsb($dst$$Register, $mem$$Address);
4493   %}
4494   ins_pipe(iload_mask_mem);
4495 %}
4496 
4497 // Load Integer to Unsigned Byte (8 bit UNsigned)
4498 instruct loadI2UB(iRegI dst, memoryB mem, immI_255 mask) %{
4499   match(Set dst (AndI (LoadI mem) mask));
4500   ins_cost(MEMORY_REF_COST);
4501 
4502   size(4);
4503 
4504   format %{ "LDRB   $dst,$mem\t! int -> ubyte" %}
4505   ins_encode %{
4506     __ ldrb($dst$$Register, $mem$$Address);
4507   %}
4508   ins_pipe(iload_mask_mem);
4509 %}
4510 
4511 // Load Integer to Short (16 bit signed)
4512 instruct loadI2S(iRegI dst, memoryS mem, immI_16 sixteen) %{
4513   match(Set dst (RShiftI (LShiftI (LoadI mem) sixteen) sixteen));
4514   ins_cost(MEMORY_REF_COST);
4515 
4516   size(4);
4517   format %{ "LDRSH   $dst,$mem\t! int -> short" %}
4518   ins_encode %{
4519     __ ldrsh($dst$$Register, $mem$$Address);
4520   %}
4521   ins_pipe(iload_mask_mem);
4522 %}
4523 
4524 // Load Integer to Unsigned Short (16 bit UNsigned)
4525 instruct loadI2US(iRegI dst, memoryS mem, immI_65535 mask) %{
4526   match(Set dst (AndI (LoadI mem) mask));
4527   ins_cost(MEMORY_REF_COST);
4528 
4529   size(4);
4530   format %{ "LDRH   $dst,$mem\t! int -> ushort/char" %}
4531   ins_encode %{
4532     __ ldrh($dst$$Register, $mem$$Address);
4533   %}
4534   ins_pipe(iload_mask_mem);
4535 %}
4536 
4537 // Load Integer into a Long Register
4538 instruct loadI2L(iRegL dst, memoryI mem) %{
4539   match(Set dst (ConvI2L (LoadI mem)));
4540 #ifdef AARCH64
4541   ins_cost(MEMORY_REF_COST);
4542 
4543   size(4);
4544   format %{ "LDRSW $dst.lo,$mem\t! int -> long"  %}
4545   ins_encode %{
4546     __ ldr_s32($dst$$Register, $mem$$Address);
4547   %}
4548 #else
4549   ins_cost(MEMORY_REF_COST);
4550 
4551   size(8);
4552   format %{ "LDR   $dst.lo,$mem\t! int -> long\n\t"
4553             "ASR   $dst.hi,$dst.lo,31\t! int->long" %}
4554   ins_encode %{
4555     __ ldr($dst$$Register, $mem$$Address);
4556     __ mov($dst$$Register->successor(), AsmOperand($dst$$Register, asr, 31));
4557   %}
4558 #endif
4559   ins_pipe(iload_mask_mem);
4560 %}
4561 
4562 // Load Integer with mask 0xFF into a Long Register
4563 instruct loadI2L_immI_255(iRegL dst, memoryB mem, immI_255 mask) %{
4564   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
4565 #ifdef AARCH64
4566   ins_cost(MEMORY_REF_COST);
4567 
4568   size(4);
4569   format %{ "LDRB   $dst.lo,$mem\t! int & 0xFF -> long"  %}
4570   ins_encode %{
4571     __ ldrb($dst$$Register, $mem$$Address);
4572   %}
4573 #else
4574   ins_cost(MEMORY_REF_COST);
4575 
4576   size(8);
4577   format %{ "LDRB   $dst.lo,$mem\t! int & 0xFF -> long\n\t"
4578             "MOV    $dst.hi, 0" %}
4579   ins_encode %{
4580     __ ldrb($dst$$Register, $mem$$Address);
4581     __ mov($dst$$Register->successor(), 0);
4582   %}
4583 #endif
4584   ins_pipe(iload_mem);
4585 %}
4586 
4587 // Load Integer with mask 0xFFFF into a Long Register
4588 instruct loadI2L_immI_65535(iRegL dst, memoryS mem, immI_65535 mask) %{
4589   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
4590   ins_cost(MEMORY_REF_COST);
4591 
4592 #ifdef AARCH64
4593   size(4);
4594   format %{ "LDRH   $dst,$mem\t! int & 0xFFFF -> long" %}
4595   ins_encode %{
4596     __ ldrh($dst$$Register, $mem$$Address);
4597   %}
4598 #else
4599   size(8);
4600   format %{ "LDRH   $dst,$mem\t! int & 0xFFFF -> long\n\t"
4601             "MOV    $dst.hi, 0" %}
4602   ins_encode %{
4603     __ ldrh($dst$$Register, $mem$$Address);
4604     __ mov($dst$$Register->successor(), 0);
4605   %}
4606 #endif
4607   ins_pipe(iload_mask_mem);
4608 %}
4609 
4610 #ifdef AARCH64
4611 // Load Integer with an immediate mask into a Long Register
4612 instruct loadI2L_limmI(iRegL dst, memoryI mem, limmI mask) %{
4613   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
4614   ins_cost(MEMORY_REF_COST + 1*DEFAULT_COST);
4615 
4616   size(8);
4617   format %{ "LDRSW $dst,$mem\t! int -> long\n\t"
4618             "AND   $dst,$dst,$mask" %}
4619 
4620   ins_encode %{
4621     __ ldr_s32($dst$$Register, $mem$$Address);
4622     __ andr($dst$$Register, $dst$$Register, (uintx)$mask$$constant);
4623   %}
4624   ins_pipe(iload_mem);
4625 %}
4626 #else
4627 // Load Integer with a 31-bit immediate mask into a Long Register
4628 instruct loadI2L_limmU31(iRegL dst, memoryI mem, limmU31 mask) %{
4629   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
4630   ins_cost(MEMORY_REF_COST + 2*DEFAULT_COST);
4631 
4632   size(12);
4633   format %{ "LDR   $dst.lo,$mem\t! int -> long\n\t"
4634             "MOV    $dst.hi, 0\n\t"
4635             "AND   $dst,$dst,$mask" %}
4636 
4637   ins_encode %{
4638     __ ldr($dst$$Register, $mem$$Address);
4639     __ mov($dst$$Register->successor(), 0);
4640     __ andr($dst$$Register, $dst$$Register, $mask$$constant);
4641   %}
4642   ins_pipe(iload_mem);
4643 %}
4644 #endif
4645 
4646 #ifdef AARCH64
4647 // Load Integer with mask into a Long Register
4648 // FIXME: use signedRegI mask, remove tmp?
4649 instruct loadI2L_immI(iRegL dst, memoryI mem, immI mask, iRegI tmp) %{
4650   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
4651   effect(TEMP dst, TEMP tmp);
4652 
4653   ins_cost(MEMORY_REF_COST + 3*DEFAULT_COST);
4654   format %{ "LDRSW    $mem,$dst\t! int & 31-bit mask -> long\n\t"
4655             "MOV_SLOW $tmp,$mask\n\t"
4656             "AND      $dst,$tmp,$dst" %}
4657   ins_encode %{
4658     __ ldrsw($dst$$Register, $mem$$Address);
4659     __ mov_slow($tmp$$Register, $mask$$constant);
4660     __ andr($dst$$Register, $dst$$Register, $tmp$$Register);
4661   %}
4662   ins_pipe(iload_mem);
4663 %}
4664 #else
4665 // Load Integer with a 31-bit mask into a Long Register
4666 // FIXME: use iRegI mask, remove tmp?
4667 instruct loadI2L_immU31(iRegL dst, memoryI mem, immU31 mask, iRegI tmp) %{
4668   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
4669   effect(TEMP dst, TEMP tmp);
4670 
4671   ins_cost(MEMORY_REF_COST + 4*DEFAULT_COST);
4672   size(20);
4673   format %{ "LDR      $mem,$dst\t! int & 31-bit mask -> long\n\t"
4674             "MOV      $dst.hi, 0\n\t"
4675             "MOV_SLOW $tmp,$mask\n\t"
4676             "AND      $dst,$tmp,$dst" %}
4677   ins_encode %{
4678     __ ldr($dst$$Register, $mem$$Address);
4679     __ mov($dst$$Register->successor(), 0);
4680     __ mov_slow($tmp$$Register, $mask$$constant);
4681     __ andr($dst$$Register, $dst$$Register, $tmp$$Register);
4682   %}
4683   ins_pipe(iload_mem);
4684 %}
4685 #endif
4686 
4687 // Load Unsigned Integer into a Long Register
4688 instruct loadUI2L(iRegL dst, memoryI mem, immL_32bits mask) %{
4689   match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
4690   ins_cost(MEMORY_REF_COST);
4691 
4692 #ifdef AARCH64
4693 //size(4);
4694   format %{ "LDR_w $dst,$mem\t! uint -> long" %}
4695   ins_encode %{
4696     __ ldr_w($dst$$Register, $mem$$Address);
4697   %}
4698 #else
4699   size(8);
4700   format %{ "LDR   $dst.lo,$mem\t! uint -> long\n\t"
4701             "MOV   $dst.hi,0" %}
4702   ins_encode %{
4703     __ ldr($dst$$Register, $mem$$Address);
4704     __ mov($dst$$Register->successor(), 0);
4705   %}
4706 #endif
4707   ins_pipe(iload_mem);
4708 %}
4709 
4710 // Load Long
4711 
4712 #ifdef AARCH64
4713 // XXX This variant shouldn't be necessary if 6217251 is implemented
4714 instruct loadLoff(iRegLd dst, memoryScaledL mem, aimmX off, iRegP tmp) %{
4715   match(Set dst (LoadL (AddP mem off)));
4716   ins_cost(MEMORY_REF_COST + DEFAULT_COST); // assume shift/sign-extend is free
4717   effect(TEMP tmp);
4718   size(4 * 2);
4719 
4720   format %{ "LDR    $dst,$mem+$off\t! long temp=$tmp" %}
4721   ins_encode %{
4722     Register base = reg_to_register_object($mem$$base);
4723     __ add($tmp$$Register, base, $off$$constant);
4724     Address nmem = Address::make_raw($tmp$$reg, $mem$$index, $mem$$scale, $mem$$disp, relocInfo::none);
4725     __ ldr($dst$$Register, nmem);
4726   %}
4727   ins_pipe(iload_mem);
4728 %}
4729 #endif
4730 
4731 instruct loadL(iRegLd dst, memoryL mem ) %{
4732 #ifdef AARCH64
4733   // already atomic for Aarch64
4734 #else
4735   predicate(!((LoadLNode*)n)->require_atomic_access());
4736 #endif
4737   match(Set dst (LoadL mem));
4738   effect(TEMP dst);
4739   ins_cost(MEMORY_REF_COST);
4740 
4741   size(4);
4742   format %{ "ldr_64  $dst,$mem\t! long" %}
4743   ins_encode %{
4744     __ ldr_64($dst$$Register, $mem$$Address);
4745   %}
4746   ins_pipe(iload_mem);
4747 %}
4748 
4749 #ifndef AARCH64
4750 instruct loadL_2instr(iRegL dst, memorylong mem ) %{
4751   predicate(!((LoadLNode*)n)->require_atomic_access());
4752   match(Set dst (LoadL mem));
4753   ins_cost(MEMORY_REF_COST + DEFAULT_COST);
4754 
4755   size(8);
4756   format %{ "LDR    $dst.lo,$mem \t! long order of instrs reversed if $dst.lo == base($mem)\n\t"
4757             "LDR    $dst.hi,$mem+4 or $mem" %}
4758   ins_encode %{
4759     Address Amemlo = Address::make_raw($mem$$base, $mem$$index, $mem$$scale, $mem$$disp, relocInfo::none);
4760     Address Amemhi = Address::make_raw($mem$$base, $mem$$index, $mem$$scale, $mem$$disp + 4, relocInfo::none);
4761 
4762     if ($dst$$Register == reg_to_register_object($mem$$base)) {
4763       __ ldr($dst$$Register->successor(), Amemhi);
4764       __ ldr($dst$$Register, Amemlo);
4765     } else {
4766       __ ldr($dst$$Register, Amemlo);
4767       __ ldr($dst$$Register->successor(), Amemhi);
4768     }
4769   %}
4770   ins_pipe(iload_mem);
4771 %}
4772 
4773 instruct loadL_volatile(iRegL dst, indirect mem ) %{
4774   predicate(((LoadLNode*)n)->require_atomic_access());
4775   match(Set dst (LoadL mem));
4776   ins_cost(MEMORY_REF_COST);
4777 
4778   size(4);
4779   format %{ "LDMIA    $dst,$mem\t! long" %}
4780   ins_encode %{
4781     // FIXME: why is ldmia considered atomic?  Should be ldrexd
4782     RegisterSet set($dst$$Register);
4783     set = set | reg_to_register_object($dst$$reg + 1);
4784     __ ldmia(reg_to_register_object($mem$$base), set);
4785   %}
4786   ins_pipe(iload_mem);
4787 %}
4788 
4789 instruct loadL_volatile_fp(iRegL dst, memoryD mem ) %{
4790   predicate(((LoadLNode*)n)->require_atomic_access());
4791   match(Set dst (LoadL mem));
4792   ins_cost(MEMORY_REF_COST);
4793 
4794   size(8);
4795   format %{ "FLDD      S14, $mem"
4796             "FMRRD    $dst, S14\t! long \n't" %}
4797   ins_encode %{
4798     __ fldd(S14, $mem$$Address);
4799     __ fmrrd($dst$$Register, $dst$$Register->successor(), S14);
4800   %}
4801   ins_pipe(iload_mem);
4802 %}
4803 
4804 instruct loadL_unaligned(iRegL dst, memorylong mem ) %{
4805   match(Set dst (LoadL_unaligned mem));
4806   ins_cost(MEMORY_REF_COST);
4807 
4808   size(8);
4809   format %{ "LDR    $dst.lo,$mem\t! long order of instrs reversed if $dst.lo == base($mem)\n\t"
4810             "LDR    $dst.hi,$mem+4" %}
4811   ins_encode %{
4812     Address Amemlo = Address::make_raw($mem$$base, $mem$$index, $mem$$scale, $mem$$disp, relocInfo::none);
4813     Address Amemhi = Address::make_raw($mem$$base, $mem$$index, $mem$$scale, $mem$$disp + 4, relocInfo::none);
4814 
4815     if ($dst$$Register == reg_to_register_object($mem$$base)) {
4816       __ ldr($dst$$Register->successor(), Amemhi);
4817       __ ldr($dst$$Register, Amemlo);
4818     } else {
4819       __ ldr($dst$$Register, Amemlo);
4820       __ ldr($dst$$Register->successor(), Amemhi);
4821     }
4822   %}
4823   ins_pipe(iload_mem);
4824 %}
4825 #endif // !AARCH64
4826 
4827 // Load Range
4828 instruct loadRange(iRegI dst, memoryI mem) %{
4829   match(Set dst (LoadRange mem));
4830   ins_cost(MEMORY_REF_COST);
4831 
4832   size(4);
4833   format %{ "LDR_u32 $dst,$mem\t! range" %}
4834   ins_encode %{
4835     __ ldr_u32($dst$$Register, $mem$$Address);
4836   %}
4837   ins_pipe(iload_mem);
4838 %}
4839 
4840 // Load Pointer
4841 
4842 #ifdef AARCH64
4843 // XXX This variant shouldn't be necessary if 6217251 is implemented
4844 instruct loadPoff(iRegP dst, memoryScaledP mem, aimmX off, iRegP tmp) %{
4845   match(Set dst (LoadP (AddP mem off)));
4846   ins_cost(MEMORY_REF_COST + DEFAULT_COST); // assume shift/sign-extend is free
4847   effect(TEMP tmp);
4848   size(4 * 2);
4849 
4850   format %{ "LDR    $dst,$mem+$off\t! ptr temp=$tmp" %}
4851   ins_encode %{
4852     Register base = reg_to_register_object($mem$$base);
4853     __ add($tmp$$Register, base, $off$$constant);
4854     Address nmem = Address::make_raw($tmp$$reg, $mem$$index, $mem$$scale, $mem$$disp, relocInfo::none);
4855     __ ldr($dst$$Register, nmem);
4856   %}
4857   ins_pipe(iload_mem);
4858 %}
4859 #endif
4860 
4861 instruct loadP(iRegP dst, memoryP mem) %{
4862   match(Set dst (LoadP mem));
4863   ins_cost(MEMORY_REF_COST);
4864   size(4);
4865 
4866   format %{ "LDR   $dst,$mem\t! ptr" %}
4867   ins_encode %{
4868     __ ldr($dst$$Register, $mem$$Address);
4869   %}
4870   ins_pipe(iload_mem);
4871 %}
4872 
4873 #ifdef XXX
4874 // FIXME XXXX
4875 //instruct loadSP(iRegP dst, memoryP mem) %{
4876 instruct loadSP(SPRegP dst, memoryP mem, iRegP tmp) %{
4877   match(Set dst (LoadP mem));
4878   effect(TEMP tmp);
4879   ins_cost(MEMORY_REF_COST+1);
4880   size(8);
4881 
4882   format %{ "LDR   $tmp,$mem\t! ptr\n\t"
4883             "MOV   $dst,$tmp\t! ptr" %}
4884   ins_encode %{
4885     __ ldr($tmp$$Register, $mem$$Address);
4886     __ mov($dst$$Register, $tmp$$Register);
4887   %}
4888   ins_pipe(iload_mem);
4889 %}
4890 #endif
4891 
4892 #ifdef _LP64
4893 // Load Compressed Pointer
4894 
4895 // XXX This variant shouldn't be necessary if 6217251 is implemented
4896 instruct loadNoff(iRegN dst, memoryScaledI mem, aimmX off, iRegP tmp) %{
4897   match(Set dst (LoadN (AddP mem off)));
4898   ins_cost(MEMORY_REF_COST + DEFAULT_COST); // assume shift/sign-extend is free
4899   effect(TEMP tmp);
4900   size(4 * 2);
4901 
4902   format %{ "ldr_u32 $dst,$mem+$off\t! compressed ptr temp=$tmp" %}
4903   ins_encode %{
4904     Register base = reg_to_register_object($mem$$base);
4905     __ add($tmp$$Register, base, $off$$constant);
4906     Address nmem = Address::make_raw($tmp$$reg, $mem$$index, $mem$$scale, $mem$$disp, relocInfo::none);
4907     __ ldr_u32($dst$$Register, nmem);
4908   %}
4909   ins_pipe(iload_mem);
4910 %}
4911 
4912 instruct loadN(iRegN dst, memoryI mem) %{
4913   match(Set dst (LoadN mem));
4914   ins_cost(MEMORY_REF_COST);
4915   size(4);
4916 
4917   format %{ "ldr_u32 $dst,$mem\t! compressed ptr" %}
4918   ins_encode %{
4919     __ ldr_u32($dst$$Register, $mem$$Address);
4920   %}
4921   ins_pipe(iload_mem);
4922 %}
4923 #endif
4924 
4925 // Load Klass Pointer
4926 instruct loadKlass(iRegP dst, memoryI mem) %{
4927   match(Set dst (LoadKlass mem));
4928   ins_cost(MEMORY_REF_COST);
4929   size(4);
4930 
4931   format %{ "LDR   $dst,$mem\t! klass ptr" %}
4932   ins_encode %{
4933     __ ldr($dst$$Register, $mem$$Address);
4934   %}
4935   ins_pipe(iload_mem);
4936 %}
4937 
4938 #ifdef _LP64
4939 // Load narrow Klass Pointer
4940 instruct loadNKlass(iRegN dst, memoryI mem) %{
4941   match(Set dst (LoadNKlass mem));
4942   ins_cost(MEMORY_REF_COST);
4943   size(4);
4944 
4945   format %{ "ldr_u32 $dst,$mem\t! compressed klass ptr" %}
4946   ins_encode %{
4947     __ ldr_u32($dst$$Register, $mem$$Address);
4948   %}
4949   ins_pipe(iload_mem);
4950 %}
4951 #endif
4952 
4953 #ifdef AARCH64
4954 // XXX This variant shouldn't be necessary if 6217251 is implemented
4955 instruct loadDoff(regD dst, memoryScaledD mem, aimmX off, iRegP tmp) %{
4956   match(Set dst (LoadD (AddP mem off)));
4957   ins_cost(MEMORY_REF_COST + DEFAULT_COST); // assume shift/sign-extend is free
4958   effect(TEMP tmp);
4959   size(4 * 2);
4960 
4961   format %{ "ldr    $dst,$mem+$off\t! double temp=$tmp" %}
4962   ins_encode %{
4963     Register base = reg_to_register_object($mem$$base);
4964     __ add($tmp$$Register, base, $off$$constant);
4965     Address nmem = Address::make_raw($tmp$$reg, $mem$$index, $mem$$scale, $mem$$disp, relocInfo::none);
4966     __ ldr_d($dst$$FloatRegister, nmem);
4967   %}
4968   ins_pipe(floadD_mem);
4969 %}
4970 #endif
4971 
4972 instruct loadD(regD dst, memoryD mem) %{
4973   match(Set dst (LoadD mem));
4974   ins_cost(MEMORY_REF_COST);
4975 
4976   size(4);
4977   // FIXME: needs to be atomic, but  ARMv7 A.R.M. guarantees
4978   // only LDREXD and STREXD are 64-bit single-copy atomic
4979   format %{ "FLDD   $dst,$mem" %}
4980   ins_encode %{
4981     __ ldr_double($dst$$FloatRegister, $mem$$Address);
4982   %}
4983   ins_pipe(floadD_mem);
4984 %}
4985 
4986 #ifndef AARCH64
4987 // Load Double - UNaligned
4988 instruct loadD_unaligned(regD_low dst, memoryF2 mem ) %{
4989   match(Set dst (LoadD_unaligned mem));
4990   ins_cost(MEMORY_REF_COST*2+DEFAULT_COST);
4991   size(8);
4992   format %{ "FLDS    $dst.lo,$mem\t! misaligned double\n"
4993           "\tFLDS    $dst.hi,$mem+4\t!" %}
4994   ins_encode %{
4995     Address Amemlo = Address::make_raw($mem$$base, $mem$$index, $mem$$scale, $mem$$disp, relocInfo::none);
4996     Address Amemhi = Address::make_raw($mem$$base, $mem$$index, $mem$$scale, $mem$$disp + 4, relocInfo::none);
4997       __ flds($dst$$FloatRegister, Amemlo);
4998       __ flds($dst$$FloatRegister->successor(), Amemhi);
4999   %}
5000   ins_pipe(iload_mem);
5001 %}
5002 #endif
5003 
5004 #ifdef AARCH64
5005 // XXX This variant shouldn't be necessary if 6217251 is implemented
5006 instruct loadFoff(regF dst, memoryScaledF mem, aimmX off, iRegP tmp) %{
5007   match(Set dst (LoadF (AddP mem off)));
5008   ins_cost(MEMORY_REF_COST + DEFAULT_COST); // assume shift/sign-extend is free
5009   effect(TEMP tmp);
5010   size(4 * 2);
5011 
5012   format %{ "ldr    $dst,$mem+$off\t! float temp=$tmp" %}
5013   ins_encode %{
5014     Register base = reg_to_register_object($mem$$base);
5015     __ add($tmp$$Register, base, $off$$constant);
5016     Address nmem = Address::make_raw($tmp$$reg, $mem$$index, $mem$$scale, $mem$$disp, relocInfo::none);
5017     __ ldr_s($dst$$FloatRegister, nmem);
5018   %}
5019   ins_pipe(floadF_mem);
5020 %}
5021 #endif
5022 
5023 instruct loadF(regF dst, memoryF mem) %{
5024   match(Set dst (LoadF mem));
5025 
5026   ins_cost(MEMORY_REF_COST);
5027   size(4);
5028   format %{ "FLDS    $dst,$mem" %}
5029   ins_encode %{
5030     __ ldr_float($dst$$FloatRegister, $mem$$Address);
5031   %}
5032   ins_pipe(floadF_mem);
5033 %}
5034 
5035 #ifdef AARCH64
5036 instruct load_limmI(iRegI dst, limmI src) %{
5037   match(Set dst src);
5038   ins_cost(DEFAULT_COST + 1); // + 1 because MOV is preferred
5039   format %{ "ORR_w  $dst, ZR, $src\t! int"  %}
5040   ins_encode %{
5041     __ orr_w($dst$$Register, ZR, (uintx)$src$$constant);
5042   %}
5043   ins_pipe(ialu_imm);
5044 %}
5045 #endif
5046 
5047 // // Load Constant
5048 instruct loadConI( iRegI dst, immI src ) %{
5049   match(Set dst src);
5050   ins_cost(DEFAULT_COST * 3/2);
5051   format %{ "MOV_SLOW    $dst, $src" %}
5052   ins_encode %{
5053     __ mov_slow($dst$$Register, $src$$constant);
5054   %}
5055   ins_pipe(ialu_hi_lo_reg);
5056 %}
5057 
5058 instruct loadConIMov( iRegI dst, immIMov src ) %{
5059   match(Set dst src);
5060   size(4);
5061   format %{ "MOV    $dst, $src" %}
5062   ins_encode %{
5063     __ mov($dst$$Register, $src$$constant);
5064   %}
5065   ins_pipe(ialu_imm);
5066 %}
5067 
5068 #ifndef AARCH64
5069 instruct loadConIMovn( iRegI dst, immIRotn src ) %{
5070   match(Set dst src);
5071   size(4);
5072   format %{ "MVN    $dst, ~$src" %}
5073   ins_encode %{
5074     __ mvn($dst$$Register, ~$src$$constant);
5075   %}
5076   ins_pipe(ialu_imm_n);
5077 %}
5078 #endif
5079 
5080 instruct loadConI16( iRegI dst, immI16 src ) %{
5081   match(Set dst src);
5082   size(4);
5083 #ifdef AARCH64
5084   format %{ "MOVZ_w  $dst, $src" %}
5085 #else
5086   format %{ "MOVW    $dst, $src" %}
5087 #endif
5088   ins_encode %{
5089 #ifdef AARCH64
5090     __ mov_w($dst$$Register, $src$$constant);
5091 #else
5092     __ movw($dst$$Register, $src$$constant);
5093 #endif
5094   %}
5095   ins_pipe(ialu_imm_n);
5096 %}
5097 
5098 instruct loadConP(iRegP dst, immP src) %{
5099   match(Set dst src);
5100   ins_cost(DEFAULT_COST * 3/2);
5101   format %{ "MOV_SLOW    $dst,$src\t!ptr" %}
5102   ins_encode %{
5103     relocInfo::relocType constant_reloc = _opnds[1]->constant_reloc();
5104     intptr_t val = $src$$constant;
5105     if (constant_reloc == relocInfo::oop_type) {
5106       __ mov_oop($dst$$Register, (jobject)val);
5107     } else if (constant_reloc == relocInfo::metadata_type) {
5108       __ mov_metadata($dst$$Register, (Metadata*)val);
5109     } else {
5110       __ mov_slow($dst$$Register, val);
5111     }
5112   %}
5113   ins_pipe(loadConP);
5114 %}
5115 
5116 
5117 instruct loadConP_poll(iRegP dst, immP_poll src) %{
5118   match(Set dst src);
5119   ins_cost(DEFAULT_COST);
5120   format %{ "MOV_SLOW    $dst,$src\t!ptr" %}
5121   ins_encode %{
5122       __ mov_slow($dst$$Register, $src$$constant);
5123   %}
5124   ins_pipe(loadConP_poll);
5125 %}
5126 
5127 #ifdef AARCH64
5128 instruct loadConP0(iRegP dst, immP0 src) %{
5129   match(Set dst src);
5130   ins_cost(DEFAULT_COST);
5131   format %{ "MOV    $dst,ZR\t!ptr" %}
5132   ins_encode %{
5133     __ mov($dst$$Register, ZR);
5134   %}
5135   ins_pipe(ialu_none);
5136 %}
5137 
5138 instruct loadConN(iRegN dst, immN src) %{
5139   match(Set dst src);
5140   ins_cost(DEFAULT_COST * 3/2);
5141   format %{ "SET    $dst,$src\t! compressed ptr" %}
5142   ins_encode %{
5143     Register dst = $dst$$Register;
5144     // FIXME: use $constanttablebase?
5145     __ set_narrow_oop(dst, (jobject)$src$$constant);
5146   %}
5147   ins_pipe(ialu_hi_lo_reg);
5148 %}
5149 
5150 instruct loadConN0(iRegN dst, immN0 src) %{
5151   match(Set dst src);
5152   ins_cost(DEFAULT_COST);
5153   format %{ "MOV    $dst,ZR\t! compressed ptr" %}
5154   ins_encode %{
5155     __ mov($dst$$Register, ZR);
5156   %}
5157   ins_pipe(ialu_none);
5158 %}
5159 
5160 instruct loadConNKlass(iRegN dst, immNKlass src) %{
5161   match(Set dst src);
5162   ins_cost(DEFAULT_COST * 3/2);
5163   format %{ "SET    $dst,$src\t! compressed klass ptr" %}
5164   ins_encode %{
5165     Register dst = $dst$$Register;
5166     // FIXME: use $constanttablebase?
5167     __ set_narrow_klass(dst, (Klass*)$src$$constant);
5168   %}
5169   ins_pipe(ialu_hi_lo_reg);
5170 %}
5171 
5172 instruct load_limmL(iRegL dst, limmL src) %{
5173   match(Set dst src);
5174   ins_cost(DEFAULT_COST);
5175   format %{ "ORR    $dst, ZR, $src\t! long"  %}
5176   ins_encode %{
5177     __ orr($dst$$Register, ZR, (uintx)$src$$constant);
5178   %}
5179   ins_pipe(loadConL);
5180 %}
5181 instruct load_immLMov(iRegL dst, immLMov src) %{
5182   match(Set dst src);
5183   ins_cost(DEFAULT_COST);
5184   format %{ "MOV    $dst, $src\t! long"  %}
5185   ins_encode %{
5186     __ mov($dst$$Register, $src$$constant);
5187   %}
5188   ins_pipe(loadConL);
5189 %}
5190 instruct loadConL(iRegL dst, immL src) %{
5191   match(Set dst src);
5192   ins_cost(DEFAULT_COST * 4); // worst case
5193   format %{ "mov_slow   $dst, $src\t! long"  %}
5194   ins_encode %{
5195     // FIXME: use $constanttablebase?
5196     __ mov_slow($dst$$Register, $src$$constant);
5197   %}
5198   ins_pipe(loadConL);
5199 %}
5200 #else
5201 instruct loadConL(iRegL dst, immL src) %{
5202   match(Set dst src);
5203   ins_cost(DEFAULT_COST * 4);
5204   format %{ "MOV_SLOW   $dst.lo, $src & 0x0FFFFFFFFL \t! long\n\t"
5205             "MOV_SLOW   $dst.hi, $src >> 32" %}
5206   ins_encode %{
5207     __ mov_slow(reg_to_register_object($dst$$reg), $src$$constant & 0x0FFFFFFFFL);
5208     __ mov_slow(reg_to_register_object($dst$$reg + 1), ((julong)($src$$constant)) >> 32);
5209   %}
5210   ins_pipe(loadConL);
5211 %}
5212 
5213 instruct loadConL16( iRegL dst, immL16 src ) %{
5214   match(Set dst src);
5215   ins_cost(DEFAULT_COST * 2);
5216 
5217   size(8);
5218   format %{ "MOVW    $dst.lo, $src \n\t"
5219             "MOVW    $dst.hi, 0 \n\t" %}
5220   ins_encode %{
5221     __ movw($dst$$Register, $src$$constant);
5222     __ movw($dst$$Register->successor(), 0);
5223   %}
5224   ins_pipe(ialu_imm);
5225 %}
5226 #endif
5227 
5228 instruct loadConF_imm8(regF dst, imm8F src) %{
5229   match(Set dst src);
5230   ins_cost(DEFAULT_COST);
5231   size(4);
5232 
5233   format %{ "FCONSTS      $dst, $src"%}
5234 
5235   ins_encode %{
5236     __ fconsts($dst$$FloatRegister, Assembler::float_num($src$$constant).imm8());
5237   %}
5238   ins_pipe(loadConFD); // FIXME
5239 %}
5240 
5241 #ifdef AARCH64
5242 instruct loadIConF(iRegI dst, immF src) %{
5243   match(Set dst src);
5244   ins_cost(DEFAULT_COST * 2);
5245 
5246   format %{ "MOV_SLOW  $dst, $src\t! loadIConF"  %}
5247 
5248   ins_encode %{
5249     // FIXME revisit once 6961697 is in
5250     union {
5251       jfloat f;
5252       int i;
5253     } v;
5254     v.f = $src$$constant;
5255     __ mov_slow($dst$$Register, v.i);
5256   %}
5257   ins_pipe(ialu_imm);
5258 %}
5259 #endif
5260 
5261 instruct loadConF(regF dst, immF src, iRegI tmp) %{
5262   match(Set dst src);
5263   ins_cost(DEFAULT_COST * 2);
5264   effect(TEMP tmp);
5265   size(3*4);
5266 
5267   format %{ "MOV_SLOW  $tmp, $src\n\t"
5268             "FMSR      $dst, $tmp"%}
5269 
5270   ins_encode %{
5271     // FIXME revisit once 6961697 is in
5272     union {
5273       jfloat f;
5274       int i;
5275     } v;
5276     v.f = $src$$constant;
5277     __ mov_slow($tmp$$Register, v.i);
5278     __ fmsr($dst$$FloatRegister, $tmp$$Register);
5279   %}
5280   ins_pipe(loadConFD); // FIXME
5281 %}
5282 
5283 instruct loadConD_imm8(regD dst, imm8D src) %{
5284   match(Set dst src);
5285   ins_cost(DEFAULT_COST);
5286   size(4);
5287 
5288   format %{ "FCONSTD      $dst, $src"%}
5289 
5290   ins_encode %{
5291     __ fconstd($dst$$FloatRegister, Assembler::double_num($src$$constant).imm8());
5292   %}
5293   ins_pipe(loadConFD); // FIXME
5294 %}
5295 
5296 instruct loadConD(regD dst, immD src, iRegP tmp) %{
5297   match(Set dst src);
5298   effect(TEMP tmp);
5299   ins_cost(MEMORY_REF_COST);
5300   format %{ "FLDD  $dst, [$constanttablebase + $constantoffset]\t! load from constant table: double=$src" %}
5301 
5302   ins_encode %{
5303     Register r = $constanttablebase;
5304     int offset  = $constantoffset($src);
5305     if (!is_memoryD(offset)) {                // can't use a predicate
5306                                               // in load constant instructs
5307       __ add_slow($tmp$$Register, r, offset);
5308       r = $tmp$$Register;
5309       offset = 0;
5310     }
5311     __ ldr_double($dst$$FloatRegister, Address(r, offset));
5312   %}
5313   ins_pipe(loadConFD);
5314 %}
5315 
5316 // Prefetch instructions.
5317 // Must be safe to execute with invalid address (cannot fault).
5318 
5319 instruct prefetchAlloc( memoryP mem ) %{

5320   match( PrefetchAllocation mem );
5321   ins_cost(MEMORY_REF_COST);
5322   size(4);
5323 
5324   format %{ "PLDW $mem\t! Prefetch allocation" %}
5325   ins_encode %{
5326 #ifdef AARCH64
5327     __ prfm(pstl1keep, $mem$$Address);
5328 #else
5329     __ pldw($mem$$Address);

















5330 #endif
5331   %}
5332   ins_pipe(iload_mem);
5333 %}
5334 
5335 //----------Store Instructions-------------------------------------------------
5336 // Store Byte
5337 instruct storeB(memoryB mem, store_RegI src) %{
5338   match(Set mem (StoreB mem src));
5339   ins_cost(MEMORY_REF_COST);
5340 
5341   size(4);
5342   format %{ "STRB    $src,$mem\t! byte" %}
5343   ins_encode %{
5344     __ strb($src$$Register, $mem$$Address);
5345   %}
5346   ins_pipe(istore_mem_reg);
5347 %}
5348 
5349 instruct storeCM(memoryB mem, store_RegI src) %{
5350   match(Set mem (StoreCM mem src));
5351   ins_cost(MEMORY_REF_COST);
5352 
5353   size(4);
5354   format %{ "STRB    $src,$mem\t! CMS card-mark byte" %}
5355   ins_encode %{
5356     __ strb($src$$Register, $mem$$Address);
5357   %}
5358   ins_pipe(istore_mem_reg);
5359 %}
5360 
5361 // Store Char/Short
5362 
5363 #ifdef AARCH64
5364 // XXX This variant shouldn't be necessary if 6217251 is implemented
5365 instruct storeCoff(store_RegI src, memoryScaledS mem, aimmX off, iRegP tmp) %{
5366   match(Set mem (StoreC (AddP mem off) src));
5367   ins_cost(MEMORY_REF_COST + DEFAULT_COST); // assume shift/sign-extend is free
5368   effect(TEMP tmp);
5369   size(4 * 2);
5370 
5371   format %{ "STRH    $src,$mem+$off\t! short temp=$tmp" %}
5372   ins_encode %{
5373     Register base = reg_to_register_object($mem$$base);
5374     __ add($tmp$$Register, base, $off$$constant);
5375     Address nmem = Address::make_raw($tmp$$reg, $mem$$index, $mem$$scale, $mem$$disp, relocInfo::none);
5376     __ strh($src$$Register, nmem);
5377   %}
5378   ins_pipe(istore_mem_reg);
5379 %}
5380 #endif
5381 
5382 instruct storeC(memoryS mem, store_RegI src) %{
5383   match(Set mem (StoreC mem src));
5384   ins_cost(MEMORY_REF_COST);
5385 
5386   size(4);
5387   format %{ "STRH    $src,$mem\t! short" %}
5388   ins_encode %{
5389     __ strh($src$$Register, $mem$$Address);
5390   %}
5391   ins_pipe(istore_mem_reg);
5392 %}
5393 
5394 // Store Integer
5395 
5396 #ifdef AARCH64
5397 // XXX This variant shouldn't be necessary if 6217251 is implemented
5398 instruct storeIoff(store_RegI src, memoryScaledI mem, aimmX off, iRegP tmp) %{
5399   match(Set mem (StoreI (AddP mem off) src));
5400   ins_cost(MEMORY_REF_COST + DEFAULT_COST); // assume shift/sign-extend is free
5401   effect(TEMP tmp);
5402   size(4 * 2);
5403 
5404   format %{ "str_32 $src,$mem+$off\t! int temp=$tmp" %}
5405   ins_encode %{
5406     Register base = reg_to_register_object($mem$$base);
5407     __ add($tmp$$Register, base, $off$$constant);
5408     Address nmem = Address::make_raw($tmp$$reg, $mem$$index, $mem$$scale, $mem$$disp, relocInfo::none);
5409     __ str_32($src$$Register, nmem);
5410   %}
5411   ins_pipe(istore_mem_reg);
5412 %}
5413 #endif
5414 
5415 instruct storeI(memoryI mem, store_RegI src) %{
5416   match(Set mem (StoreI mem src));
5417   ins_cost(MEMORY_REF_COST);
5418 
5419   size(4);
5420   format %{ "str_32 $src,$mem" %}
5421   ins_encode %{
5422     __ str_32($src$$Register, $mem$$Address);
5423   %}
5424   ins_pipe(istore_mem_reg);
5425 %}
5426 
5427 // Store Long
5428 
5429 #ifdef AARCH64
5430 // XXX This variant shouldn't be necessary if 6217251 is implemented
5431 instruct storeLoff(store_RegLd src, memoryScaledL mem, aimmX off, iRegP tmp) %{
5432   match(Set mem (StoreL (AddP mem off) src));
5433   ins_cost(MEMORY_REF_COST + DEFAULT_COST); // assume shift/sign-extend is free
5434   effect(TEMP tmp);
5435   size(4 * 2);
5436 
5437   format %{ "str_64 $src,$mem+$off\t! long temp=$tmp" %}
5438   ins_encode %{
5439     Register base = reg_to_register_object($mem$$base);
5440     __ add($tmp$$Register, base, $off$$constant);
5441     Address nmem = Address::make_raw($tmp$$reg, $mem$$index, $mem$$scale, $mem$$disp, relocInfo::none);
5442     __ str_64($src$$Register, nmem);
5443   %}
5444   ins_pipe(istore_mem_reg);
5445 %}
5446 #endif
5447 
5448 instruct storeL(memoryL mem, store_RegLd src) %{
5449 #ifdef AARCH64
5450   // already atomic for Aarch64
5451 #else
5452   predicate(!((StoreLNode*)n)->require_atomic_access());
5453 #endif
5454   match(Set mem (StoreL mem src));
5455   ins_cost(MEMORY_REF_COST);
5456 
5457   size(4);
5458   format %{ "str_64  $src,$mem\t! long\n\t" %}
5459 
5460   ins_encode %{
5461     __ str_64($src$$Register, $mem$$Address);
5462   %}
5463   ins_pipe(istore_mem_reg);
5464 %}
5465 
5466 #ifndef AARCH64
5467 instruct storeL_2instr(memorylong mem, iRegL src) %{
5468   predicate(!((StoreLNode*)n)->require_atomic_access());
5469   match(Set mem (StoreL mem src));
5470   ins_cost(MEMORY_REF_COST + DEFAULT_COST);
5471 
5472   size(8);
5473   format %{ "STR    $src.lo,$mem\t! long\n\t"
5474             "STR    $src.hi,$mem+4" %}
5475 
5476   ins_encode %{
5477     Address Amemlo = Address::make_raw($mem$$base, $mem$$index, $mem$$scale, $mem$$disp, relocInfo::none);
5478     Address Amemhi = Address::make_raw($mem$$base, $mem$$index, $mem$$scale, $mem$$disp + 4, relocInfo::none);
5479     __ str($src$$Register, Amemlo);
5480     __ str($src$$Register->successor(), Amemhi);
5481   %}
5482   ins_pipe(istore_mem_reg);
5483 %}
5484 
5485 instruct storeL_volatile(indirect mem, iRegL src) %{
5486   predicate(((StoreLNode*)n)->require_atomic_access());
5487   match(Set mem (StoreL mem src));
5488   ins_cost(MEMORY_REF_COST);
5489   size(4);
5490   format %{ "STMIA    $src,$mem\t! long" %}
5491   ins_encode %{
5492     // FIXME: why is stmia considered atomic?  Should be strexd
5493     RegisterSet set($src$$Register);
5494     set = set | reg_to_register_object($src$$reg + 1);
5495     __ stmia(reg_to_register_object($mem$$base), set);
5496   %}
5497   ins_pipe(istore_mem_reg);
5498 %}
5499 #endif // !AARCH64
5500 
5501 #ifndef AARCH64
5502 instruct storeL_volatile_fp(memoryD mem, iRegL src) %{
5503   predicate(((StoreLNode*)n)->require_atomic_access());
5504   match(Set mem (StoreL mem src));
5505   ins_cost(MEMORY_REF_COST);
5506   size(8);
5507   format %{ "FMDRR    S14, $src\t! long \n\t"
5508             "FSTD     S14, $mem" %}
5509   ins_encode %{
5510     __ fmdrr(S14, $src$$Register, $src$$Register->successor());
5511     __ fstd(S14, $mem$$Address);
5512   %}
5513   ins_pipe(istore_mem_reg);
5514 %}
5515 #endif
5516 
5517 #ifdef XXX
5518 // Move SP Pointer
5519 //instruct movSP(sp_ptr_RegP dst, SPRegP src) %{
5520 //instruct movSP(iRegP dst, SPRegP src) %{
5521 instruct movSP(store_ptr_RegP dst, SPRegP src) %{
5522   match(Set dst src);
5523 //predicate(!_kids[1]->_leaf->is_Proj() || _kids[1]->_leaf->as_Proj()->_con == TypeFunc::FramePtr);
5524   ins_cost(MEMORY_REF_COST);
5525   size(4);
5526 
5527   format %{ "MOV    $dst,$src\t! SP ptr\n\t" %}
5528   ins_encode %{
5529     assert(false, "XXX1 got here");
5530     __ mov($dst$$Register, SP);
5531     __ mov($dst$$Register, $src$$Register);
5532   %}
5533   ins_pipe(ialu_reg);
5534 %}
5535 #endif
5536 
5537 #ifdef AARCH64
5538 // FIXME
5539 // Store SP Pointer
5540 instruct storeSP(memoryP mem, SPRegP src, iRegP tmp) %{
5541   match(Set mem (StoreP mem src));
5542   predicate(_kids[1]->_leaf->is_Proj() && _kids[1]->_leaf->as_Proj()->_con == TypeFunc::FramePtr);
5543   // Multiple StoreP rules, different only in register mask.
5544   // Matcher makes the last always valid.  The others will
5545   // only be valid if they cost less than the last valid
5546   // rule.  So cost(rule1) < cost(rule2) < cost(last)
5547   // Unlike immediates, register constraints are not checked
5548   // at match time.
5549   ins_cost(MEMORY_REF_COST+DEFAULT_COST+4);
5550   effect(TEMP tmp);
5551   size(8);
5552 
5553   format %{ "MOV    $tmp,$src\t! SP ptr\n\t"
5554             "STR    $tmp,$mem\t! SP ptr" %}
5555   ins_encode %{
5556     assert($src$$Register == SP, "SP expected");
5557     __ mov($tmp$$Register, $src$$Register);
5558     __ str($tmp$$Register, $mem$$Address);
5559   %}
5560   ins_pipe(istore_mem_spORreg); // FIXME
5561 %}
5562 #endif // AARCH64
5563 
5564 // Store Pointer
5565 
5566 #ifdef AARCH64
5567 // XXX This variant shouldn't be necessary if 6217251 is implemented
5568 instruct storePoff(store_ptr_RegP src, memoryScaledP mem, aimmX off, iRegP tmp) %{
5569   predicate(!_kids[1]->_leaf->is_Proj() || _kids[1]->_leaf->as_Proj()->_con != TypeFunc::FramePtr);
5570   match(Set mem (StoreP (AddP mem off) src));
5571   ins_cost(MEMORY_REF_COST + DEFAULT_COST); // assume shift/sign-extend is free
5572   effect(TEMP tmp);
5573   size(4 * 2);
5574 
5575   format %{ "STR    $src,$mem+$off\t! ptr temp=$tmp" %}
5576   ins_encode %{
5577     Register base = reg_to_register_object($mem$$base);
5578     __ add($tmp$$Register, base, $off$$constant);
5579     Address nmem = Address::make_raw($tmp$$reg, $mem$$index, $mem$$scale, $mem$$disp, relocInfo::none);
5580     __ str($src$$Register, nmem);
5581   %}
5582   ins_pipe(istore_mem_reg);
5583 %}
5584 #endif
5585 
5586 instruct storeP(memoryP mem, store_ptr_RegP src) %{
5587   match(Set mem (StoreP mem src));
5588 #ifdef AARCH64
5589   predicate(!_kids[1]->_leaf->is_Proj() || _kids[1]->_leaf->as_Proj()->_con != TypeFunc::FramePtr);
5590 #endif
5591   ins_cost(MEMORY_REF_COST);
5592   size(4);
5593 
5594   format %{ "STR    $src,$mem\t! ptr" %}
5595   ins_encode %{
5596     __ str($src$$Register, $mem$$Address);
5597   %}
5598   ins_pipe(istore_mem_spORreg);
5599 %}
5600 
5601 #ifdef AARCH64
5602 // Store NULL Pointer
5603 instruct storeP0(memoryP mem, immP0 src) %{
5604   match(Set mem (StoreP mem src));
5605   ins_cost(MEMORY_REF_COST);
5606   size(4);
5607 
5608   format %{ "STR    ZR,$mem\t! ptr" %}
5609   ins_encode %{
5610     __ str(ZR, $mem$$Address);
5611   %}
5612   ins_pipe(istore_mem_spORreg);
5613 %}
5614 #endif // AARCH64
5615 
5616 #ifdef _LP64
5617 // Store Compressed Pointer
5618 
5619 #ifdef AARCH64
5620 // XXX This variant shouldn't be necessary if 6217251 is implemented
5621 instruct storeNoff(store_RegN src, memoryScaledI mem, aimmX off, iRegP tmp) %{
5622   match(Set mem (StoreN (AddP mem off) src));
5623   ins_cost(MEMORY_REF_COST + DEFAULT_COST); // assume shift/sign-extend is free
5624   effect(TEMP tmp);
5625   size(4 * 2);
5626 
5627   format %{ "str_32 $src,$mem+$off\t! compressed ptr temp=$tmp" %}
5628   ins_encode %{
5629     Register base = reg_to_register_object($mem$$base);
5630     __ add($tmp$$Register, base, $off$$constant);
5631     Address nmem = Address::make_raw($tmp$$reg, $mem$$index, $mem$$scale, $mem$$disp, relocInfo::none);
5632     __ str_32($src$$Register, nmem);
5633   %}
5634   ins_pipe(istore_mem_reg);
5635 %}
5636 #endif
5637 
5638 instruct storeN(memoryI mem, store_RegN src) %{
5639   match(Set mem (StoreN mem src));
5640   ins_cost(MEMORY_REF_COST);
5641   size(4);
5642 
5643   format %{ "str_32 $src,$mem\t! compressed ptr" %}
5644   ins_encode %{
5645     __ str_32($src$$Register, $mem$$Address);
5646   %}
5647   ins_pipe(istore_mem_reg);
5648 %}
5649 
5650 #ifdef AARCH64
5651 // Store NULL Pointer
5652 instruct storeN0(memoryI mem, immN0 src) %{
5653   match(Set mem (StoreN mem src));
5654   ins_cost(MEMORY_REF_COST);
5655   size(4);
5656 
5657   format %{ "str_32 ZR,$mem\t! compressed ptr" %}
5658   ins_encode %{
5659     __ str_32(ZR, $mem$$Address);
5660   %}
5661   ins_pipe(istore_mem_reg);
5662 %}
5663 #endif
5664 
5665 // Store Compressed Klass Pointer
5666 instruct storeNKlass(memoryI mem, store_RegN src) %{
5667   match(Set mem (StoreNKlass mem src));
5668   ins_cost(MEMORY_REF_COST);
5669   size(4);
5670 
5671   format %{ "str_32 $src,$mem\t! compressed klass ptr" %}
5672   ins_encode %{
5673     __ str_32($src$$Register, $mem$$Address);
5674   %}
5675   ins_pipe(istore_mem_reg);
5676 %}
5677 #endif
5678 
5679 // Store Double
5680 
5681 #ifdef AARCH64
5682 // XXX This variant shouldn't be necessary if 6217251 is implemented
5683 instruct storeDoff(regD src, memoryScaledD mem, aimmX off, iRegP tmp) %{
5684   match(Set mem (StoreD (AddP mem off) src));
5685   ins_cost(MEMORY_REF_COST + DEFAULT_COST); // assume shift/sign-extend is free
5686   effect(TEMP tmp);
5687   size(4 * 2);
5688 
5689   format %{ "STR    $src,$mem+$off\t! double temp=$tmp" %}
5690   ins_encode %{
5691     Register base = reg_to_register_object($mem$$base);
5692     __ add($tmp$$Register, base, $off$$constant);
5693     Address nmem = Address::make_raw($tmp$$reg, $mem$$index, $mem$$scale, $mem$$disp, relocInfo::none);
5694     __ str_d($src$$FloatRegister, nmem);
5695   %}
5696   ins_pipe(fstoreD_mem_reg);
5697 %}
5698 #endif
5699 
5700 instruct storeD(memoryD mem, regD src) %{
5701   match(Set mem (StoreD mem src));
5702   ins_cost(MEMORY_REF_COST);
5703 
5704   size(4);
5705   // FIXME: needs to be atomic, but  ARMv7 A.R.M. guarantees
5706   // only LDREXD and STREXD are 64-bit single-copy atomic
5707   format %{ "FSTD   $src,$mem" %}
5708   ins_encode %{
5709     __ str_double($src$$FloatRegister, $mem$$Address);
5710   %}
5711   ins_pipe(fstoreD_mem_reg);
5712 %}
5713 
5714 #ifdef AARCH64
5715 instruct movI2F(regF dst, iRegI src) %{
5716   match(Set dst src);
5717   size(4);
5718 
5719   format %{ "FMOV_sw $dst,$src\t! movI2F" %}
5720   ins_encode %{
5721     __ fmov_sw($dst$$FloatRegister, $src$$Register);
5722   %}
5723   ins_pipe(ialu_reg); // FIXME
5724 %}
5725 
5726 instruct movF2I(iRegI dst, regF src) %{
5727   match(Set dst src);
5728   size(4);
5729 
5730   format %{ "FMOV_ws $dst,$src\t! movF2I" %}
5731   ins_encode %{
5732     __ fmov_ws($dst$$Register, $src$$FloatRegister);
5733   %}
5734   ins_pipe(ialu_reg); // FIXME
5735 %}
5736 #endif
5737 
5738 // Store Float
5739 
5740 #ifdef AARCH64
5741 // XXX This variant shouldn't be necessary if 6217251 is implemented
5742 instruct storeFoff(regF src, memoryScaledF mem, aimmX off, iRegP tmp) %{
5743   match(Set mem (StoreF (AddP mem off) src));
5744   ins_cost(MEMORY_REF_COST + DEFAULT_COST); // assume shift/sign-extend is free
5745   effect(TEMP tmp);
5746   size(4 * 2);
5747 
5748   format %{ "str_s  $src,$mem+$off\t! float temp=$tmp" %}
5749   ins_encode %{
5750     Register base = reg_to_register_object($mem$$base);
5751     __ add($tmp$$Register, base, $off$$constant);
5752     Address nmem = Address::make_raw($tmp$$reg, $mem$$index, $mem$$scale, $mem$$disp, relocInfo::none);
5753     __ str_s($src$$FloatRegister, nmem);
5754   %}
5755   ins_pipe(fstoreF_mem_reg);
5756 %}
5757 #endif
5758 
5759 instruct storeF( memoryF mem, regF src) %{
5760   match(Set mem (StoreF mem src));
5761   ins_cost(MEMORY_REF_COST);
5762 
5763   size(4);
5764   format %{ "FSTS    $src,$mem" %}
5765   ins_encode %{
5766     __ str_float($src$$FloatRegister, $mem$$Address);
5767   %}
5768   ins_pipe(fstoreF_mem_reg);
5769 %}
5770 
5771 #ifdef AARCH64
5772 // Convert oop pointer into compressed form
5773 instruct encodeHeapOop(iRegN dst, iRegP src, flagsReg ccr) %{
5774   predicate(n->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
5775   match(Set dst (EncodeP src));
5776   effect(KILL ccr);
5777   format %{ "encode_heap_oop $dst, $src" %}
5778   ins_encode %{
5779     __ encode_heap_oop($dst$$Register, $src$$Register);
5780   %}
5781   ins_pipe(ialu_reg);
5782 %}
5783 
5784 instruct encodeHeapOop_not_null(iRegN dst, iRegP src) %{
5785   predicate(n->bottom_type()->make_ptr()->ptr() == TypePtr::NotNull);
5786   match(Set dst (EncodeP src));
5787   format %{ "encode_heap_oop_not_null $dst, $src" %}
5788   ins_encode %{
5789     __ encode_heap_oop_not_null($dst$$Register, $src$$Register);
5790   %}
5791   ins_pipe(ialu_reg);
5792 %}
5793 
5794 instruct decodeHeapOop(iRegP dst, iRegN src, flagsReg ccr) %{
5795   predicate(n->bottom_type()->is_oopptr()->ptr() != TypePtr::NotNull &&
5796             n->bottom_type()->is_oopptr()->ptr() != TypePtr::Constant);
5797   match(Set dst (DecodeN src));
5798   effect(KILL ccr);
5799   format %{ "decode_heap_oop $dst, $src" %}
5800   ins_encode %{
5801     __ decode_heap_oop($dst$$Register, $src$$Register);
5802   %}
5803   ins_pipe(ialu_reg);
5804 %}
5805 
5806 instruct decodeHeapOop_not_null(iRegP dst, iRegN src) %{
5807   predicate(n->bottom_type()->is_oopptr()->ptr() == TypePtr::NotNull ||
5808             n->bottom_type()->is_oopptr()->ptr() == TypePtr::Constant);
5809   match(Set dst (DecodeN src));
5810   format %{ "decode_heap_oop_not_null $dst, $src" %}
5811   ins_encode %{
5812     __ decode_heap_oop_not_null($dst$$Register, $src$$Register);
5813   %}
5814   ins_pipe(ialu_reg);
5815 %}
5816 
5817 instruct encodeKlass_not_null(iRegN dst, iRegP src) %{
5818   match(Set dst (EncodePKlass src));
5819   format %{ "encode_klass_not_null $dst, $src" %}
5820   ins_encode %{
5821     __ encode_klass_not_null($dst$$Register, $src$$Register);
5822   %}
5823   ins_pipe(ialu_reg);
5824 %}
5825 
5826 instruct decodeKlass_not_null(iRegP dst, iRegN src) %{
5827   match(Set dst (DecodeNKlass src));
5828   format %{ "decode_klass_not_null $dst, $src" %}
5829   ins_encode %{
5830     __ decode_klass_not_null($dst$$Register, $src$$Register);
5831   %}
5832   ins_pipe(ialu_reg);
5833 %}
5834 #endif // AARCH64
5835 
5836 //----------MemBar Instructions-----------------------------------------------
5837 // Memory barrier flavors
5838 
5839 // TODO: take advantage of Aarch64 load-acquire, store-release, etc
5840 // pattern-match out unnecessary membars
5841 instruct membar_storestore() %{
5842   match(MemBarStoreStore);
5843   ins_cost(4*MEMORY_REF_COST);
5844 
5845   size(4);
5846   format %{ "MEMBAR-storestore" %}
5847   ins_encode %{
5848     __ membar(MacroAssembler::Membar_mask_bits(MacroAssembler::StoreStore), noreg);
5849   %}
5850   ins_pipe(long_memory_op);
5851 %}
5852 
5853 instruct membar_acquire() %{
5854   match(MemBarAcquire);
5855   match(LoadFence);
5856   ins_cost(4*MEMORY_REF_COST);
5857 
5858   size(4);
5859   format %{ "MEMBAR-acquire" %}
5860   ins_encode %{
5861     __ membar(MacroAssembler::Membar_mask_bits(MacroAssembler::LoadLoad | MacroAssembler::LoadStore), noreg);
5862   %}
5863   ins_pipe(long_memory_op);
5864 %}
5865 
5866 instruct membar_acquire_lock() %{
5867   match(MemBarAcquireLock);
5868   ins_cost(0);
5869 
5870   size(0);
5871   format %{ "!MEMBAR-acquire (CAS in prior FastLock so empty encoding)" %}
5872   ins_encode( );
5873   ins_pipe(empty);
5874 %}
5875 
5876 instruct membar_release() %{
5877   match(MemBarRelease);
5878   match(StoreFence);
5879   ins_cost(4*MEMORY_REF_COST);
5880 
5881   size(4);
5882   format %{ "MEMBAR-release" %}
5883   ins_encode %{
5884     __ membar(MacroAssembler::Membar_mask_bits(MacroAssembler::StoreStore | MacroAssembler::LoadStore), noreg);
5885   %}
5886   ins_pipe(long_memory_op);
5887 %}
5888 
5889 instruct membar_release_lock() %{
5890   match(MemBarReleaseLock);
5891   ins_cost(0);
5892 
5893   size(0);
5894   format %{ "!MEMBAR-release (CAS in succeeding FastUnlock so empty encoding)" %}
5895   ins_encode( );
5896   ins_pipe(empty);
5897 %}
5898 
5899 instruct membar_volatile() %{
5900   match(MemBarVolatile);
5901   ins_cost(4*MEMORY_REF_COST);
5902 
5903   size(4);
5904   format %{ "MEMBAR-volatile" %}
5905   ins_encode %{
5906     __ membar(MacroAssembler::StoreLoad, noreg);
5907   %}
5908   ins_pipe(long_memory_op);
5909 %}
5910 
5911 instruct unnecessary_membar_volatile() %{
5912   match(MemBarVolatile);
5913   predicate(Matcher::post_store_load_barrier(n));
5914   ins_cost(0);
5915 
5916   size(0);
5917   format %{ "!MEMBAR-volatile (unnecessary so empty encoding)" %}
5918   ins_encode( );
5919   ins_pipe(empty);
5920 %}
5921 
5922 //----------Register Move Instructions-----------------------------------------
5923 // instruct roundDouble_nop(regD dst) %{
5924 //   match(Set dst (RoundDouble dst));
5925 //   ins_pipe(empty);
5926 // %}
5927 
5928 
5929 // instruct roundFloat_nop(regF dst) %{
5930 //   match(Set dst (RoundFloat dst));
5931 //   ins_pipe(empty);
5932 // %}
5933 
5934 
5935 #ifdef AARCH64
5936 // 0 constant in register
5937 instruct zrImmI0(ZRRegI dst, immI0 imm) %{
5938   match(Set dst imm);
5939   size(0);
5940   ins_cost(0);
5941 
5942   format %{ "! ZR (int 0)" %}
5943   ins_encode( /*empty encoding*/ );
5944   ins_pipe(ialu_none);
5945 %}
5946 
5947 // 0 constant in register
5948 instruct zrImmL0(ZRRegL dst, immL0 imm) %{
5949   match(Set dst imm);
5950   size(0);
5951   ins_cost(0);
5952 
5953   format %{ "! ZR (long 0)" %}
5954   ins_encode( /*empty encoding*/ );
5955   ins_pipe(ialu_none);
5956 %}
5957 
5958 #ifdef XXX
5959 // 0 constant in register
5960 instruct zrImmN0(ZRRegN dst, immN0 imm) %{
5961   match(Set dst imm);
5962   size(0);
5963   ins_cost(0);
5964 
5965   format %{ "! ZR (compressed pointer NULL)" %}
5966   ins_encode( /*empty encoding*/ );
5967   ins_pipe(ialu_none);
5968 %}
5969 
5970 // 0 constant in register
5971 instruct zrImmP0(ZRRegP dst, immP0 imm) %{
5972   match(Set dst imm);
5973   size(0);
5974   ins_cost(0);
5975 
5976   format %{ "! ZR (NULL)" %}
5977   ins_encode( /*empty encoding*/ );
5978   ins_pipe(ialu_none);
5979 %}
5980 #endif
5981 #endif // AARCH64
5982 
5983 // Cast Index to Pointer for unsafe natives
5984 instruct castX2P(iRegX src, iRegP dst) %{
5985   match(Set dst (CastX2P src));
5986 
5987   format %{ "MOV    $dst,$src\t! IntX->Ptr if $dst != $src" %}
5988   ins_encode %{
5989     if ($dst$$Register !=  $src$$Register) {
5990       __ mov($dst$$Register, $src$$Register);
5991     }
5992   %}
5993   ins_pipe(ialu_reg);
5994 %}
5995 
5996 // Cast Pointer to Index for unsafe natives
5997 instruct castP2X(iRegP src, iRegX dst) %{
5998   match(Set dst (CastP2X src));
5999 
6000   format %{ "MOV    $dst,$src\t! Ptr->IntX if $dst != $src" %}
6001   ins_encode %{
6002     if ($dst$$Register !=  $src$$Register) {
6003       __ mov($dst$$Register, $src$$Register);
6004     }
6005   %}
6006   ins_pipe(ialu_reg);
6007 %}
6008 
6009 #ifndef AARCH64
6010 //----------Conditional Move---------------------------------------------------
6011 // Conditional move
6012 instruct cmovIP_reg(cmpOpP cmp, flagsRegP pcc, iRegI dst, iRegI src) %{
6013   match(Set dst (CMoveI (Binary cmp pcc) (Binary dst src)));
6014   ins_cost(150);
6015   size(4);
6016   format %{ "MOV$cmp  $dst,$src\t! int" %}
6017   ins_encode %{
6018     __ mov($dst$$Register, $src$$Register, (AsmCondition)($cmp$$cmpcode));
6019   %}
6020   ins_pipe(ialu_reg);
6021 %}
6022 #endif
6023 
6024 #ifdef AARCH64
6025 instruct cmovI_reg3(cmpOp cmp, flagsReg icc, iRegI dst, iRegI src1, iRegI src2) %{
6026   match(Set dst (CMoveI (Binary cmp icc) (Binary src2 src1)));
6027   ins_cost(150);
6028   size(4);
6029   format %{ "CSEL $dst,$src1,$src2,$cmp\t! int" %}
6030   ins_encode %{
6031     __ csel($dst$$Register, $src1$$Register, $src2$$Register, (AsmCondition)($cmp$$cmpcode));
6032   %}
6033   ins_pipe(ialu_reg);
6034 %}
6035 
6036 instruct cmovL_reg3(cmpOp cmp, flagsReg icc, iRegL dst, iRegL src1, iRegL src2) %{
6037   match(Set dst (CMoveL (Binary cmp icc) (Binary src2 src1)));
6038   ins_cost(150);
6039   size(4);
6040   format %{ "CSEL $dst,$src1,$src2,$cmp\t! long" %}
6041   ins_encode %{
6042     __ csel($dst$$Register, $src1$$Register, $src2$$Register, (AsmCondition)($cmp$$cmpcode));
6043   %}
6044   ins_pipe(ialu_reg);
6045 %}
6046 
6047 instruct cmovP_reg3(cmpOp cmp, flagsReg icc, iRegP dst, iRegP src1, iRegP src2) %{
6048   match(Set dst (CMoveP (Binary cmp icc) (Binary src2 src1)));
6049   ins_cost(150);
6050   size(4);
6051   format %{ "CSEL $dst,$src1,$src2,$cmp\t! ptr" %}
6052   ins_encode %{
6053     __ csel($dst$$Register, $src1$$Register, $src2$$Register, (AsmCondition)($cmp$$cmpcode));
6054   %}
6055   ins_pipe(ialu_reg);
6056 %}
6057 
6058 instruct cmovN_reg3(cmpOp cmp, flagsReg icc, iRegN dst, iRegN src1, iRegN src2) %{
6059   match(Set dst (CMoveN (Binary cmp icc) (Binary src2 src1)));
6060   ins_cost(150);
6061   size(4);
6062   format %{ "CSEL $dst,$src1,$src2,$cmp\t! compressed ptr" %}
6063   ins_encode %{
6064     __ csel($dst$$Register, $src1$$Register, $src2$$Register, (AsmCondition)($cmp$$cmpcode));
6065   %}
6066   ins_pipe(ialu_reg);
6067 %}
6068 
6069 instruct cmovIP_reg3(cmpOpP cmp, flagsRegP icc, iRegI dst, iRegI src1, iRegI src2) %{
6070   match(Set dst (CMoveI (Binary cmp icc) (Binary src2 src1)));
6071   ins_cost(150);
6072   size(4);
6073   format %{ "CSEL $dst,$src1,$src2,$cmp\t! int" %}
6074   ins_encode %{
6075     __ csel($dst$$Register, $src1$$Register, $src2$$Register, (AsmCondition)($cmp$$cmpcode));
6076   %}
6077   ins_pipe(ialu_reg);
6078 %}
6079 
6080 instruct cmovLP_reg3(cmpOpP cmp, flagsRegP icc, iRegL dst, iRegL src1, iRegL src2) %{
6081   match(Set dst (CMoveL (Binary cmp icc) (Binary src2 src1)));
6082   ins_cost(150);
6083   size(4);
6084   format %{ "CSEL $dst,$src1,$src2,$cmp\t! long" %}
6085   ins_encode %{
6086     __ csel($dst$$Register, $src1$$Register, $src2$$Register, (AsmCondition)($cmp$$cmpcode));
6087   %}
6088   ins_pipe(ialu_reg);
6089 %}
6090 
6091 instruct cmovPP_reg3(cmpOpP cmp, flagsRegP icc, iRegP dst, iRegP src1, iRegP src2) %{
6092   match(Set dst (CMoveP (Binary cmp icc) (Binary src2 src1)));
6093   ins_cost(150);
6094   size(4);
6095   format %{ "CSEL $dst,$src1,$src2,$cmp\t! ptr" %}
6096   ins_encode %{
6097     __ csel($dst$$Register, $src1$$Register, $src2$$Register, (AsmCondition)($cmp$$cmpcode));
6098   %}
6099   ins_pipe(ialu_reg);
6100 %}
6101 
6102 instruct cmovNP_reg3(cmpOpP cmp, flagsRegP icc, iRegN dst, iRegN src1, iRegN src2) %{
6103   match(Set dst (CMoveN (Binary cmp icc) (Binary src2 src1)));
6104   ins_cost(150);
6105   size(4);
6106   format %{ "CSEL $dst,$src1,$src2,$cmp\t! compressed ptr" %}
6107   ins_encode %{
6108     __ csel($dst$$Register, $src1$$Register, $src2$$Register, (AsmCondition)($cmp$$cmpcode));
6109   %}
6110   ins_pipe(ialu_reg);
6111 %}
6112 
6113 instruct cmovIU_reg3(cmpOpU cmp, flagsRegU icc, iRegI dst, iRegI src1, iRegI src2) %{
6114   match(Set dst (CMoveI (Binary cmp icc) (Binary src2 src1)));
6115   ins_cost(150);
6116   size(4);
6117   format %{ "CSEL $dst,$src1,$src2,$cmp\t! int" %}
6118   ins_encode %{
6119     __ csel($dst$$Register, $src1$$Register, $src2$$Register, (AsmCondition)($cmp$$cmpcode));
6120   %}
6121   ins_pipe(ialu_reg);
6122 %}
6123 
6124 instruct cmovLU_reg3(cmpOpU cmp, flagsRegU icc, iRegL dst, iRegL src1, iRegL src2) %{
6125   match(Set dst (CMoveL (Binary cmp icc) (Binary src2 src1)));
6126   ins_cost(150);
6127   size(4);
6128   format %{ "CSEL $dst,$src1,$src2,$cmp\t! long" %}
6129   ins_encode %{
6130     __ csel($dst$$Register, $src1$$Register, $src2$$Register, (AsmCondition)($cmp$$cmpcode));
6131   %}
6132   ins_pipe(ialu_reg);
6133 %}
6134 
6135 instruct cmovPU_reg3(cmpOpU cmp, flagsRegU icc, iRegP dst, iRegP src1, iRegP src2) %{
6136   match(Set dst (CMoveP (Binary cmp icc) (Binary src2 src1)));
6137   ins_cost(150);
6138   size(4);
6139   format %{ "CSEL $dst,$src1,$src2,$cmp\t! ptr" %}
6140   ins_encode %{
6141     __ csel($dst$$Register, $src1$$Register, $src2$$Register, (AsmCondition)($cmp$$cmpcode));
6142   %}
6143   ins_pipe(ialu_reg);
6144 %}
6145 
6146 instruct cmovNU_reg3(cmpOpU cmp, flagsRegU icc, iRegN dst, iRegN src1, iRegN src2) %{
6147   match(Set dst (CMoveN (Binary cmp icc) (Binary src2 src1)));
6148   ins_cost(150);
6149   size(4);
6150   format %{ "CSEL $dst,$src1,$src2,$cmp\t! compressed ptr" %}
6151   ins_encode %{
6152     __ csel($dst$$Register, $src1$$Register, $src2$$Register, (AsmCondition)($cmp$$cmpcode));
6153   %}
6154   ins_pipe(ialu_reg);
6155 %}
6156 
6157 instruct cmovIZ_reg3(cmpOp0 cmp, flagsReg_EQNELTGE icc, iRegI dst, iRegI src1, iRegI src2) %{
6158   match(Set dst (CMoveI (Binary cmp icc) (Binary src2 src1)));
6159   ins_cost(150);
6160   size(4);
6161   format %{ "CSEL $dst,$src1,$src2,$cmp\t! int" %}
6162   ins_encode %{
6163     __ csel($dst$$Register, $src1$$Register, $src2$$Register, (AsmCondition)($cmp$$cmpcode));
6164   %}
6165   ins_pipe(ialu_reg);
6166 %}
6167 
6168 instruct cmovLZ_reg3(cmpOp0 cmp, flagsReg_EQNELTGE icc, iRegL dst, iRegL src1, iRegL src2) %{
6169   match(Set dst (CMoveL (Binary cmp icc) (Binary src2 src1)));
6170   ins_cost(150);
6171   size(4);
6172   format %{ "CSEL $dst,$src1,$src2,$cmp\t! long" %}
6173   ins_encode %{
6174     __ csel($dst$$Register, $src1$$Register, $src2$$Register, (AsmCondition)($cmp$$cmpcode));
6175   %}
6176   ins_pipe(ialu_reg);
6177 %}
6178 
6179 instruct cmovPZ_reg3(cmpOp0 cmp, flagsReg_EQNELTGE icc, iRegP dst, iRegP src1, iRegP src2) %{
6180   match(Set dst (CMoveP (Binary cmp icc) (Binary src2 src1)));
6181   ins_cost(150);
6182   size(4);
6183   format %{ "CSEL $dst,$src1,$src2,$cmp\t! ptr" %}
6184   ins_encode %{
6185     __ csel($dst$$Register, $src1$$Register, $src2$$Register, (AsmCondition)($cmp$$cmpcode));
6186   %}
6187   ins_pipe(ialu_reg);
6188 %}
6189 
6190 instruct cmovNZ_reg3(cmpOp0 cmp, flagsReg_EQNELTGE icc, iRegN dst, iRegN src1, iRegN src2) %{
6191   match(Set dst (CMoveN (Binary cmp icc) (Binary src2 src1)));
6192   ins_cost(150);
6193   size(4);
6194   format %{ "CSEL $dst,$src1,$src2,$cmp\t! compressed ptr" %}
6195   ins_encode %{
6196     __ csel($dst$$Register, $src1$$Register, $src2$$Register, (AsmCondition)($cmp$$cmpcode));
6197   %}
6198   ins_pipe(ialu_reg);
6199 %}
6200 #endif // AARCH64
6201 
6202 #ifndef AARCH64
6203 instruct cmovIP_immMov(cmpOpP cmp, flagsRegP pcc, iRegI dst, immIMov src) %{
6204   match(Set dst (CMoveI (Binary cmp pcc) (Binary dst src)));
6205   ins_cost(140);
6206   size(4);
6207   format %{ "MOV$cmp  $dst,$src" %}
6208   ins_encode %{
6209     __ mov($dst$$Register, $src$$constant, (AsmCondition)($cmp$$cmpcode));
6210   %}
6211   ins_pipe(ialu_imm);
6212 %}
6213 
6214 instruct cmovIP_imm16(cmpOpP cmp, flagsRegP pcc, iRegI dst, immI16 src) %{
6215   match(Set dst (CMoveI (Binary cmp pcc) (Binary dst src)));
6216   ins_cost(140);
6217   size(4);
6218   format %{ "MOVw$cmp  $dst,$src" %}
6219   ins_encode %{
6220     __ movw($dst$$Register, $src$$constant, (AsmCondition)($cmp$$cmpcode));
6221   %}
6222   ins_pipe(ialu_imm);
6223 %}
6224 #endif
6225 
6226 instruct cmovI_reg(cmpOp cmp, flagsReg icc, iRegI dst, iRegI src) %{
6227   match(Set dst (CMoveI (Binary cmp icc) (Binary dst src)));
6228   ins_cost(150);
6229   size(4);
6230   format %{ "MOV$cmp  $dst,$src" %}
6231   ins_encode %{
6232     __ mov($dst$$Register, $src$$Register, (AsmCondition)($cmp$$cmpcode));
6233   %}
6234   ins_pipe(ialu_reg);
6235 %}
6236 
6237 #ifdef AARCH64
6238 instruct cmovL_reg(cmpOp cmp, flagsReg icc, iRegL dst, iRegL src) %{
6239   match(Set dst (CMoveL (Binary cmp icc) (Binary dst src)));
6240   ins_cost(150);
6241   size(4);
6242   format %{ "MOV$cmp  $dst,$src\t! long" %}
6243   ins_encode %{
6244     __ mov($dst$$Register, $src$$Register, (AsmCondition)($cmp$$cmpcode));
6245   %}
6246   ins_pipe(ialu_reg);
6247 %}
6248 #endif
6249 
6250 #ifndef AARCH64
6251 instruct cmovI_immMov(cmpOp cmp, flagsReg icc, iRegI dst, immIMov src) %{
6252   match(Set dst (CMoveI (Binary cmp icc) (Binary dst src)));
6253   ins_cost(140);
6254   size(4);
6255   format %{ "MOV$cmp  $dst,$src" %}
6256   ins_encode %{
6257     __ mov($dst$$Register, $src$$constant, (AsmCondition)($cmp$$cmpcode));
6258   %}
6259   ins_pipe(ialu_imm);
6260 %}
6261 
6262 instruct cmovII_imm16(cmpOp cmp, flagsReg icc, iRegI dst, immI16 src) %{
6263   match(Set dst (CMoveI (Binary cmp icc) (Binary dst src)));
6264   ins_cost(140);
6265   size(4);
6266   format %{ "MOVw$cmp  $dst,$src" %}
6267   ins_encode %{
6268     __ movw($dst$$Register, $src$$constant, (AsmCondition)($cmp$$cmpcode));
6269   %}
6270   ins_pipe(ialu_imm);
6271 %}
6272 #endif
6273 
6274 instruct cmovII_reg_EQNELTGE(cmpOp0 cmp, flagsReg_EQNELTGE icc, iRegI dst, iRegI src) %{
6275   match(Set dst (CMoveI (Binary cmp icc) (Binary dst src)));
6276   predicate(_kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::eq ||
6277             _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ne ||
6278             _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::lt ||
6279             _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ge);
6280   ins_cost(150);
6281   size(4);
6282   format %{ "MOV$cmp  $dst,$src" %}
6283   ins_encode %{
6284     __ mov($dst$$Register, $src$$Register, (AsmCondition)($cmp$$cmpcode));
6285   %}
6286   ins_pipe(ialu_reg);
6287 %}
6288 
6289 #ifndef AARCH64
6290 instruct cmovII_immMov_EQNELTGE(cmpOp0 cmp, flagsReg_EQNELTGE icc, iRegI dst, immIMov src) %{
6291   match(Set dst (CMoveI (Binary cmp icc) (Binary dst src)));
6292   predicate(_kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::eq ||
6293             _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ne ||
6294             _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::lt ||
6295             _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ge);
6296   ins_cost(140);
6297   size(4);
6298   format %{ "MOV$cmp  $dst,$src" %}
6299   ins_encode %{
6300     __ mov($dst$$Register, $src$$constant, (AsmCondition)($cmp$$cmpcode));
6301   %}
6302   ins_pipe(ialu_imm);
6303 %}
6304 
6305 instruct cmovII_imm16_EQNELTGE(cmpOp0 cmp, flagsReg_EQNELTGE icc, iRegI dst, immI16 src) %{
6306   match(Set dst (CMoveI (Binary cmp icc) (Binary dst src)));
6307   predicate(_kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::eq ||
6308             _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ne ||
6309             _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::lt ||
6310             _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ge);
6311   ins_cost(140);
6312   size(4);
6313   format %{ "MOVW$cmp  $dst,$src" %}
6314   ins_encode %{
6315     __ movw($dst$$Register, $src$$constant, (AsmCondition)($cmp$$cmpcode));
6316   %}
6317   ins_pipe(ialu_imm);
6318 %}
6319 #endif
6320 
6321 instruct cmovIIu_reg(cmpOpU cmp, flagsRegU icc, iRegI dst, iRegI src) %{
6322   match(Set dst (CMoveI (Binary cmp icc) (Binary dst src)));
6323   ins_cost(150);
6324   size(4);
6325   format %{ "MOV$cmp  $dst,$src" %}
6326   ins_encode %{
6327     __ mov($dst$$Register, $src$$Register, (AsmCondition)($cmp$$cmpcode));
6328   %}
6329   ins_pipe(ialu_reg);
6330 %}
6331 
6332 #ifndef AARCH64
6333 instruct cmovIIu_immMov(cmpOpU cmp, flagsRegU icc, iRegI dst, immIMov src) %{
6334   match(Set dst (CMoveI (Binary cmp icc) (Binary dst src)));
6335   ins_cost(140);
6336   size(4);
6337   format %{ "MOV$cmp  $dst,$src" %}
6338   ins_encode %{
6339     __ mov($dst$$Register, $src$$constant, (AsmCondition)($cmp$$cmpcode));
6340   %}
6341   ins_pipe(ialu_imm);
6342 %}
6343 
6344 instruct cmovIIu_imm16(cmpOpU cmp, flagsRegU icc, iRegI dst, immI16 src) %{
6345   match(Set dst (CMoveI (Binary cmp icc) (Binary dst src)));
6346   ins_cost(140);
6347   size(4);
6348   format %{ "MOVW$cmp  $dst,$src" %}
6349   ins_encode %{
6350     __ movw($dst$$Register, $src$$constant, (AsmCondition)($cmp$$cmpcode));
6351   %}
6352   ins_pipe(ialu_imm);
6353 %}
6354 #endif
6355 
6356 // Conditional move
6357 instruct cmovPP_reg(cmpOpP cmp, flagsRegP pcc, iRegP dst, iRegP src) %{
6358   match(Set dst (CMoveP (Binary cmp pcc) (Binary dst src)));
6359   ins_cost(150);
6360   size(4);
6361   format %{ "MOV$cmp  $dst,$src" %}
6362   ins_encode %{
6363     __ mov($dst$$Register, $src$$Register, (AsmCondition)($cmp$$cmpcode));
6364   %}
6365   ins_pipe(ialu_reg);
6366 %}
6367 
6368 instruct cmovPP_imm(cmpOpP cmp, flagsRegP pcc, iRegP dst, immP0 src) %{
6369   match(Set dst (CMoveP (Binary cmp pcc) (Binary dst src)));
6370   ins_cost(140);
6371   size(4);
6372 #ifdef AARCH64
6373   format %{ "MOV$cmp  $dst,ZR" %}
6374 #else
6375   format %{ "MOV$cmp  $dst,$src" %}
6376 #endif
6377   ins_encode %{
6378 #ifdef AARCH64
6379     __ mov($dst$$Register,             ZR, (AsmCondition)($cmp$$cmpcode));
6380 #else
6381     __ mov($dst$$Register, $src$$constant, (AsmCondition)($cmp$$cmpcode));
6382 #endif
6383   %}
6384   ins_pipe(ialu_imm);
6385 %}
6386 
6387 // This instruction also works with CmpN so we don't need cmovPN_reg.
6388 instruct cmovPI_reg(cmpOp cmp, flagsReg icc, iRegP dst, iRegP src) %{
6389   match(Set dst (CMoveP (Binary cmp icc) (Binary dst src)));
6390   ins_cost(150);
6391 
6392   size(4);
6393   format %{ "MOV$cmp  $dst,$src\t! ptr" %}
6394   ins_encode %{
6395     __ mov($dst$$Register, $src$$Register, (AsmCondition)($cmp$$cmpcode));
6396   %}
6397   ins_pipe(ialu_reg);
6398 %}
6399 
6400 instruct cmovPI_reg_EQNELTGE(cmpOp0 cmp, flagsReg_EQNELTGE icc, iRegP dst, iRegP src) %{
6401   match(Set dst (CMoveP (Binary cmp icc) (Binary dst src)));
6402   predicate(_kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::eq ||
6403             _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ne ||
6404             _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::lt ||
6405             _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ge);
6406   ins_cost(150);
6407 
6408   size(4);
6409   format %{ "MOV$cmp  $dst,$src\t! ptr" %}
6410   ins_encode %{
6411     __ mov($dst$$Register, $src$$Register, (AsmCondition)($cmp$$cmpcode));
6412   %}
6413   ins_pipe(ialu_reg);
6414 %}
6415 
6416 instruct cmovPIu_reg(cmpOpU cmp, flagsRegU icc, iRegP dst, iRegP src) %{
6417   match(Set dst (CMoveP (Binary cmp icc) (Binary dst src)));
6418   ins_cost(150);
6419 
6420   size(4);
6421   format %{ "MOV$cmp  $dst,$src\t! ptr" %}
6422   ins_encode %{
6423     __ mov($dst$$Register, $src$$Register, (AsmCondition)($cmp$$cmpcode));
6424   %}
6425   ins_pipe(ialu_reg);
6426 %}
6427 
6428 instruct cmovPI_imm(cmpOp cmp, flagsReg icc, iRegP dst, immP0 src) %{
6429   match(Set dst (CMoveP (Binary cmp icc) (Binary dst src)));
6430   ins_cost(140);
6431 
6432   size(4);
6433 #ifdef AARCH64
6434   format %{ "MOV$cmp  $dst,ZR\t! ptr" %}
6435 #else
6436   format %{ "MOV$cmp  $dst,$src\t! ptr" %}
6437 #endif
6438   ins_encode %{
6439 #ifdef AARCH64
6440     __ mov($dst$$Register,             ZR, (AsmCondition)($cmp$$cmpcode));
6441 #else
6442     __ mov($dst$$Register, $src$$constant, (AsmCondition)($cmp$$cmpcode));
6443 #endif
6444   %}
6445   ins_pipe(ialu_imm);
6446 %}
6447 
6448 instruct cmovPI_imm_EQNELTGE(cmpOp0 cmp, flagsReg_EQNELTGE icc, iRegP dst, immP0 src) %{
6449   match(Set dst (CMoveP (Binary cmp icc) (Binary dst src)));
6450   predicate(_kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::eq ||
6451             _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ne ||
6452             _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::lt ||
6453             _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ge);
6454   ins_cost(140);
6455 
6456   size(4);
6457 #ifdef AARCH64
6458   format %{ "MOV$cmp  $dst,ZR\t! ptr" %}
6459 #else
6460   format %{ "MOV$cmp  $dst,$src\t! ptr" %}
6461 #endif
6462   ins_encode %{
6463 #ifdef AARCH64
6464     __ mov($dst$$Register,             ZR, (AsmCondition)($cmp$$cmpcode));
6465 #else
6466     __ mov($dst$$Register, $src$$constant, (AsmCondition)($cmp$$cmpcode));
6467 #endif
6468   %}
6469   ins_pipe(ialu_imm);
6470 %}
6471 
6472 instruct cmovPIu_imm(cmpOpU cmp, flagsRegU icc, iRegP dst, immP0 src) %{
6473   match(Set dst (CMoveP (Binary cmp icc) (Binary dst src)));
6474   ins_cost(140);
6475 
6476   size(4);
6477 #ifdef AARCH64
6478   format %{ "MOV$cmp  $dst,ZR\t! ptr" %}
6479 #else
6480   format %{ "MOV$cmp  $dst,$src\t! ptr" %}
6481 #endif
6482   ins_encode %{
6483 #ifdef AARCH64
6484     __ mov($dst$$Register,             ZR, (AsmCondition)($cmp$$cmpcode));
6485 #else
6486     __ mov($dst$$Register, $src$$constant, (AsmCondition)($cmp$$cmpcode));
6487 #endif
6488   %}
6489   ins_pipe(ialu_imm);
6490 %}
6491 
6492 #ifdef AARCH64
6493 // Conditional move
6494 instruct cmovF_reg(cmpOp cmp, flagsReg icc, regF dst, regF src1, regF src2) %{
6495   match(Set dst (CMoveF (Binary cmp icc) (Binary src2 src1)));
6496   ins_cost(150);
6497   size(4);
6498   format %{ "FCSEL_s $dst,$src1,$src2,$cmp" %}
6499   ins_encode %{
6500     __ fcsel_s($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister, (AsmCondition)($cmp$$cmpcode));
6501   %}
6502   ins_pipe(int_conditional_float_move);
6503 %}
6504 
6505 instruct cmovD_reg(cmpOp cmp, flagsReg icc, regD dst, regD src1, regD src2) %{
6506   match(Set dst (CMoveD (Binary cmp icc) (Binary src2 src1)));
6507   ins_cost(150);
6508   size(4);
6509   format %{ "FCSEL_d $dst,$src1,$src2,$cmp" %}
6510   ins_encode %{
6511     __ fcsel_d($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister, (AsmCondition)($cmp$$cmpcode));
6512   %}
6513   ins_pipe(int_conditional_float_move);
6514 %}
6515 
6516 instruct cmovFP_reg(cmpOpP cmp, flagsRegP icc, regF dst, regF src1, regF src2) %{
6517   match(Set dst (CMoveF (Binary cmp icc) (Binary src2 src1)));
6518   ins_cost(150);
6519   size(4);
6520   format %{ "FCSEL_s $dst,$src1,$src2,$cmp" %}
6521   ins_encode %{
6522     __ fcsel_s($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister, (AsmCondition)($cmp$$cmpcode));
6523   %}
6524   ins_pipe(int_conditional_float_move);
6525 %}
6526 
6527 instruct cmovDP_reg(cmpOpP cmp, flagsRegP icc, regD dst, regD src1, regD src2) %{
6528   match(Set dst (CMoveD (Binary cmp icc) (Binary src2 src1)));
6529   ins_cost(150);
6530   size(4);
6531   format %{ "FCSEL_d $dst,$src1,$src2,$cmp" %}
6532   ins_encode %{
6533     __ fcsel_d($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister, (AsmCondition)($cmp$$cmpcode));
6534   %}
6535   ins_pipe(int_conditional_float_move);
6536 %}
6537 
6538 instruct cmovFU_reg(cmpOpU cmp, flagsRegU icc, regF dst, regF src1, regF src2) %{
6539   match(Set dst (CMoveF (Binary cmp icc) (Binary src2 src1)));
6540   ins_cost(150);
6541   size(4);
6542   format %{ "FCSEL_s $dst,$src1,$src2,$cmp" %}
6543   ins_encode %{
6544     __ fcsel_s($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister, (AsmCondition)($cmp$$cmpcode));
6545   %}
6546   ins_pipe(int_conditional_float_move);
6547 %}
6548 
6549 instruct cmovDU_reg(cmpOpU cmp, flagsRegU icc, regD dst, regD src1, regD src2) %{
6550   match(Set dst (CMoveD (Binary cmp icc) (Binary src2 src1)));
6551   ins_cost(150);
6552   size(4);
6553   format %{ "FCSEL_d $dst,$src1,$src2,$cmp" %}
6554   ins_encode %{
6555     __ fcsel_d($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister, (AsmCondition)($cmp$$cmpcode));
6556   %}
6557   ins_pipe(int_conditional_float_move);
6558 %}
6559 
6560 instruct cmovFZ_reg(cmpOp0 cmp, flagsReg_EQNELTGE icc, regF dst, regF src1, regF src2) %{
6561   match(Set dst (CMoveF (Binary cmp icc) (Binary src2 src1)));
6562   ins_cost(150);
6563   size(4);
6564   format %{ "FCSEL_s $dst,$src1,$src2,$cmp" %}
6565   ins_encode %{
6566     __ fcsel_s($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister, (AsmCondition)($cmp$$cmpcode));
6567   %}
6568   ins_pipe(int_conditional_float_move);
6569 %}
6570 
6571 instruct cmovDZ_reg(cmpOp0 cmp, flagsReg_EQNELTGE icc, regD dst, regD src1, regD src2) %{
6572   match(Set dst (CMoveD (Binary cmp icc) (Binary src2 src1)));
6573   ins_cost(150);
6574   size(4);
6575   format %{ "FCSEL_d $dst,$src1,$src2,$cmp" %}
6576   ins_encode %{
6577     __ fcsel_d($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister, (AsmCondition)($cmp$$cmpcode));
6578   %}
6579   ins_pipe(int_conditional_float_move);
6580 %}
6581 
6582 #else // !AARCH64
6583 
6584 // Conditional move
6585 instruct cmovFP_reg(cmpOpP cmp, flagsRegP pcc, regF dst, regF src) %{
6586   match(Set dst (CMoveF (Binary cmp pcc) (Binary dst src)));
6587   ins_cost(150);
6588   size(4);
6589   format %{ "FCPYS$cmp $dst,$src" %}
6590   ins_encode %{
6591     __ fcpys($dst$$FloatRegister, $src$$FloatRegister, (AsmCondition)($cmp$$cmpcode));
6592   %}
6593   ins_pipe(int_conditional_float_move);
6594 %}
6595 
6596 instruct cmovFI_reg(cmpOp cmp, flagsReg icc, regF dst, regF src) %{
6597   match(Set dst (CMoveF (Binary cmp icc) (Binary dst src)));
6598   ins_cost(150);
6599 
6600   size(4);
6601   format %{ "FCPYS$cmp $dst,$src" %}
6602   ins_encode %{
6603     __ fcpys($dst$$FloatRegister, $src$$FloatRegister, (AsmCondition)($cmp$$cmpcode));
6604   %}
6605   ins_pipe(int_conditional_float_move);
6606 %}
6607 
6608 instruct cmovFI_reg_EQNELTGE(cmpOp0 cmp, flagsReg_EQNELTGE icc, regF dst, regF src) %{
6609   match(Set dst (CMoveF (Binary cmp icc) (Binary dst src)));
6610   predicate(_kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::eq ||
6611             _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ne ||
6612             _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::lt ||
6613             _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ge);
6614   ins_cost(150);
6615 
6616   size(4);
6617   format %{ "FCPYS$cmp $dst,$src" %}
6618   ins_encode %{
6619     __ fcpys($dst$$FloatRegister, $src$$FloatRegister, (AsmCondition)($cmp$$cmpcode));
6620   %}
6621   ins_pipe(int_conditional_float_move);
6622 %}
6623 
6624 instruct cmovFIu_reg(cmpOpU cmp, flagsRegU icc, regF dst, regF src) %{
6625   match(Set dst (CMoveF (Binary cmp icc) (Binary dst src)));
6626   ins_cost(150);
6627 
6628   size(4);
6629   format %{ "FCPYS$cmp $dst,$src" %}
6630   ins_encode %{
6631     __ fcpys($dst$$FloatRegister, $src$$FloatRegister, (AsmCondition)($cmp$$cmpcode));
6632   %}
6633   ins_pipe(int_conditional_float_move);
6634 %}
6635 
6636 // Conditional move
6637 instruct cmovDP_reg(cmpOpP cmp, flagsRegP pcc, regD dst, regD src) %{
6638   match(Set dst (CMoveD (Binary cmp pcc) (Binary dst src)));
6639   ins_cost(150);
6640   size(4);
6641   format %{ "FCPYD$cmp $dst,$src" %}
6642   ins_encode %{
6643     __ fcpyd($dst$$FloatRegister, $src$$FloatRegister, (AsmCondition)($cmp$$cmpcode));
6644   %}
6645   ins_pipe(int_conditional_double_move);
6646 %}
6647 
6648 instruct cmovDI_reg(cmpOp cmp, flagsReg icc, regD dst, regD src) %{
6649   match(Set dst (CMoveD (Binary cmp icc) (Binary dst src)));
6650   ins_cost(150);
6651 
6652   size(4);
6653   format %{ "FCPYD$cmp $dst,$src" %}
6654   ins_encode %{
6655     __ fcpyd($dst$$FloatRegister, $src$$FloatRegister, (AsmCondition)($cmp$$cmpcode));
6656   %}
6657   ins_pipe(int_conditional_double_move);
6658 %}
6659 
6660 instruct cmovDI_reg_EQNELTGE(cmpOp0 cmp, flagsReg_EQNELTGE icc, regD dst, regD src) %{
6661   match(Set dst (CMoveD (Binary cmp icc) (Binary dst src)));
6662   predicate(_kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::eq || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ne || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::lt || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ge);
6663   ins_cost(150);
6664 
6665   size(4);
6666   format %{ "FCPYD$cmp $dst,$src" %}
6667   ins_encode %{
6668     __ fcpyd($dst$$FloatRegister, $src$$FloatRegister, (AsmCondition)($cmp$$cmpcode));
6669   %}
6670   ins_pipe(int_conditional_double_move);
6671 %}
6672 
6673 instruct cmovDIu_reg(cmpOpU cmp, flagsRegU icc, regD dst, regD src) %{
6674   match(Set dst (CMoveD (Binary cmp icc) (Binary dst src)));
6675   ins_cost(150);
6676 
6677   size(4);
6678   format %{ "FCPYD$cmp $dst,$src" %}
6679   ins_encode %{
6680     __ fcpyd($dst$$FloatRegister, $src$$FloatRegister, (AsmCondition)($cmp$$cmpcode));
6681   %}
6682   ins_pipe(int_conditional_double_move);
6683 %}
6684 
6685 // Conditional move
6686 instruct cmovLP_reg(cmpOpP cmp, flagsRegP pcc, iRegL dst, iRegL src) %{
6687   match(Set dst (CMoveL (Binary cmp pcc) (Binary dst src)));
6688   ins_cost(150);
6689 
6690   size(8);
6691   format %{ "MOV$cmp  $dst.lo,$src.lo\t! long\n\t"
6692             "MOV$cmp  $dst.hi,$src.hi" %}
6693   ins_encode %{
6694     __ mov($dst$$Register, $src$$Register, (AsmCondition)($cmp$$cmpcode));
6695     __ mov($dst$$Register->successor(), $src$$Register->successor(), (AsmCondition)($cmp$$cmpcode));
6696   %}
6697   ins_pipe(ialu_reg);
6698 %}
6699 
6700 // TODO: try immLRot2 instead, (0, $con$$constant) becomes
6701 // (hi($con$$constant), lo($con$$constant)) becomes
6702 instruct cmovLP_immRot(cmpOpP cmp, flagsRegP pcc, iRegL dst, immLlowRot src) %{
6703   match(Set dst (CMoveL (Binary cmp pcc) (Binary dst src)));
6704   ins_cost(140);
6705 
6706   size(8);
6707   format %{ "MOV$cmp  $dst.lo,$src\t! long\n\t"
6708             "MOV$cmp  $dst.hi,0" %}
6709   ins_encode %{
6710     __ mov($dst$$Register, $src$$constant, (AsmCondition)($cmp$$cmpcode));
6711     __ mov($dst$$Register->successor(), 0, (AsmCondition)($cmp$$cmpcode));
6712   %}
6713   ins_pipe(ialu_imm);
6714 %}
6715 
6716 instruct cmovLP_imm16(cmpOpP cmp, flagsRegP pcc, iRegL dst, immL16 src) %{
6717   match(Set dst (CMoveL (Binary cmp pcc) (Binary dst src)));
6718   ins_cost(140);
6719 
6720   size(8);
6721   format %{ "MOV$cmp  $dst.lo,$src\t! long\n\t"
6722             "MOV$cmp  $dst.hi,0" %}
6723   ins_encode %{
6724     __ movw($dst$$Register, $src$$constant, (AsmCondition)($cmp$$cmpcode));
6725     __ mov($dst$$Register->successor(), 0, (AsmCondition)($cmp$$cmpcode));
6726   %}
6727   ins_pipe(ialu_imm);
6728 %}
6729 
6730 instruct cmovLI_reg(cmpOp cmp, flagsReg icc, iRegL dst, iRegL src) %{
6731   match(Set dst (CMoveL (Binary cmp icc) (Binary dst src)));
6732   ins_cost(150);
6733 
6734   size(8);
6735   format %{ "MOV$cmp  $dst.lo,$src.lo\t! long\n\t"
6736             "MOV$cmp  $dst.hi,$src.hi" %}
6737   ins_encode %{
6738     __ mov($dst$$Register, $src$$Register, (AsmCondition)($cmp$$cmpcode));
6739     __ mov($dst$$Register->successor(), $src$$Register->successor(), (AsmCondition)($cmp$$cmpcode));
6740   %}
6741   ins_pipe(ialu_reg);
6742 %}
6743 
6744 instruct cmovLI_reg_EQNELTGE(cmpOp0 cmp, flagsReg_EQNELTGE icc, iRegL dst, iRegL src) %{
6745   match(Set dst (CMoveL (Binary cmp icc) (Binary dst src)));
6746   predicate(_kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::eq ||
6747             _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ne ||
6748             _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::lt ||
6749             _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ge);
6750   ins_cost(150);
6751 
6752   size(8);
6753   format %{ "MOV$cmp  $dst.lo,$src.lo\t! long\n\t"
6754             "MOV$cmp  $dst.hi,$src.hi" %}
6755   ins_encode %{
6756     __ mov($dst$$Register, $src$$Register, (AsmCondition)($cmp$$cmpcode));
6757     __ mov($dst$$Register->successor(), $src$$Register->successor(), (AsmCondition)($cmp$$cmpcode));
6758   %}
6759   ins_pipe(ialu_reg);
6760 %}
6761 
6762 // TODO: try immLRot2 instead, (0, $con$$constant) becomes
6763 // (hi($con$$constant), lo($con$$constant)) becomes
6764 instruct cmovLI_immRot(cmpOp cmp, flagsReg icc, iRegL dst, immLlowRot src) %{
6765   match(Set dst (CMoveL (Binary cmp icc) (Binary dst src)));
6766   ins_cost(140);
6767 
6768   size(8);
6769   format %{ "MOV$cmp  $dst.lo,$src\t! long\n\t"
6770             "MOV$cmp  $dst.hi,0" %}
6771   ins_encode %{
6772     __ mov($dst$$Register, $src$$constant, (AsmCondition)($cmp$$cmpcode));
6773     __ mov($dst$$Register->successor(), 0, (AsmCondition)($cmp$$cmpcode));
6774   %}
6775   ins_pipe(ialu_imm);
6776 %}
6777 
6778 // TODO: try immLRot2 instead, (0, $con$$constant) becomes
6779 // (hi($con$$constant), lo($con$$constant)) becomes
6780 instruct cmovLI_immRot_EQNELTGE(cmpOp0 cmp, flagsReg_EQNELTGE icc, iRegL dst, immLlowRot src) %{
6781   match(Set dst (CMoveL (Binary cmp icc) (Binary dst src)));
6782   predicate(_kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::eq ||
6783             _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ne ||
6784             _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::lt ||
6785             _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ge);
6786   ins_cost(140);
6787 
6788   size(8);
6789   format %{ "MOV$cmp  $dst.lo,$src\t! long\n\t"
6790             "MOV$cmp  $dst.hi,0" %}
6791   ins_encode %{
6792     __ mov($dst$$Register, $src$$constant, (AsmCondition)($cmp$$cmpcode));
6793     __ mov($dst$$Register->successor(), 0, (AsmCondition)($cmp$$cmpcode));
6794   %}
6795   ins_pipe(ialu_imm);
6796 %}
6797 
6798 instruct cmovLI_imm16(cmpOp cmp, flagsReg icc, iRegL dst, immL16 src) %{
6799   match(Set dst (CMoveL (Binary cmp icc) (Binary dst src)));
6800   ins_cost(140);
6801 
6802   size(8);
6803   format %{ "MOV$cmp  $dst.lo,$src\t! long\n\t"
6804             "MOV$cmp  $dst.hi,0" %}
6805   ins_encode %{
6806     __ movw($dst$$Register, $src$$constant, (AsmCondition)($cmp$$cmpcode));
6807     __ movw($dst$$Register->successor(), 0, (AsmCondition)($cmp$$cmpcode));
6808   %}
6809   ins_pipe(ialu_imm);
6810 %}
6811 
6812 instruct cmovLI_imm16_EQNELTGE(cmpOp0 cmp, flagsReg_EQNELTGE icc, iRegL dst, immL16 src) %{
6813   match(Set dst (CMoveL (Binary cmp icc) (Binary dst src)));
6814   predicate(_kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::eq ||
6815             _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ne ||
6816             _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::lt ||
6817             _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ge);
6818   ins_cost(140);
6819 
6820   size(8);
6821   format %{ "MOV$cmp  $dst.lo,$src\t! long\n\t"
6822             "MOV$cmp  $dst.hi,0" %}
6823   ins_encode %{
6824     __ movw($dst$$Register, $src$$constant, (AsmCondition)($cmp$$cmpcode));
6825     __ movw($dst$$Register->successor(), 0, (AsmCondition)($cmp$$cmpcode));
6826   %}
6827   ins_pipe(ialu_imm);
6828 %}
6829 
6830 instruct cmovLIu_reg(cmpOpU cmp, flagsRegU icc, iRegL dst, iRegL src) %{
6831   match(Set dst (CMoveL (Binary cmp icc) (Binary dst src)));
6832   ins_cost(150);
6833 
6834   size(8);
6835   format %{ "MOV$cmp  $dst.lo,$src.lo\t! long\n\t"
6836             "MOV$cmp  $dst.hi,$src.hi" %}
6837   ins_encode %{
6838     __ mov($dst$$Register, $src$$Register, (AsmCondition)($cmp$$cmpcode));
6839     __ mov($dst$$Register->successor(), $src$$Register->successor(), (AsmCondition)($cmp$$cmpcode));
6840   %}
6841   ins_pipe(ialu_reg);
6842 %}
6843 #endif // !AARCH64
6844 
6845 
6846 //----------OS and Locking Instructions----------------------------------------
6847 
6848 // This name is KNOWN by the ADLC and cannot be changed.
6849 // The ADLC forces a 'TypeRawPtr::BOTTOM' output type
6850 // for this guy.
6851 instruct tlsLoadP(RthreadRegP dst) %{
6852   match(Set dst (ThreadLocal));
6853 
6854   size(0);
6855   ins_cost(0);
6856   format %{ "! TLS is in $dst" %}
6857   ins_encode( /*empty encoding*/ );
6858   ins_pipe(ialu_none);
6859 %}
6860 
6861 instruct checkCastPP( iRegP dst ) %{
6862   match(Set dst (CheckCastPP dst));
6863 
6864   size(0);
6865   format %{ "! checkcastPP of $dst" %}
6866   ins_encode( /*empty encoding*/ );
6867   ins_pipe(empty);
6868 %}
6869 
6870 
6871 instruct castPP( iRegP dst ) %{
6872   match(Set dst (CastPP dst));
6873   format %{ "! castPP of $dst" %}
6874   ins_encode( /*empty encoding*/ );
6875   ins_pipe(empty);
6876 %}
6877 
6878 instruct castII( iRegI dst ) %{
6879   match(Set dst (CastII dst));
6880   format %{ "! castII of $dst" %}
6881   ins_encode( /*empty encoding*/ );
6882   ins_cost(0);
6883   ins_pipe(empty);
6884 %}
6885 
6886 //----------Arithmetic Instructions--------------------------------------------
6887 // Addition Instructions
6888 // Register Addition
6889 instruct addI_reg_reg(iRegI dst, iRegI src1, iRegI src2) %{
6890   match(Set dst (AddI src1 src2));
6891 
6892   size(4);
6893   format %{ "add_32 $dst,$src1,$src2\t! int" %}
6894   ins_encode %{
6895     __ add_32($dst$$Register, $src1$$Register, $src2$$Register);
6896   %}
6897   ins_pipe(ialu_reg_reg);
6898 %}
6899 
6900 #ifndef AARCH64
6901 instruct addshlI_reg_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3) %{
6902   match(Set dst (AddI (LShiftI src1 src2) src3));
6903 
6904   size(4);
6905   format %{ "add_32 $dst,$src3,$src1<<$src2\t! int" %}
6906   ins_encode %{
6907     __ add_32($dst$$Register, $src3$$Register, AsmOperand($src1$$Register, lsl, $src2$$Register));
6908   %}
6909   ins_pipe(ialu_reg_reg);
6910 %}
6911 #endif
6912 
6913 #ifdef AARCH64
6914 #ifdef TODO
6915 instruct addshlL_reg_imm_reg(iRegL dst, iRegL src1, immU6 src2, iRegL src3) %{
6916   match(Set dst (AddL (LShiftL src1 src2) src3));
6917 
6918   size(4);
6919   format %{ "ADD    $dst,$src3,$src1<<$src2\t! long" %}
6920   ins_encode %{
6921     __ add($dst$$Register, $src3$$Register, AsmOperand($src1$$Register, lsl, $src2$$constant));
6922   %}
6923   ins_pipe(ialu_reg_reg);
6924 %}
6925 #endif
6926 #endif
6927 
6928 instruct addshlI_reg_imm_reg(iRegI dst, iRegI src1, immU5 src2, iRegI src3) %{
6929   match(Set dst (AddI (LShiftI src1 src2) src3));
6930 
6931   size(4);
6932   format %{ "add_32 $dst,$src3,$src1<<$src2\t! int" %}
6933   ins_encode %{
6934     __ add_32($dst$$Register, $src3$$Register, AsmOperand($src1$$Register, lsl, $src2$$constant));
6935   %}
6936   ins_pipe(ialu_reg_reg);
6937 %}
6938 
6939 #ifndef AARCH64
6940 instruct addsarI_reg_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3) %{
6941   match(Set dst (AddI (RShiftI src1 src2) src3));
6942 
6943   size(4);
6944   format %{ "add_32 $dst,$src3,$src1>>$src2\t! int" %}
6945   ins_encode %{
6946     __ add_32($dst$$Register, $src3$$Register, AsmOperand($src1$$Register, asr, $src2$$Register));
6947   %}
6948   ins_pipe(ialu_reg_reg);
6949 %}
6950 #endif
6951 
6952 instruct addsarI_reg_imm_reg(iRegI dst, iRegI src1, immU5 src2, iRegI src3) %{
6953   match(Set dst (AddI (RShiftI src1 src2) src3));
6954 
6955   size(4);
6956   format %{ "add_32 $dst,$src3,$src1>>$src2\t! int" %}
6957   ins_encode %{
6958     __ add_32($dst$$Register, $src3$$Register, AsmOperand($src1$$Register, asr, $src2$$constant));
6959   %}
6960   ins_pipe(ialu_reg_reg);
6961 %}
6962 
6963 #ifndef AARCH64
6964 instruct addshrI_reg_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3) %{
6965   match(Set dst (AddI (URShiftI src1 src2) src3));
6966 
6967   size(4);
6968   format %{ "add_32 $dst,$src3,$src1>>>$src2\t! int" %}
6969   ins_encode %{
6970     __ add_32($dst$$Register, $src3$$Register, AsmOperand($src1$$Register, lsr, $src2$$Register));
6971   %}
6972   ins_pipe(ialu_reg_reg);
6973 %}
6974 #endif
6975 
6976 instruct addshrI_reg_imm_reg(iRegI dst, iRegI src1, immU5 src2, iRegI src3) %{
6977   match(Set dst (AddI (URShiftI src1 src2) src3));
6978 
6979   size(4);
6980   format %{ "add_32 $dst,$src3,$src1>>>$src2\t! int" %}
6981   ins_encode %{
6982     __ add_32($dst$$Register, $src3$$Register, AsmOperand($src1$$Register, lsr, $src2$$constant));
6983   %}
6984   ins_pipe(ialu_reg_reg);
6985 %}
6986 
6987 // Immediate Addition
6988 instruct addI_reg_aimmI(iRegI dst, iRegI src1, aimmI src2) %{
6989   match(Set dst (AddI src1 src2));
6990 
6991   size(4);
6992   format %{ "add_32 $dst,$src1,$src2\t! int" %}
6993   ins_encode %{
6994     __ add_32($dst$$Register, $src1$$Register, $src2$$constant);
6995   %}
6996   ins_pipe(ialu_reg_imm);
6997 %}
6998 
6999 // Pointer Register Addition
7000 instruct addP_reg_reg(iRegP dst, iRegP src1, iRegX src2) %{
7001   match(Set dst (AddP src1 src2));
7002 
7003   size(4);
7004   format %{ "ADD    $dst,$src1,$src2\t! ptr" %}
7005   ins_encode %{
7006     __ add($dst$$Register, $src1$$Register, $src2$$Register);
7007   %}
7008   ins_pipe(ialu_reg_reg);
7009 %}
7010 
7011 #ifdef AARCH64
7012 // unshifted I2L operand
7013 operand unshiftedI2L(iRegI src2) %{
7014 //constraint(ALLOC_IN_RC(sp_ptr_reg));
7015   match(ConvI2L src2);
7016 
7017   op_cost(1);
7018   format %{ "$src2.w" %}
7019   interface(MEMORY_INTER) %{
7020     base($src2);
7021     index(0xff);
7022     scale(0x0);
7023     disp(0x0);
7024   %}
7025 %}
7026 
7027 // shifted I2L operand
7028 operand shiftedI2L(iRegI src2, immI_0_4 src3) %{
7029 //constraint(ALLOC_IN_RC(sp_ptr_reg));
7030   match(LShiftX (ConvI2L src2) src3);
7031 
7032   op_cost(1);
7033   format %{ "$src2.w << $src3" %}
7034   interface(MEMORY_INTER) %{
7035     base($src2);
7036     index(0xff);
7037     scale($src3);
7038     disp(0x0);
7039   %}
7040 %}
7041 
7042 opclass shiftedRegI(shiftedI2L, unshiftedI2L);
7043 
7044 instruct shlL_reg_regI(iRegL dst, iRegI src1, immU6 src2) %{
7045   match(Set dst (LShiftL (ConvI2L src1) src2));
7046 
7047   size(4);
7048   format %{ "LSL    $dst,$src1.w,$src2\t! ptr" %}
7049   ins_encode %{
7050     int c = $src2$$constant;
7051     int r = 64 - c;
7052     int s = 31;
7053     if (s >= r) {
7054       s = r - 1;
7055     }
7056     __ sbfm($dst$$Register, $src1$$Register, r, s);
7057   %}
7058   ins_pipe(ialu_reg_reg);
7059 %}
7060 
7061 instruct addP_reg_regI(iRegP dst, iRegP src1, shiftedRegI src2) %{
7062   match(Set dst (AddP src1 src2));
7063 
7064   ins_cost(DEFAULT_COST * 3/2);
7065   size(4);
7066   format %{ "ADD    $dst,$src1,$src2, sxtw\t! ptr" %}
7067   ins_encode %{
7068     Register base = reg_to_register_object($src2$$base);
7069     __ add($dst$$Register, $src1$$Register, base, ex_sxtw, $src2$$scale);
7070   %}
7071   ins_pipe(ialu_reg_reg);
7072 %}
7073 #endif
7074 
7075 // shifted iRegX operand
7076 operand shiftedX(iRegX src2, shimmX src3) %{
7077 //constraint(ALLOC_IN_RC(sp_ptr_reg));
7078   match(LShiftX src2 src3);
7079 
7080   op_cost(1);
7081   format %{ "$src2 << $src3" %}
7082   interface(MEMORY_INTER) %{
7083     base($src2);
7084     index(0xff);
7085     scale($src3);
7086     disp(0x0);
7087   %}
7088 %}
7089 
7090 instruct addshlP_reg_reg_imm(iRegP dst, iRegP src1, shiftedX src2) %{
7091   match(Set dst (AddP src1 src2));
7092 
7093   ins_cost(DEFAULT_COST * 3/2);
7094   size(4);
7095   format %{ "ADD    $dst,$src1,$src2\t! ptr" %}
7096   ins_encode %{
7097     Register base = reg_to_register_object($src2$$base);
7098     __ add($dst$$Register, $src1$$Register, AsmOperand(base, lsl, $src2$$scale));
7099   %}
7100   ins_pipe(ialu_reg_reg);
7101 %}
7102 
7103 // Pointer Immediate Addition
7104 instruct addP_reg_aimmX(iRegP dst, iRegP src1, aimmX src2) %{
7105   match(Set dst (AddP src1 src2));
7106 
7107   size(4);
7108   format %{ "ADD    $dst,$src1,$src2\t! ptr" %}
7109   ins_encode %{
7110     __ add($dst$$Register, $src1$$Register, $src2$$constant);
7111   %}
7112   ins_pipe(ialu_reg_imm);
7113 %}
7114 
7115 // Long Addition
7116 #ifdef AARCH64
7117 instruct addL_reg_reg(iRegL dst, iRegL src1, iRegL src2) %{
7118   match(Set dst (AddL src1 src2));
7119   size(4);
7120   format %{ "ADD     $dst,$src1,$src2\t! long" %}
7121   ins_encode %{
7122     __ add($dst$$Register, $src1$$Register, $src2$$Register);
7123   %}
7124   ins_pipe(ialu_reg_reg);
7125 %}
7126 
7127 instruct addL_reg_regI(iRegL dst, iRegL src1, shiftedRegI src2) %{
7128   match(Set dst (AddL src1 src2));
7129 
7130   ins_cost(DEFAULT_COST * 3/2);
7131   size(4);
7132   format %{ "ADD    $dst,$src1,$src2, sxtw\t! long" %}
7133   ins_encode %{
7134     Register base = reg_to_register_object($src2$$base);
7135     __ add($dst$$Register, $src1$$Register, base, ex_sxtw, $src2$$scale);
7136   %}
7137   ins_pipe(ialu_reg_reg);
7138 %}
7139 #else
7140 instruct addL_reg_reg(iRegL dst, iRegL src1, iRegL src2, flagsReg ccr) %{
7141   match(Set dst (AddL src1 src2));
7142   effect(KILL ccr);
7143   size(8);
7144   format %{ "ADDS    $dst.lo,$src1.lo,$src2.lo\t! long\n\t"
7145             "ADC     $dst.hi,$src1.hi,$src2.hi" %}
7146   ins_encode %{
7147     __ adds($dst$$Register, $src1$$Register, $src2$$Register);
7148     __ adc($dst$$Register->successor(), $src1$$Register->successor(), $src2$$Register->successor());
7149   %}
7150   ins_pipe(ialu_reg_reg);
7151 %}
7152 #endif
7153 
7154 #ifdef AARCH64
7155 // Immediate Addition
7156 instruct addL_reg_aimm(iRegL dst, iRegL src1, aimmL src2) %{
7157   match(Set dst (AddL src1 src2));
7158 
7159   size(4);
7160   format %{ "ADD    $dst,$src1,$src2\t! long" %}
7161   ins_encode %{
7162     __ add($dst$$Register, $src1$$Register, $src2$$constant);
7163   %}
7164   ins_pipe(ialu_reg_imm);
7165 %}
7166 
7167 instruct addL_reg_immLneg(iRegL dst, iRegL src1, aimmLneg src2) %{
7168   match(Set dst (SubL src1 src2));
7169 
7170   size(4);
7171   format %{ "ADD    $dst,$src1,-($src2)\t! long" %}
7172   ins_encode %{
7173     __ add($dst$$Register, $src1$$Register, -$src2$$constant);
7174   %}
7175   ins_pipe(ialu_reg_imm);
7176 %}
7177 #else
7178 // TODO
7179 #endif
7180 
7181 #ifndef AARCH64
7182 // TODO: try immLRot2 instead, (0, $con$$constant) becomes
7183 // (hi($con$$constant), lo($con$$constant)) becomes
7184 instruct addL_reg_immRot(iRegL dst, iRegL src1, immLlowRot con, flagsReg ccr) %{
7185   match(Set dst (AddL src1 con));
7186   effect(KILL ccr);
7187   size(8);
7188   format %{ "ADDS    $dst.lo,$src1.lo,$con\t! long\n\t"
7189             "ADC     $dst.hi,$src1.hi,0" %}
7190   ins_encode %{
7191     __ adds($dst$$Register, $src1$$Register, $con$$constant);
7192     __ adc($dst$$Register->successor(), $src1$$Register->successor(), 0);
7193   %}
7194   ins_pipe(ialu_reg_imm);
7195 %}
7196 #endif
7197 
7198 //----------Conditional_store--------------------------------------------------
7199 // Conditional-store of the updated heap-top.
7200 // Used during allocation of the shared heap.
7201 // Sets flags (EQ) on success.
7202 
7203 // TODO: optimize out barriers with AArch64 load-acquire/store-release
7204 // LoadP-locked.
7205 instruct loadPLocked(iRegP dst, memoryex mem) %{
7206   match(Set dst (LoadPLocked mem));
7207   size(4);
7208   format %{ "LDREX  $dst,$mem" %}
7209   ins_encode %{
7210 #ifdef AARCH64
7211     Register base = reg_to_register_object($mem$$base);
7212     __ ldxr($dst$$Register, base);
7213 #else
7214     __ ldrex($dst$$Register,$mem$$Address);
7215 #endif
7216   %}
7217   ins_pipe(iload_mem);
7218 %}
7219 
7220 instruct storePConditional( memoryex heap_top_ptr, iRegP oldval, iRegP newval, iRegI tmp, flagsRegP pcc ) %{
7221   predicate(_kids[1]->_kids[0]->_leaf->Opcode() == Op_LoadPLocked); // only works in conjunction with a LoadPLocked node
7222   match(Set pcc (StorePConditional heap_top_ptr (Binary oldval newval)));
7223   effect( TEMP tmp );
7224   size(8);
7225   format %{ "STREX  $tmp,$newval,$heap_top_ptr\n\t"
7226             "CMP    $tmp, 0" %}
7227   ins_encode %{
7228 #ifdef AARCH64
7229     Register base = reg_to_register_object($heap_top_ptr$$base);
7230     __ stxr($tmp$$Register, $newval$$Register, base);
7231 #else
7232     __ strex($tmp$$Register, $newval$$Register, $heap_top_ptr$$Address);
7233 #endif
7234     __ cmp($tmp$$Register, 0);
7235   %}
7236   ins_pipe( long_memory_op );
7237 %}
7238 
7239 // Conditional-store of an intx value.
7240 instruct storeXConditional( memoryex mem, iRegX oldval, iRegX newval, iRegX tmp, flagsReg icc ) %{
7241 #ifdef AARCH64
7242   match(Set icc (StoreLConditional mem (Binary oldval newval)));
7243   effect( TEMP tmp );
7244   size(28);
7245   format %{ "loop:\n\t"
7246             "LDXR     $tmp, $mem\t! If $oldval==[$mem] Then store $newval into [$mem], DOESN'T set $newval=[$mem] in any case\n\t"
7247             "SUBS     $tmp, $tmp, $oldval\n\t"
7248             "B.ne     done\n\t"
7249             "STXR     $tmp, $newval, $mem\n\t"
7250             "CBNZ_w   $tmp, loop\n\t"
7251             "CMP      $tmp, 0\n\t"
7252             "done:\n\t"
7253             "membar   LoadStore|LoadLoad" %}
7254 #else
7255   match(Set icc (StoreIConditional mem (Binary oldval newval)));
7256   effect( TEMP tmp );
7257   size(28);
7258   format %{ "loop: \n\t"
7259             "LDREX    $tmp, $mem\t! If $oldval==[$mem] Then store $newval into [$mem], DOESN'T set $newval=[$mem] in any case\n\t"
7260             "XORS     $tmp,$tmp, $oldval\n\t"
7261             "STREX.eq $tmp, $newval, $mem\n\t"
7262             "CMP.eq   $tmp, 1 \n\t"
7263             "B.eq     loop \n\t"
7264             "TEQ      $tmp, 0\n\t"
7265             "membar   LoadStore|LoadLoad" %}
7266 #endif
7267   ins_encode %{
7268     Label loop;
7269     __ bind(loop);
7270 #ifdef AARCH64
7271 // FIXME: use load-acquire/store-release, remove membar?
7272     Label done;
7273     Register base = reg_to_register_object($mem$$base);
7274     __ ldxr($tmp$$Register, base);
7275     __ subs($tmp$$Register, $tmp$$Register, $oldval$$Register);
7276     __ b(done, ne);
7277     __ stxr($tmp$$Register, $newval$$Register, base);
7278     __ cbnz_w($tmp$$Register, loop);
7279     __ cmp($tmp$$Register, 0);
7280     __ bind(done);
7281 #else
7282     __ ldrex($tmp$$Register, $mem$$Address);
7283     __ eors($tmp$$Register, $tmp$$Register, $oldval$$Register);
7284     __ strex($tmp$$Register, $newval$$Register, $mem$$Address, eq);
7285     __ cmp($tmp$$Register, 1, eq);
7286     __ b(loop, eq);
7287     __ teq($tmp$$Register, 0);
7288 #endif
7289     // used by biased locking only. Requires a membar.
7290     __ membar(MacroAssembler::Membar_mask_bits(MacroAssembler::LoadStore | MacroAssembler::LoadLoad), noreg);
7291   %}
7292   ins_pipe( long_memory_op );
7293 %}
7294 
7295 // No flag versions for CompareAndSwap{P,I,L} because matcher can't match them
7296 
7297 #ifdef AARCH64
7298 // TODO: if combined with membar, elide membar and use
7299 // load-acquire/store-release if appropriate
7300 instruct compareAndSwapL_bool(memoryex mem, iRegL oldval, iRegL newval, iRegI res, iRegI tmp, flagsReg ccr) %{
7301   match(Set res (CompareAndSwapL mem (Binary oldval newval)));
7302   effect( KILL ccr, TEMP tmp);
7303   size(24);
7304   format %{ "loop:\n\t"
7305             "LDXR     $tmp, $mem\t! If $oldval==[$mem] Then store $newval into [$mem]\n\t"
7306             "CMP      $tmp, $oldval\n\t"
7307             "B.ne     done\n\t"
7308             "STXR     $tmp, $newval, $mem\n\t"
7309             "CBNZ_w   $tmp, loop\n\t"
7310             "done:\n\t"
7311             "CSET_w   $res, eq" %}
7312   ins_encode %{
7313     Register base = reg_to_register_object($mem$$base);
7314     Label loop, done;
7315     __ bind(loop);
7316     __ ldxr($tmp$$Register, base);
7317     __ cmp($tmp$$Register, $oldval$$Register);
7318     __ b(done, ne);
7319     __ stxr($tmp$$Register, $newval$$Register, base);
7320     __ cbnz_w($tmp$$Register, loop);
7321     __ bind(done);
7322     __ cset_w($res$$Register, eq);
7323   %}
7324   ins_pipe( long_memory_op );
7325 %}
7326 
7327 instruct compareAndSwapI_bool(memoryex mem, iRegI oldval, iRegI newval, iRegI res, iRegI tmp, flagsReg ccr) %{
7328   match(Set res (CompareAndSwapI mem (Binary oldval newval)));
7329   effect( KILL ccr, TEMP tmp);
7330   size(24);
7331   format %{ "loop:\n\t"
7332             "LDXR_w   $tmp, $mem\t! If $oldval==[$mem] Then store $newval into [$mem]\n\t"
7333             "CMP_w    $tmp, $oldval\n\t"
7334             "B.ne     done\n\t"
7335             "STXR_w   $tmp, $newval, $mem\n\t"
7336             "CBNZ_w   $tmp, loop\n\t"
7337             "done:\n\t"
7338             "CSET_w   $res, eq" %}
7339   ins_encode %{
7340     Register base = reg_to_register_object($mem$$base);
7341     Label loop, done;
7342     __ bind(loop);
7343     __ ldxr_w($tmp$$Register, base);
7344     __ cmp_w($tmp$$Register, $oldval$$Register);
7345     __ b(done, ne);
7346     __ stxr_w($tmp$$Register, $newval$$Register,  base);
7347     __ cbnz_w($tmp$$Register, loop);
7348     __ bind(done);
7349     __ cset_w($res$$Register, eq);
7350   %}
7351   ins_pipe( long_memory_op );
7352 %}
7353 
7354 // tmp must use iRegI instead of iRegN until 8051805 is fixed.
7355 instruct compareAndSwapN_bool(memoryex mem, iRegN oldval, iRegN newval, iRegI res, iRegI tmp, flagsReg ccr) %{
7356   match(Set res (CompareAndSwapN mem (Binary oldval newval)));
7357   effect( KILL ccr, TEMP tmp);
7358   size(24);
7359   format %{ "loop:\n\t"
7360             "LDXR_w   $tmp, $mem\t! If $oldval==[$mem] Then store $newval into [$mem]\n\t"
7361             "CMP_w    $tmp, $oldval\n\t"
7362             "B.ne     done\n\t"
7363             "STXR_w   $tmp, $newval, $mem\n\t"
7364             "CBNZ_w   $tmp, loop\n\t"
7365             "done:\n\t"
7366             "CSET_w   $res, eq" %}
7367   ins_encode %{
7368     Register base = reg_to_register_object($mem$$base);
7369     Label loop, done;
7370     __ bind(loop);
7371     __ ldxr_w($tmp$$Register, base);
7372     __ cmp_w($tmp$$Register, $oldval$$Register);
7373     __ b(done, ne);
7374     __ stxr_w($tmp$$Register, $newval$$Register,  base);
7375     __ cbnz_w($tmp$$Register, loop);
7376     __ bind(done);
7377     __ cset_w($res$$Register, eq);
7378   %}
7379   ins_pipe( long_memory_op );
7380 %}
7381 
7382 instruct compareAndSwapP_bool(memoryex mem, iRegP oldval, iRegP newval, iRegI res, iRegI tmp, flagsReg ccr) %{
7383   match(Set res (CompareAndSwapP mem (Binary oldval newval)));
7384   effect( KILL ccr, TEMP tmp);
7385   size(24);
7386   format %{ "loop:\n\t"
7387             "LDXR     $tmp, $mem\t! If $oldval==[$mem] Then store $newval into [$mem]\n\t"
7388             "CMP      $tmp, $oldval\n\t"
7389             "B.ne     done\n\t"
7390             "STXR     $tmp, $newval, $mem\n\t"
7391             "CBNZ_w   $tmp, loop\n\t"
7392             "done:\n\t"
7393             "CSET_w   $res, eq" %}
7394   ins_encode %{
7395     Register base = reg_to_register_object($mem$$base);
7396     Label loop, done;
7397     __ bind(loop);
7398     __ ldxr($tmp$$Register, base);
7399     __ cmp($tmp$$Register, $oldval$$Register);
7400     __ b(done, ne);
7401     __ stxr($tmp$$Register, $newval$$Register,  base);
7402     __ cbnz_w($tmp$$Register, loop);
7403     __ bind(done);
7404     __ cset_w($res$$Register, eq);
7405   %}
7406   ins_pipe( long_memory_op );
7407 %}
7408 #else // !AARCH64
7409 instruct compareAndSwapL_bool(memoryex mem, iRegL oldval, iRegLd newval, iRegI res, iRegLd tmp, flagsReg ccr ) %{
7410   match(Set res (CompareAndSwapL mem (Binary oldval newval)));
7411   effect( KILL ccr, TEMP tmp);
7412   size(32);
7413   format %{ "loop: \n\t"
7414             "LDREXD   $tmp, $mem\t! If $oldval==[$mem] Then store $newval into [$mem]\n\t"
7415             "CMP      $tmp.lo, $oldval.lo\n\t"
7416             "CMP.eq   $tmp.hi, $oldval.hi\n\t"
7417             "STREXD.eq $tmp, $newval, $mem\n\t"
7418             "MOV.ne   $tmp, 0 \n\t"
7419             "XORS.eq  $tmp,$tmp, 1 \n\t"
7420             "B.eq     loop \n\t"
7421             "MOV      $res, $tmp" %}
7422   ins_encode %{
7423     Label loop;
7424     __ bind(loop);
7425     __ ldrexd($tmp$$Register, $mem$$Address);
7426     __ cmp($tmp$$Register, $oldval$$Register);
7427     __ cmp($tmp$$Register->successor(), $oldval$$Register->successor(), eq);
7428     __ strexd($tmp$$Register, $newval$$Register, $mem$$Address, eq);
7429     __ mov($tmp$$Register, 0, ne);
7430     __ eors($tmp$$Register, $tmp$$Register, 1, eq);
7431     __ b(loop, eq);
7432     __ mov($res$$Register, $tmp$$Register);
7433   %}
7434   ins_pipe( long_memory_op );
7435 %}
7436 
7437 
7438 instruct compareAndSwapI_bool(memoryex mem, iRegI oldval, iRegI newval, iRegI res, iRegI tmp, flagsReg ccr ) %{
7439   match(Set res (CompareAndSwapI mem (Binary oldval newval)));
7440   effect( KILL ccr, TEMP tmp);
7441   size(28);
7442   format %{ "loop: \n\t"
7443             "LDREX    $tmp, $mem\t! If $oldval==[$mem] Then store $newval into [$mem]\n\t"
7444             "CMP      $tmp, $oldval\n\t"
7445             "STREX.eq $tmp, $newval, $mem\n\t"
7446             "MOV.ne   $tmp, 0 \n\t"
7447             "XORS.eq  $tmp,$tmp, 1 \n\t"
7448             "B.eq     loop \n\t"
7449             "MOV      $res, $tmp" %}
7450 
7451   ins_encode %{
7452     Label loop;
7453     __ bind(loop);
7454     __ ldrex($tmp$$Register,$mem$$Address);
7455     __ cmp($tmp$$Register, $oldval$$Register);
7456     __ strex($tmp$$Register, $newval$$Register, $mem$$Address, eq);
7457     __ mov($tmp$$Register, 0, ne);
7458     __ eors($tmp$$Register, $tmp$$Register, 1, eq);
7459     __ b(loop, eq);
7460     __ mov($res$$Register, $tmp$$Register);
7461   %}
7462   ins_pipe( long_memory_op );
7463 %}
7464 
7465 instruct compareAndSwapP_bool(memoryex mem, iRegP oldval, iRegP newval, iRegI res, iRegI tmp, flagsReg ccr ) %{
7466   match(Set res (CompareAndSwapP mem (Binary oldval newval)));
7467   effect( KILL ccr, TEMP tmp);
7468   size(28);
7469   format %{ "loop: \n\t"
7470             "LDREX    $tmp, $mem\t! If $oldval==[$mem] Then store $newval into [$mem]\n\t"
7471             "CMP      $tmp, $oldval\n\t"
7472             "STREX.eq $tmp, $newval, $mem\n\t"
7473             "MOV.ne   $tmp, 0 \n\t"
7474             "EORS.eq  $tmp,$tmp, 1 \n\t"
7475             "B.eq     loop \n\t"
7476             "MOV      $res, $tmp" %}
7477 
7478   ins_encode %{
7479     Label loop;
7480     __ bind(loop);
7481     __ ldrex($tmp$$Register,$mem$$Address);
7482     __ cmp($tmp$$Register, $oldval$$Register);
7483     __ strex($tmp$$Register, $newval$$Register, $mem$$Address, eq);
7484     __ mov($tmp$$Register, 0, ne);
7485     __ eors($tmp$$Register, $tmp$$Register, 1, eq);
7486     __ b(loop, eq);
7487     __ mov($res$$Register, $tmp$$Register);
7488   %}
7489   ins_pipe( long_memory_op );
7490 %}
7491 #endif // !AARCH64
7492 
7493 #ifdef AARCH64
7494 instruct xaddI_aimmI_no_res(memoryex mem, aimmI add, Universe dummy, iRegI tmp1, iRegI tmp2) %{
7495   predicate(n->as_LoadStore()->result_not_used());
7496   match(Set dummy (GetAndAddI mem add));
7497   effect(TEMP tmp1, TEMP tmp2);
7498   size(16);
7499   format %{ "loop:\n\t"
7500             "LDXR_w   $tmp1, $mem\n\t"
7501             "ADD_w    $tmp1, $tmp1, $add\n\t"
7502             "STXR_w   $tmp2, $tmp1, $mem\n\t"
7503             "CBNZ_w   $tmp2, loop" %}
7504 
7505   ins_encode %{
7506     Label loop;
7507     Register base = reg_to_register_object($mem$$base);
7508     __ bind(loop);
7509     __ ldxr_w($tmp1$$Register, base);
7510     __ add_w($tmp1$$Register, $tmp1$$Register, $add$$constant);
7511     __ stxr_w($tmp2$$Register, $tmp1$$Register, base);
7512     __ cbnz_w($tmp2$$Register, loop);
7513   %}
7514   ins_pipe( long_memory_op );
7515 %}
7516 #else
7517 instruct xaddI_aimmI_no_res(memoryex mem, aimmI add, Universe dummy, iRegI tmp1, iRegI tmp2, flagsReg ccr) %{
7518   predicate(n->as_LoadStore()->result_not_used());
7519   match(Set dummy (GetAndAddI mem add));
7520   effect(KILL ccr, TEMP tmp1, TEMP tmp2);
7521   size(20);
7522   format %{ "loop: \n\t"
7523             "LDREX    $tmp1, $mem\n\t"
7524             "ADD      $tmp1, $tmp1, $add\n\t"
7525             "STREX    $tmp2, $tmp1, $mem\n\t"
7526             "CMP      $tmp2, 0 \n\t"
7527             "B.ne     loop \n\t" %}
7528 
7529   ins_encode %{
7530     Label loop;
7531     __ bind(loop);
7532     __ ldrex($tmp1$$Register,$mem$$Address);
7533     __ add($tmp1$$Register, $tmp1$$Register, $add$$constant);
7534     __ strex($tmp2$$Register, $tmp1$$Register, $mem$$Address);
7535     __ cmp($tmp2$$Register, 0);
7536     __ b(loop, ne);
7537   %}
7538   ins_pipe( long_memory_op );
7539 %}
7540 #endif
7541 
7542 #ifdef AARCH64
7543 instruct xaddI_reg_no_res(memoryex mem, iRegI add, Universe dummy, iRegI tmp1, iRegI tmp2) %{
7544   predicate(n->as_LoadStore()->result_not_used());
7545   match(Set dummy (GetAndAddI mem add));
7546   effect(TEMP tmp1, TEMP tmp2);
7547   size(16);
7548   format %{ "loop:\n\t"
7549             "LDXR_w   $tmp1, $mem\n\t"
7550             "ADD_w    $tmp1, $tmp1, $add\n\t"
7551             "STXR_w   $tmp2, $tmp1, $mem\n\t"
7552             "CBNZ_w   $tmp2, loop" %}
7553 
7554   ins_encode %{
7555     Label loop;
7556     Register base = reg_to_register_object($mem$$base);
7557     __ bind(loop);
7558     __ ldxr_w($tmp1$$Register, base);
7559     __ add_w($tmp1$$Register, $tmp1$$Register, $add$$Register);
7560     __ stxr_w($tmp2$$Register, $tmp1$$Register, base);
7561     __ cbnz_w($tmp2$$Register, loop);
7562   %}
7563   ins_pipe( long_memory_op );
7564 %}
7565 #else
7566 instruct xaddI_reg_no_res(memoryex mem, iRegI add, Universe dummy, iRegI tmp1, iRegI tmp2, flagsReg ccr) %{
7567   predicate(n->as_LoadStore()->result_not_used());
7568   match(Set dummy (GetAndAddI mem add));
7569   effect(KILL ccr, TEMP tmp1, TEMP tmp2);
7570   size(20);
7571   format %{ "loop: \n\t"
7572             "LDREX    $tmp1, $mem\n\t"
7573             "ADD      $tmp1, $tmp1, $add\n\t"
7574             "STREX    $tmp2, $tmp1, $mem\n\t"
7575             "CMP      $tmp2, 0 \n\t"
7576             "B.ne     loop \n\t" %}
7577 
7578   ins_encode %{
7579     Label loop;
7580     __ bind(loop);
7581     __ ldrex($tmp1$$Register,$mem$$Address);
7582     __ add($tmp1$$Register, $tmp1$$Register, $add$$Register);
7583     __ strex($tmp2$$Register, $tmp1$$Register, $mem$$Address);
7584     __ cmp($tmp2$$Register, 0);
7585     __ b(loop, ne);
7586   %}
7587   ins_pipe( long_memory_op );
7588 %}
7589 #endif
7590 
7591 #ifdef AARCH64
7592 instruct xaddI_aimmI(memoryex mem, aimmI add, iRegI res, iRegI tmp1, iRegI tmp2) %{
7593   match(Set res (GetAndAddI mem add));
7594   effect(TEMP tmp1, TEMP tmp2, TEMP res);
7595   size(16);
7596   format %{ "loop:\n\t"
7597             "LDXR_w   $res, $mem\n\t"
7598             "ADD_w    $tmp1, $res, $add\n\t"
7599             "STXR_w   $tmp2, $tmp1, $mem\n\t"
7600             "CBNZ_w   $tmp2, loop" %}
7601 
7602   ins_encode %{
7603     Label loop;
7604     Register base = reg_to_register_object($mem$$base);
7605     __ bind(loop);
7606     __ ldxr_w($res$$Register, base);
7607     __ add_w($tmp1$$Register, $res$$Register, $add$$constant);
7608     __ stxr_w($tmp2$$Register, $tmp1$$Register, base);
7609     __ cbnz_w($tmp2$$Register, loop);
7610   %}
7611   ins_pipe( long_memory_op );
7612 %}
7613 #else
7614 instruct xaddI_aimmI(memoryex mem, aimmI add, iRegI res, iRegI tmp1, iRegI tmp2, flagsReg ccr) %{
7615   match(Set res (GetAndAddI mem add));
7616   effect(KILL ccr, TEMP tmp1, TEMP tmp2, TEMP res);
7617   size(20);
7618   format %{ "loop: \n\t"
7619             "LDREX    $res, $mem\n\t"
7620             "ADD      $tmp1, $res, $add\n\t"
7621             "STREX    $tmp2, $tmp1, $mem\n\t"
7622             "CMP      $tmp2, 0 \n\t"
7623             "B.ne     loop \n\t" %}
7624 
7625   ins_encode %{
7626     Label loop;
7627     __ bind(loop);
7628     __ ldrex($res$$Register,$mem$$Address);
7629     __ add($tmp1$$Register, $res$$Register, $add$$constant);
7630     __ strex($tmp2$$Register, $tmp1$$Register, $mem$$Address);
7631     __ cmp($tmp2$$Register, 0);
7632     __ b(loop, ne);
7633   %}
7634   ins_pipe( long_memory_op );
7635 %}
7636 #endif
7637 
7638 #ifdef AARCH64
7639 instruct xaddI_reg(memoryex mem, iRegI add, iRegI res, iRegI tmp1, iRegI tmp2) %{
7640   match(Set res (GetAndAddI mem add));
7641   effect(TEMP tmp1, TEMP tmp2, TEMP res);
7642   size(16);
7643   format %{ "loop:\n\t"
7644             "LDXR_w   $res, $mem\n\t"
7645             "ADD_w    $tmp1, $res, $add\n\t"
7646             "STXR_w   $tmp2, $tmp1, $mem\n\t"
7647             "CBNZ_w   $tmp2, loop" %}
7648 
7649   ins_encode %{
7650     Label loop;
7651     Register base = reg_to_register_object($mem$$base);
7652     __ bind(loop);
7653     __ ldxr_w($res$$Register, base);
7654     __ add_w($tmp1$$Register, $res$$Register, $add$$Register);
7655     __ stxr_w($tmp2$$Register, $tmp1$$Register, base);
7656     __ cbnz_w($tmp2$$Register, loop);
7657   %}
7658   ins_pipe( long_memory_op );
7659 %}
7660 #else
7661 instruct xaddI_reg(memoryex mem, iRegI add, iRegI res, iRegI tmp1, iRegI tmp2, flagsReg ccr) %{
7662   match(Set res (GetAndAddI mem add));
7663   effect(KILL ccr, TEMP tmp1, TEMP tmp2, TEMP res);
7664   size(20);
7665   format %{ "loop: \n\t"
7666             "LDREX    $res, $mem\n\t"
7667             "ADD      $tmp1, $res, $add\n\t"
7668             "STREX    $tmp2, $tmp1, $mem\n\t"
7669             "CMP      $tmp2, 0 \n\t"
7670             "B.ne     loop \n\t" %}
7671 
7672   ins_encode %{
7673     Label loop;
7674     __ bind(loop);
7675     __ ldrex($res$$Register,$mem$$Address);
7676     __ add($tmp1$$Register, $res$$Register, $add$$Register);
7677     __ strex($tmp2$$Register, $tmp1$$Register, $mem$$Address);
7678     __ cmp($tmp2$$Register, 0);
7679     __ b(loop, ne);
7680   %}
7681   ins_pipe( long_memory_op );
7682 %}
7683 #endif
7684 
7685 #ifdef AARCH64
7686 instruct xaddL_reg_no_res(memoryex mem, iRegL add, Universe dummy, iRegL tmp1, iRegI tmp2) %{
7687   predicate(n->as_LoadStore()->result_not_used());
7688   match(Set dummy (GetAndAddL mem add));
7689   effect(TEMP tmp1, TEMP tmp2);
7690   size(16);
7691   format %{ "loop:\n\t"
7692             "LDXR     $tmp1, $mem\n\t"
7693             "ADD      $tmp1, $tmp1, $add\n\t"
7694             "STXR     $tmp2, $tmp1, $mem\n\t"
7695             "CBNZ_w   $tmp2, loop" %}
7696 
7697   ins_encode %{
7698     Label loop;
7699     Register base = reg_to_register_object($mem$$base);
7700     __ bind(loop);
7701     __ ldxr($tmp1$$Register, base);
7702     __ add($tmp1$$Register, $tmp1$$Register, $add$$Register);
7703     __ stxr($tmp2$$Register, $tmp1$$Register, base);
7704     __ cbnz_w($tmp2$$Register, loop);
7705   %}
7706   ins_pipe( long_memory_op );
7707 %}
7708 #else
7709 instruct xaddL_reg_no_res(memoryex mem, iRegL add, Universe dummy, iRegLd tmp1, iRegI tmp2, flagsReg ccr) %{
7710   predicate(n->as_LoadStore()->result_not_used());
7711   match(Set dummy (GetAndAddL mem add));
7712   effect( KILL ccr, TEMP tmp1, TEMP tmp2);
7713   size(24);
7714   format %{ "loop: \n\t"
7715             "LDREXD   $tmp1, $mem\n\t"
7716             "ADDS     $tmp1.lo, $tmp1.lo, $add.lo\n\t"
7717             "ADC      $tmp1.hi, $tmp1.hi, $add.hi\n\t"
7718             "STREXD   $tmp2, $tmp1, $mem\n\t"
7719             "CMP      $tmp2, 0 \n\t"
7720             "B.ne     loop \n\t" %}
7721 
7722   ins_encode %{
7723     Label loop;
7724     __ bind(loop);
7725     __ ldrexd($tmp1$$Register, $mem$$Address);
7726     __ adds($tmp1$$Register, $tmp1$$Register, $add$$Register);
7727     __ adc($tmp1$$Register->successor(), $tmp1$$Register->successor(), $add$$Register->successor());
7728     __ strexd($tmp2$$Register, $tmp1$$Register, $mem$$Address);
7729     __ cmp($tmp2$$Register, 0);
7730     __ b(loop, ne);
7731   %}
7732   ins_pipe( long_memory_op );
7733 %}
7734 #endif
7735 
7736 #ifdef AARCH64
7737 instruct xaddL_imm_no_res(memoryex mem, aimmL add, Universe dummy, iRegL tmp1, iRegI tmp2) %{
7738   predicate(n->as_LoadStore()->result_not_used());
7739   match(Set dummy (GetAndAddL mem add));
7740   effect(TEMP tmp1, TEMP tmp2);
7741   size(16);
7742   format %{ "loop:\n\t"
7743             "LDXR     $tmp1, $mem\n\t"
7744             "ADD      $tmp1, $tmp1, $add\n\t"
7745             "STXR     $tmp2, $tmp1, $mem\n\t"
7746             "CBNZ_w   $tmp2, loop" %}
7747 
7748   ins_encode %{
7749     Label loop;
7750     Register base = reg_to_register_object($mem$$base);
7751     __ bind(loop);
7752     __ ldxr($tmp1$$Register, base);
7753     __ add($tmp1$$Register, $tmp1$$Register, $add$$constant);
7754     __ stxr($tmp2$$Register, $tmp1$$Register, base);
7755     __ cbnz_w($tmp2$$Register, loop);
7756   %}
7757   ins_pipe( long_memory_op );
7758 %}
7759 #else
7760 // TODO: try immLRot2 instead, (0, $con$$constant) becomes
7761 // (hi($con$$constant), lo($con$$constant)) becomes
7762 instruct xaddL_immRot_no_res(memoryex mem, immLlowRot add, Universe dummy, iRegLd tmp1, iRegI tmp2, flagsReg ccr) %{
7763   predicate(n->as_LoadStore()->result_not_used());
7764   match(Set dummy (GetAndAddL mem add));
7765   effect( KILL ccr, TEMP tmp1, TEMP tmp2);
7766   size(24);
7767   format %{ "loop: \n\t"
7768             "LDREXD   $tmp1, $mem\n\t"
7769             "ADDS     $tmp1.lo, $tmp1.lo, $add\n\t"
7770             "ADC      $tmp1.hi, $tmp1.hi, 0\n\t"
7771             "STREXD   $tmp2, $tmp1, $mem\n\t"
7772             "CMP      $tmp2, 0 \n\t"
7773             "B.ne     loop \n\t" %}
7774 
7775   ins_encode %{
7776     Label loop;
7777     __ bind(loop);
7778     __ ldrexd($tmp1$$Register, $mem$$Address);
7779     __ adds($tmp1$$Register, $tmp1$$Register, $add$$constant);
7780     __ adc($tmp1$$Register->successor(), $tmp1$$Register->successor(), 0);
7781     __ strexd($tmp2$$Register, $tmp1$$Register, $mem$$Address);
7782     __ cmp($tmp2$$Register, 0);
7783     __ b(loop, ne);
7784   %}
7785   ins_pipe( long_memory_op );
7786 %}
7787 #endif
7788 
7789 #ifdef AARCH64
7790 instruct xaddL_reg(memoryex mem, iRegL add, iRegL res, iRegL tmp1, iRegI tmp2) %{
7791   match(Set res (GetAndAddL mem add));
7792   effect(TEMP tmp1, TEMP tmp2, TEMP res);
7793   size(16);
7794   format %{ "loop:\n\t"
7795             "LDXR     $res, $mem\n\t"
7796             "ADD      $tmp1, $res, $add\n\t"
7797             "STXR     $tmp2, $tmp1, $mem\n\t"
7798             "CBNZ_w   $tmp2, loop" %}
7799 
7800   ins_encode %{
7801     Label loop;
7802     Register base = reg_to_register_object($mem$$base);
7803     __ bind(loop);
7804     __ ldxr($res$$Register, base);
7805     __ add($tmp1$$Register, $res$$Register, $add$$Register);
7806     __ stxr($tmp2$$Register, $tmp1$$Register, base);
7807     __ cbnz_w($tmp2$$Register, loop);
7808   %}
7809   ins_pipe( long_memory_op );
7810 %}
7811 #else
7812 instruct xaddL_reg(memoryex mem, iRegL add, iRegLd res, iRegLd tmp1, iRegI tmp2, flagsReg ccr) %{
7813   match(Set res (GetAndAddL mem add));
7814   effect( KILL ccr, TEMP tmp1, TEMP tmp2, TEMP res);
7815   size(24);
7816   format %{ "loop: \n\t"
7817             "LDREXD   $res, $mem\n\t"
7818             "ADDS     $tmp1.lo, $res.lo, $add.lo\n\t"
7819             "ADC      $tmp1.hi, $res.hi, $add.hi\n\t"
7820             "STREXD   $tmp2, $tmp1, $mem\n\t"
7821             "CMP      $tmp2, 0 \n\t"
7822             "B.ne     loop \n\t" %}
7823 
7824   ins_encode %{
7825     Label loop;
7826     __ bind(loop);
7827     __ ldrexd($res$$Register, $mem$$Address);
7828     __ adds($tmp1$$Register, $res$$Register, $add$$Register);
7829     __ adc($tmp1$$Register->successor(), $res$$Register->successor(), $add$$Register->successor());
7830     __ strexd($tmp2$$Register, $tmp1$$Register, $mem$$Address);
7831     __ cmp($tmp2$$Register, 0);
7832     __ b(loop, ne);
7833   %}
7834   ins_pipe( long_memory_op );
7835 %}
7836 #endif
7837 
7838 #ifdef AARCH64
7839 instruct xaddL_imm(memoryex mem, aimmL add, iRegL res, iRegL tmp1, iRegI tmp2) %{
7840   match(Set res (GetAndAddL mem add));
7841   effect(TEMP tmp1, TEMP tmp2, TEMP res);
7842   size(16);
7843   format %{ "loop:\n\t"
7844             "LDXR     $res, $mem\n\t"
7845             "ADD      $tmp1, $res, $add\n\t"
7846             "STXR     $tmp2, $tmp1, $mem\n\t"
7847             "CBNZ_w   $tmp2, loop" %}
7848 
7849   ins_encode %{
7850     Label loop;
7851     Register base = reg_to_register_object($mem$$base);
7852     __ bind(loop);
7853     __ ldxr($res$$Register, base);
7854     __ add($tmp1$$Register, $res$$Register, $add$$constant);
7855     __ stxr($tmp2$$Register, $tmp1$$Register, base);
7856     __ cbnz_w($tmp2$$Register, loop);
7857   %}
7858   ins_pipe( long_memory_op );
7859 %}
7860 #else
7861 // TODO: try immLRot2 instead, (0, $con$$constant) becomes
7862 // (hi($con$$constant), lo($con$$constant)) becomes
7863 instruct xaddL_immRot(memoryex mem, immLlowRot add, iRegLd res, iRegLd tmp1, iRegI tmp2, flagsReg ccr) %{
7864   match(Set res (GetAndAddL mem add));
7865   effect( KILL ccr, TEMP tmp1, TEMP tmp2, TEMP res);
7866   size(24);
7867   format %{ "loop: \n\t"
7868             "LDREXD   $res, $mem\n\t"
7869             "ADDS     $tmp1.lo, $res.lo, $add\n\t"
7870             "ADC      $tmp1.hi, $res.hi, 0\n\t"
7871             "STREXD   $tmp2, $tmp1, $mem\n\t"
7872             "CMP      $tmp2, 0 \n\t"
7873             "B.ne     loop \n\t" %}
7874 
7875   ins_encode %{
7876     Label loop;
7877     __ bind(loop);
7878     __ ldrexd($res$$Register, $mem$$Address);
7879     __ adds($tmp1$$Register, $res$$Register, $add$$constant);
7880     __ adc($tmp1$$Register->successor(), $res$$Register->successor(), 0);
7881     __ strexd($tmp2$$Register, $tmp1$$Register, $mem$$Address);
7882     __ cmp($tmp2$$Register, 0);
7883     __ b(loop, ne);
7884   %}
7885   ins_pipe( long_memory_op );
7886 %}
7887 #endif
7888 
7889 #ifdef AARCH64
7890 instruct xchgI(memoryex mem, iRegI newval, iRegI res, iRegI tmp) %{
7891   match(Set res (GetAndSetI mem newval));
7892   effect(TEMP tmp, TEMP res);
7893   size(12);
7894   format %{ "loop:\n\t"
7895             "LDXR_w   $res, $mem\n\t"
7896             "STXR_w   $tmp, $newval, $mem\n\t"
7897             "CBNZ_w   $tmp, loop" %}
7898 
7899   ins_encode %{
7900     Label loop;
7901     Register base = reg_to_register_object($mem$$base);
7902     __ bind(loop);
7903     __ ldxr_w($res$$Register, base);
7904     __ stxr_w($tmp$$Register, $newval$$Register, base);
7905     __ cbnz_w($tmp$$Register, loop);
7906   %}
7907   ins_pipe( long_memory_op );
7908 %}
7909 
7910 #ifdef XXX
7911 // Disabled until 8051805 is fixed.
7912 instruct xchgN(memoryex mem, iRegN newval, iRegN res, iRegN tmp) %{
7913   match(Set res (GetAndSetN mem newval));
7914   effect(TEMP tmp, TEMP res);
7915   size(12);
7916   format %{ "loop:\n\t"
7917             "LDXR_w   $res, $mem\n\t"
7918             "STXR_w   $tmp, $newval, $mem\n\t"
7919             "CBNZ_w   $tmp, loop" %}
7920 
7921   ins_encode %{
7922     Label loop;
7923     Register base = reg_to_register_object($mem$$base);
7924     __ bind(loop);
7925     __ ldxr_w($res$$Register, base);
7926     __ stxr_w($tmp$$Register, $newval$$Register, base);
7927     __ cbnz_w($tmp$$Register, loop);
7928   %}
7929   ins_pipe( long_memory_op );
7930 %}
7931 #endif
7932 #else
7933 instruct xchgI(memoryex mem, iRegI newval, iRegI res, iRegI tmp, flagsReg ccr) %{
7934   match(Set res (GetAndSetI mem newval));
7935   effect(KILL ccr, TEMP tmp, TEMP res);
7936   size(16);
7937   format %{ "loop: \n\t"
7938             "LDREX    $res, $mem\n\t"
7939             "STREX    $tmp, $newval, $mem\n\t"
7940             "CMP      $tmp, 0 \n\t"
7941             "B.ne     loop \n\t" %}
7942 
7943   ins_encode %{
7944     Label loop;
7945     __ bind(loop);
7946     __ ldrex($res$$Register,$mem$$Address);
7947     __ strex($tmp$$Register, $newval$$Register, $mem$$Address);
7948     __ cmp($tmp$$Register, 0);
7949     __ b(loop, ne);
7950   %}
7951   ins_pipe( long_memory_op );
7952 %}
7953 #endif
7954 
7955 #ifdef AARCH64
7956 instruct xchgL(memoryex mem, iRegL newval, iRegL res, iRegI tmp) %{
7957   match(Set res (GetAndSetL mem newval));
7958   effect(TEMP tmp, TEMP res);
7959   size(12);
7960   format %{ "loop:\n\t"
7961             "LDXR     $res, $mem\n\t"
7962             "STXR     $tmp, $newval, $mem\n\t"
7963             "CBNZ_w   $tmp, loop" %}
7964 
7965   ins_encode %{
7966     Label loop;
7967     Register base = reg_to_register_object($mem$$base);
7968     __ bind(loop);
7969     __ ldxr($res$$Register, base);
7970     __ stxr($tmp$$Register, $newval$$Register, base);
7971     __ cbnz_w($tmp$$Register, loop);
7972   %}
7973   ins_pipe( long_memory_op );
7974 %}
7975 #else
7976 instruct xchgL(memoryex mem, iRegLd newval, iRegLd res, iRegI tmp, flagsReg ccr) %{
7977   match(Set res (GetAndSetL mem newval));
7978   effect( KILL ccr, TEMP tmp, TEMP res);
7979   size(16);
7980   format %{ "loop: \n\t"
7981             "LDREXD   $res, $mem\n\t"
7982             "STREXD   $tmp, $newval, $mem\n\t"
7983             "CMP      $tmp, 0 \n\t"
7984             "B.ne     loop \n\t" %}
7985 
7986   ins_encode %{
7987     Label loop;
7988     __ bind(loop);
7989     __ ldrexd($res$$Register, $mem$$Address);
7990     __ strexd($tmp$$Register, $newval$$Register, $mem$$Address);
7991     __ cmp($tmp$$Register, 0);
7992     __ b(loop, ne);
7993   %}
7994   ins_pipe( long_memory_op );
7995 %}
7996 #endif // !AARCH64
7997 
7998 #ifdef AARCH64
7999 instruct xchgP(memoryex mem, iRegP newval, iRegP res, iRegI tmp) %{
8000   match(Set res (GetAndSetP mem newval));
8001   effect(TEMP tmp, TEMP res);
8002   size(12);
8003   format %{ "loop:\n\t"
8004             "LDREX    $res, $mem\n\t"
8005             "STREX    $tmp, $newval, $mem\n\t"
8006             "CBNZ_w   $tmp, loop" %}
8007 
8008   ins_encode %{
8009     Label loop;
8010     Register base = reg_to_register_object($mem$$base);
8011     __ bind(loop);
8012     __ ldrex($res$$Register, base);
8013     __ strex($tmp$$Register, $newval$$Register, base);
8014     __ cbnz_w($tmp$$Register, loop);
8015   %}
8016   ins_pipe( long_memory_op );
8017 %}
8018 #else
8019 instruct xchgP(memoryex mem, iRegP newval, iRegP res, iRegI tmp, flagsReg ccr) %{
8020   match(Set res (GetAndSetP mem newval));
8021   effect(KILL ccr, TEMP tmp, TEMP res);
8022   size(16);
8023   format %{ "loop: \n\t"
8024             "LDREX    $res, $mem\n\t"
8025             "STREX    $tmp, $newval, $mem\n\t"
8026             "CMP      $tmp, 0 \n\t"
8027             "B.ne     loop \n\t" %}
8028 
8029   ins_encode %{
8030     Label loop;
8031     __ bind(loop);
8032     __ ldrex($res$$Register,$mem$$Address);
8033     __ strex($tmp$$Register, $newval$$Register, $mem$$Address);
8034     __ cmp($tmp$$Register, 0);
8035     __ b(loop, ne);
8036   %}
8037   ins_pipe( long_memory_op );
8038 %}
8039 #endif // !AARCH64
8040 
8041 //---------------------
8042 // Subtraction Instructions
8043 // Register Subtraction
8044 instruct subI_reg_reg(iRegI dst, iRegI src1, iRegI src2) %{
8045   match(Set dst (SubI src1 src2));
8046 
8047   size(4);
8048   format %{ "sub_32 $dst,$src1,$src2\t! int" %}
8049   ins_encode %{
8050     __ sub_32($dst$$Register, $src1$$Register, $src2$$Register);
8051   %}
8052   ins_pipe(ialu_reg_reg);
8053 %}
8054 
8055 #ifndef AARCH64
8056 instruct subshlI_reg_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3) %{
8057   match(Set dst (SubI src1 (LShiftI src2 src3)));
8058 
8059   size(4);
8060   format %{ "SUB    $dst,$src1,$src2<<$src3" %}
8061   ins_encode %{
8062     __ sub($dst$$Register, $src1$$Register, AsmOperand($src2$$Register, lsl, $src3$$Register));
8063   %}
8064   ins_pipe(ialu_reg_reg);
8065 %}
8066 #endif
8067 
8068 instruct subshlI_reg_reg_imm(iRegI dst, iRegI src1, iRegI src2, immU5 src3) %{
8069   match(Set dst (SubI src1 (LShiftI src2 src3)));
8070 
8071   size(4);
8072   format %{ "sub_32 $dst,$src1,$src2<<$src3\t! int" %}
8073   ins_encode %{
8074     __ sub_32($dst$$Register, $src1$$Register, AsmOperand($src2$$Register, lsl, $src3$$constant));
8075   %}
8076   ins_pipe(ialu_reg_reg);
8077 %}
8078 
8079 #ifndef AARCH64
8080 instruct subsarI_reg_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3) %{
8081   match(Set dst (SubI src1 (RShiftI src2 src3)));
8082 
8083   size(4);
8084   format %{ "SUB    $dst,$src1,$src2>>$src3" %}
8085   ins_encode %{
8086     __ sub($dst$$Register, $src1$$Register, AsmOperand($src2$$Register, asr, $src3$$Register));
8087   %}
8088   ins_pipe(ialu_reg_reg);
8089 %}
8090 #endif
8091 
8092 instruct subsarI_reg_reg_imm(iRegI dst, iRegI src1, iRegI src2, immU5 src3) %{
8093   match(Set dst (SubI src1 (RShiftI src2 src3)));
8094 
8095   size(4);
8096   format %{ "sub_32 $dst,$src1,$src2>>$src3\t! int" %}
8097   ins_encode %{
8098     __ sub_32($dst$$Register, $src1$$Register, AsmOperand($src2$$Register, asr, $src3$$constant));
8099   %}
8100   ins_pipe(ialu_reg_reg);
8101 %}
8102 
8103 #ifndef AARCH64
8104 instruct subshrI_reg_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3) %{
8105   match(Set dst (SubI src1 (URShiftI src2 src3)));
8106 
8107   size(4);
8108   format %{ "SUB    $dst,$src1,$src2>>>$src3" %}
8109   ins_encode %{
8110     __ sub($dst$$Register, $src1$$Register, AsmOperand($src2$$Register, lsr, $src3$$Register));
8111   %}
8112   ins_pipe(ialu_reg_reg);
8113 %}
8114 #endif
8115 
8116 instruct subshrI_reg_reg_imm(iRegI dst, iRegI src1, iRegI src2, immU5 src3) %{
8117   match(Set dst (SubI src1 (URShiftI src2 src3)));
8118 
8119   size(4);
8120   format %{ "sub_32 $dst,$src1,$src2>>>$src3\t! int" %}
8121   ins_encode %{
8122     __ sub_32($dst$$Register, $src1$$Register, AsmOperand($src2$$Register, lsr, $src3$$constant));
8123   %}
8124   ins_pipe(ialu_reg_reg);
8125 %}
8126 
8127 #ifndef AARCH64
8128 instruct rsbshlI_reg_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3) %{
8129   match(Set dst (SubI (LShiftI src1 src2) src3));
8130 
8131   size(4);
8132   format %{ "RSB    $dst,$src3,$src1<<$src2" %}
8133   ins_encode %{
8134     __ rsb($dst$$Register, $src3$$Register, AsmOperand($src1$$Register, lsl, $src2$$Register));
8135   %}
8136   ins_pipe(ialu_reg_reg);
8137 %}
8138 
8139 instruct rsbshlI_reg_imm_reg(iRegI dst, iRegI src1, immU5 src2, iRegI src3) %{
8140   match(Set dst (SubI (LShiftI src1 src2) src3));
8141 
8142   size(4);
8143   format %{ "RSB    $dst,$src3,$src1<<$src2" %}
8144   ins_encode %{
8145     __ rsb($dst$$Register, $src3$$Register, AsmOperand($src1$$Register, lsl, $src2$$constant));
8146   %}
8147   ins_pipe(ialu_reg_reg);
8148 %}
8149 
8150 instruct rsbsarI_reg_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3) %{
8151   match(Set dst (SubI (RShiftI src1 src2) src3));
8152 
8153   size(4);
8154   format %{ "RSB    $dst,$src3,$src1>>$src2" %}
8155   ins_encode %{
8156     __ rsb($dst$$Register, $src3$$Register, AsmOperand($src1$$Register, asr, $src2$$Register));
8157   %}
8158   ins_pipe(ialu_reg_reg);
8159 %}
8160 
8161 instruct rsbsarI_reg_imm_reg(iRegI dst, iRegI src1, immU5 src2, iRegI src3) %{
8162   match(Set dst (SubI (RShiftI src1 src2) src3));
8163 
8164   size(4);
8165   format %{ "RSB    $dst,$src3,$src1>>$src2" %}
8166   ins_encode %{
8167     __ rsb($dst$$Register, $src3$$Register, AsmOperand($src1$$Register, asr, $src2$$constant));
8168   %}
8169   ins_pipe(ialu_reg_reg);
8170 %}
8171 
8172 instruct rsbshrI_reg_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3) %{
8173   match(Set dst (SubI (URShiftI src1 src2) src3));
8174 
8175   size(4);
8176   format %{ "RSB    $dst,$src3,$src1>>>$src2" %}
8177   ins_encode %{
8178     __ rsb($dst$$Register, $src3$$Register, AsmOperand($src1$$Register, lsr, $src2$$Register));
8179   %}
8180   ins_pipe(ialu_reg_reg);
8181 %}
8182 
8183 instruct rsbshrI_reg_imm_reg(iRegI dst, iRegI src1, immU5 src2, iRegI src3) %{
8184   match(Set dst (SubI (URShiftI src1 src2) src3));
8185 
8186   size(4);
8187   format %{ "RSB    $dst,$src3,$src1>>>$src2" %}
8188   ins_encode %{
8189     __ rsb($dst$$Register, $src3$$Register, AsmOperand($src1$$Register, lsr, $src2$$constant));
8190   %}
8191   ins_pipe(ialu_reg_reg);
8192 %}
8193 #endif
8194 
8195 // Immediate Subtraction
8196 instruct subI_reg_aimmI(iRegI dst, iRegI src1, aimmI src2) %{
8197   match(Set dst (SubI src1 src2));
8198 
8199   size(4);
8200   format %{ "sub_32 $dst,$src1,$src2\t! int" %}
8201   ins_encode %{
8202     __ sub_32($dst$$Register, $src1$$Register, $src2$$constant);
8203   %}
8204   ins_pipe(ialu_reg_imm);
8205 %}
8206 
8207 instruct subI_reg_immRotneg(iRegI dst, iRegI src1, aimmIneg src2) %{
8208   match(Set dst (AddI src1 src2));
8209 
8210   size(4);
8211   format %{ "sub_32 $dst,$src1,-($src2)\t! int" %}
8212   ins_encode %{
8213     __ sub_32($dst$$Register, $src1$$Register, -$src2$$constant);
8214   %}
8215   ins_pipe(ialu_reg_imm);
8216 %}
8217 
8218 #ifndef AARCH64
8219 instruct subI_immRot_reg(iRegI dst, immIRot src1, iRegI src2) %{
8220   match(Set dst (SubI src1 src2));
8221 
8222   size(4);
8223   format %{ "RSB    $dst,$src2,src1" %}
8224   ins_encode %{
8225     __ rsb($dst$$Register, $src2$$Register, $src1$$constant);
8226   %}
8227   ins_pipe(ialu_zero_reg);
8228 %}
8229 #endif
8230 
8231 // Register Subtraction
8232 #ifdef AARCH64
8233 instruct subL_reg_reg(iRegL dst, iRegL src1, iRegL src2) %{
8234   match(Set dst (SubL src1 src2));
8235 
8236   size(4);
8237   format %{ "SUB    $dst,$src1,$src2\t! long" %}
8238   ins_encode %{
8239     __ sub($dst$$Register, $src1$$Register, $src2$$Register);
8240   %}
8241   ins_pipe(ialu_reg_reg);
8242 %}
8243 #else
8244 instruct subL_reg_reg(iRegL dst, iRegL src1, iRegL src2, flagsReg icc ) %{
8245   match(Set dst (SubL src1 src2));
8246   effect (KILL icc);
8247 
8248   size(8);
8249   format %{ "SUBS   $dst.lo,$src1.lo,$src2.lo\t! long\n\t"
8250             "SBC    $dst.hi,$src1.hi,$src2.hi" %}
8251   ins_encode %{
8252     __ subs($dst$$Register, $src1$$Register, $src2$$Register);
8253     __ sbc($dst$$Register->successor(), $src1$$Register->successor(), $src2$$Register->successor());
8254   %}
8255   ins_pipe(ialu_reg_reg);
8256 %}
8257 #endif
8258 
8259 #ifdef AARCH64
8260 // Immediate Subtraction
8261 instruct subL_reg_aimm(iRegL dst, iRegL src1, aimmL src2) %{
8262   match(Set dst (SubL src1 src2));
8263 
8264   size(4);
8265   format %{ "SUB    $dst,$src1,$src2\t! long" %}
8266   ins_encode %{
8267     __ sub($dst$$Register, $src1$$Register, $src2$$constant);
8268   %}
8269   ins_pipe(ialu_reg_imm);
8270 %}
8271 
8272 instruct subL_reg_immLneg(iRegL dst, iRegL src1, aimmLneg src2) %{
8273   match(Set dst (AddL src1 src2));
8274 
8275   size(4);
8276   format %{ "SUB    $dst,$src1,-($src2)\t! long" %}
8277   ins_encode %{
8278     __ sub($dst$$Register, $src1$$Register, -$src2$$constant);
8279   %}
8280   ins_pipe(ialu_reg_imm);
8281 %}
8282 #else
8283 // TODO
8284 #endif
8285 
8286 #ifndef AARCH64
8287 // Immediate Subtraction
8288 // TODO: try immLRot2 instead, (0, $con$$constant) becomes
8289 // (hi($con$$constant), lo($con$$constant)) becomes
8290 instruct subL_reg_immRot(iRegL dst, iRegL src1, immLlowRot con, flagsReg icc) %{
8291   match(Set dst (SubL src1 con));
8292   effect (KILL icc);
8293 
8294   size(8);
8295   format %{ "SUB    $dst.lo,$src1.lo,$con\t! long\n\t"
8296             "SBC    $dst.hi,$src1.hi,0" %}
8297   ins_encode %{
8298     __ subs($dst$$Register, $src1$$Register, $con$$constant);
8299     __ sbc($dst$$Register->successor(), $src1$$Register->successor(), 0);
8300   %}
8301   ins_pipe(ialu_reg_imm);
8302 %}
8303 
8304 // Long negation
8305 instruct negL_reg_reg(iRegL dst, immL0 zero, iRegL src2, flagsReg icc) %{
8306   match(Set dst (SubL zero src2));
8307   effect (KILL icc);
8308 
8309   size(8);
8310   format %{ "RSBS   $dst.lo,$src2.lo,0\t! long\n\t"
8311             "RSC    $dst.hi,$src2.hi,0" %}
8312   ins_encode %{
8313     __ rsbs($dst$$Register, $src2$$Register, 0);
8314     __ rsc($dst$$Register->successor(), $src2$$Register->successor(), 0);
8315   %}
8316   ins_pipe(ialu_zero_reg);
8317 %}
8318 #endif // !AARCH64
8319 
8320 // Multiplication Instructions
8321 // Integer Multiplication
8322 // Register Multiplication
8323 instruct mulI_reg_reg(iRegI dst, iRegI src1, iRegI src2) %{
8324   match(Set dst (MulI src1 src2));
8325 
8326   size(4);
8327   format %{ "mul_32 $dst,$src1,$src2" %}
8328   ins_encode %{
8329     __ mul_32($dst$$Register, $src1$$Register, $src2$$Register);
8330   %}
8331   ins_pipe(imul_reg_reg);
8332 %}
8333 
8334 #ifdef AARCH64
8335 instruct mulL_reg_reg(iRegL dst, iRegL src1, iRegL src2) %{
8336   match(Set dst (MulL src1 src2));
8337   size(4);
8338   format %{ "MUL  $dst,$src1,$src2\t! long" %}
8339   ins_encode %{
8340     __ mul($dst$$Register, $src1$$Register, $src2$$Register);
8341   %}
8342   ins_pipe(imul_reg_reg);
8343 %}
8344 #else
8345 instruct mulL_lo1_hi2(iRegL dst, iRegL src1, iRegL src2) %{
8346   effect(DEF dst, USE src1, USE src2);
8347   size(4);
8348   format %{ "MUL  $dst.hi,$src1.lo,$src2.hi\t! long" %}
8349   ins_encode %{
8350     __ mul($dst$$Register->successor(), $src1$$Register, $src2$$Register->successor());
8351   %}
8352   ins_pipe(imul_reg_reg);
8353 %}
8354 
8355 instruct mulL_hi1_lo2(iRegL dst, iRegL src1, iRegL src2) %{
8356   effect(USE_DEF dst, USE src1, USE src2);
8357   size(8);
8358   format %{ "MLA  $dst.hi,$src1.hi,$src2.lo,$dst.hi\t! long\n\t"
8359             "MOV  $dst.lo, 0"%}
8360   ins_encode %{
8361     __ mla($dst$$Register->successor(), $src1$$Register->successor(), $src2$$Register, $dst$$Register->successor());
8362     __ mov($dst$$Register, 0);
8363   %}
8364   ins_pipe(imul_reg_reg);
8365 %}
8366 
8367 instruct mulL_lo1_lo2(iRegL dst, iRegL src1, iRegL src2) %{
8368   effect(USE_DEF dst, USE src1, USE src2);
8369   size(4);
8370   format %{ "UMLAL  $dst.lo,$dst.hi,$src1,$src2\t! long" %}
8371   ins_encode %{
8372     __ umlal($dst$$Register, $dst$$Register->successor(), $src1$$Register, $src2$$Register);
8373   %}
8374   ins_pipe(imul_reg_reg);
8375 %}
8376 
8377 instruct mulL_reg_reg(iRegL dst, iRegL src1, iRegL src2) %{
8378   match(Set dst (MulL src1 src2));
8379 
8380   expand %{
8381     mulL_lo1_hi2(dst, src1, src2);
8382     mulL_hi1_lo2(dst, src1, src2);
8383     mulL_lo1_lo2(dst, src1, src2);
8384   %}
8385 %}
8386 #endif // !AARCH64
8387 
8388 // Integer Division
8389 // Register Division
8390 #ifdef AARCH64
8391 instruct divI_reg_reg(iRegI dst, iRegI src1, iRegI src2) %{
8392   match(Set dst (DivI src1 src2));
8393 
8394   size(4);
8395   format %{ "SDIV    $dst,$src1,$src2\t! 32-bit" %}
8396   ins_encode %{
8397     __ sdiv_w($dst$$Register, $src1$$Register, $src2$$Register);
8398   %}
8399   ins_pipe(ialu_reg_reg); // FIXME
8400 %}
8401 #else
8402 instruct divI_reg_reg(R1RegI dst, R0RegI src1, R2RegI src2, LRRegP lr, flagsReg ccr) %{
8403   match(Set dst (DivI src1 src2));
8404   effect( KILL ccr, KILL src1, KILL src2, KILL lr);
8405   ins_cost((2+71)*DEFAULT_COST);
8406 
8407   format %{ "DIV   $dst,$src1,$src2 ! call to StubRoutines::Arm::idiv_irem_entry()" %}
8408   ins_encode %{
8409     __ call(StubRoutines::Arm::idiv_irem_entry(), relocInfo::runtime_call_type);
8410   %}
8411   ins_pipe(sdiv_reg_reg);
8412 %}
8413 #endif
8414 
8415 // Register Long Division
8416 #ifdef AARCH64
8417 instruct divL_reg_reg(iRegL dst, iRegL src1, iRegL src2) %{
8418   match(Set dst (DivL src1 src2));
8419 
8420   size(4);
8421   format %{ "SDIV    $dst,$src1,$src2" %}
8422   ins_encode %{
8423     __ sdiv($dst$$Register, $src1$$Register, $src2$$Register);
8424   %}
8425   ins_pipe(ialu_reg_reg); // FIXME
8426 %}
8427 #else
8428 instruct divL_reg_reg(R0R1RegL dst, R2R3RegL src1, R0R1RegL src2) %{
8429   match(Set dst (DivL src1 src2));
8430   effect(CALL);
8431   ins_cost(DEFAULT_COST*71);
8432   format %{ "DIVL  $src1,$src2,$dst\t! long ! call to SharedRuntime::ldiv" %}
8433   ins_encode %{
8434     address target = CAST_FROM_FN_PTR(address, SharedRuntime::ldiv);
8435     __ call(target, relocInfo::runtime_call_type);
8436   %}
8437   ins_pipe(divL_reg_reg);
8438 %}
8439 #endif
8440 
8441 // Integer Remainder
8442 // Register Remainder
8443 #ifdef AARCH64
8444 #ifdef TODO
8445 instruct msubI_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3) %{
8446   match(Set dst (SubI src1 (MulI src2 src3)));
8447 
8448   size(4);
8449   format %{ "MSUB    $dst,$src2,$src3,$src1\t! 32-bit\n\t" %}
8450   ins_encode %{
8451     __ msub_w($dst$$Register, $src2$$Register, $src3$$Register, $src1$$Register);
8452   %}
8453   ins_pipe(ialu_reg_reg); // FIXME
8454 %}
8455 #endif
8456 
8457 instruct modI_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI temp) %{
8458   match(Set dst (ModI src1 src2));
8459   effect(TEMP temp);
8460 
8461   size(8);
8462   format %{ "SDIV    $temp,$src1,$src2\t! 32-bit\n\t"
8463             "MSUB    $dst,$src2,$temp,$src1\t! 32-bit\n\t" %}
8464   ins_encode %{
8465     __ sdiv_w($temp$$Register, $src1$$Register, $src2$$Register);
8466     __ msub_w($dst$$Register, $src2$$Register, $temp$$Register, $src1$$Register);
8467   %}
8468   ins_pipe(ialu_reg_reg); // FIXME
8469 %}
8470 #else
8471 instruct modI_reg_reg(R0RegI dst, R0RegI src1, R2RegI src2, R1RegI temp, LRRegP lr, flagsReg ccr ) %{
8472   match(Set dst (ModI src1 src2));
8473   effect( KILL ccr, KILL temp, KILL src2, KILL lr);
8474 
8475   format %{ "MODI   $dst,$src1,$src2\t ! call to StubRoutines::Arm::idiv_irem_entry" %}
8476   ins_encode %{
8477     __ call(StubRoutines::Arm::idiv_irem_entry(), relocInfo::runtime_call_type);
8478   %}
8479   ins_pipe(sdiv_reg_reg);
8480 %}
8481 #endif
8482 
8483 // Register Long Remainder
8484 #ifdef AARCH64
8485 instruct modL_reg_reg(iRegL dst, iRegL src1, iRegL src2, iRegL temp) %{
8486   match(Set dst (ModL src1 src2));
8487   effect(TEMP temp);
8488 
8489   size(8);
8490   format %{ "SDIV    $temp,$src1,$src2\n\t"
8491             "MSUB    $dst,$src2,$temp,$src1" %}
8492   ins_encode %{
8493     __ sdiv($temp$$Register, $src1$$Register, $src2$$Register);
8494     __ msub($dst$$Register, $src2$$Register, $temp$$Register, $src1$$Register);
8495   %}
8496   ins_pipe(ialu_reg_reg); // FIXME
8497 %}
8498 #else
8499 instruct modL_reg_reg(R0R1RegL dst, R2R3RegL src1, R0R1RegL src2) %{
8500   match(Set dst (ModL src1 src2));
8501   effect(CALL);
8502   ins_cost(MEMORY_REF_COST); // FIXME
8503   format %{ "modL    $dst,$src1,$src2\t ! call to SharedRuntime::lrem" %}
8504   ins_encode %{
8505     address target = CAST_FROM_FN_PTR(address, SharedRuntime::lrem);
8506     __ call(target, relocInfo::runtime_call_type);
8507   %}
8508   ins_pipe(divL_reg_reg);
8509 %}
8510 #endif
8511 
8512 // Integer Shift Instructions
8513 
8514 // Register Shift Left
8515 instruct shlI_reg_reg(iRegI dst, iRegI src1, iRegI src2) %{
8516   match(Set dst (LShiftI src1 src2));
8517 
8518   size(4);
8519 #ifdef AARCH64
8520   format %{ "LSLV   $dst,$src1,$src2\t! int" %}
8521   ins_encode %{
8522     __ lslv_w($dst$$Register, $src1$$Register, $src2$$Register);
8523   %}
8524 #else
8525   format %{ "LSL  $dst,$src1,$src2 \n\t" %}
8526   ins_encode %{
8527     __ mov($dst$$Register, AsmOperand($src1$$Register, lsl, $src2$$Register));
8528   %}
8529 #endif
8530   ins_pipe(ialu_reg_reg);
8531 %}
8532 
8533 // Register Shift Left Immediate
8534 instruct shlI_reg_imm5(iRegI dst, iRegI src1, immU5 src2) %{
8535   match(Set dst (LShiftI src1 src2));
8536 
8537   size(4);
8538 #ifdef AARCH64
8539   format %{ "LSL_w  $dst,$src1,$src2\t! int" %}
8540   ins_encode %{
8541     __ _lsl($dst$$Register, $src1$$Register, $src2$$constant);
8542   %}
8543 #else
8544   format %{ "LSL    $dst,$src1,$src2\t! int" %}
8545   ins_encode %{
8546     __ logical_shift_left($dst$$Register, $src1$$Register, $src2$$constant);
8547   %}
8548 #endif
8549   ins_pipe(ialu_reg_imm);
8550 %}
8551 
8552 #ifndef AARCH64
8553 instruct shlL_reg_reg_merge_hi(iRegL dst, iRegL src1, iRegI src2) %{
8554   effect(USE_DEF dst, USE src1, USE src2);
8555   size(4);
8556   format %{"OR  $dst.hi,$dst.hi,($src1.hi << $src2)"  %}
8557   ins_encode %{
8558     __ orr($dst$$Register->successor(), $dst$$Register->successor(), AsmOperand($src1$$Register->successor(), lsl, $src2$$Register));
8559   %}
8560   ins_pipe(ialu_reg_reg);
8561 %}
8562 
8563 instruct shlL_reg_reg_merge_lo(iRegL dst, iRegL src1, iRegI src2) %{
8564   effect(USE_DEF dst, USE src1, USE src2);
8565   size(4);
8566   format %{ "LSL  $dst.lo,$src1.lo,$src2 \n\t" %}
8567   ins_encode %{
8568     __ mov($dst$$Register, AsmOperand($src1$$Register, lsl, $src2$$Register));
8569   %}
8570   ins_pipe(ialu_reg_reg);
8571 %}
8572 
8573 instruct shlL_reg_reg_overlap(iRegL dst, iRegL src1, iRegI src2, flagsReg ccr) %{
8574   effect(DEF dst, USE src1, USE src2, KILL ccr);
8575   size(16);
8576   format %{ "SUBS  $dst.hi,$src2,32 \n\t"
8577             "LSLpl $dst.hi,$src1.lo,$dst.hi \n\t"
8578             "RSBmi $dst.hi,$dst.hi,0 \n\t"
8579             "LSRmi $dst.hi,$src1.lo,$dst.hi" %}
8580 
8581   ins_encode %{
8582     // $src1$$Register and $dst$$Register->successor() can't be the same
8583     __ subs($dst$$Register->successor(), $src2$$Register, 32);
8584     __ mov($dst$$Register->successor(), AsmOperand($src1$$Register, lsl, $dst$$Register->successor()), pl);
8585     __ rsb($dst$$Register->successor(), $dst$$Register->successor(), 0, mi);
8586     __ mov($dst$$Register->successor(), AsmOperand($src1$$Register, lsr, $dst$$Register->successor()), mi);
8587   %}
8588   ins_pipe(ialu_reg_reg);
8589 %}
8590 #endif // !AARCH64
8591 
8592 instruct shlL_reg_reg(iRegL dst, iRegL src1, iRegI src2) %{
8593   match(Set dst (LShiftL src1 src2));
8594 
8595 #ifdef AARCH64
8596   size(4);
8597   format %{ "LSLV  $dst,$src1,$src2\t! long" %}
8598   ins_encode %{
8599     __ lslv($dst$$Register, $src1$$Register, $src2$$Register);
8600   %}
8601   ins_pipe(ialu_reg_reg);
8602 #else
8603   expand %{
8604     flagsReg ccr;
8605     shlL_reg_reg_overlap(dst, src1, src2, ccr);
8606     shlL_reg_reg_merge_hi(dst, src1, src2);
8607     shlL_reg_reg_merge_lo(dst, src1, src2);
8608   %}
8609 #endif
8610 %}
8611 
8612 #ifdef AARCH64
8613 instruct shlL_reg_imm6(iRegL dst, iRegL src1, immU6 src2) %{
8614   match(Set dst (LShiftL src1 src2));
8615 
8616   size(4);
8617   format %{ "LSL    $dst,$src1,$src2\t! long" %}
8618   ins_encode %{
8619     __ logical_shift_left($dst$$Register, $src1$$Register, $src2$$constant);
8620   %}
8621   ins_pipe(ialu_reg_imm);
8622 %}
8623 #else
8624 // Register Shift Left Immediate
8625 instruct shlL_reg_imm6(iRegL dst, iRegL src1, immU6Big src2) %{
8626   match(Set dst (LShiftL src1 src2));
8627 
8628   size(8);
8629   format %{ "LSL   $dst.hi,$src1.lo,$src2-32\t! or mov if $src2==32\n\t"
8630             "MOV   $dst.lo, 0" %}
8631   ins_encode %{
8632     if ($src2$$constant == 32) {
8633       __ mov($dst$$Register->successor(), $src1$$Register);
8634     } else {
8635       __ mov($dst$$Register->successor(), AsmOperand($src1$$Register, lsl, $src2$$constant-32));
8636     }
8637     __ mov($dst$$Register, 0);
8638   %}
8639   ins_pipe(ialu_reg_imm);
8640 %}
8641 
8642 instruct shlL_reg_imm5(iRegL dst, iRegL src1, immU5 src2) %{
8643   match(Set dst (LShiftL src1 src2));
8644 
8645   size(12);
8646   format %{ "LSL   $dst.hi,$src1.lo,$src2\n\t"
8647             "OR    $dst.hi, $dst.hi, $src1.lo >> 32-$src2\n\t"
8648             "LSL   $dst.lo,$src1.lo,$src2" %}
8649   ins_encode %{
8650     // The order of the following 3 instructions matters: src1.lo and
8651     // dst.hi can't overlap but src.hi and dst.hi can.
8652     __ mov($dst$$Register->successor(), AsmOperand($src1$$Register->successor(), lsl, $src2$$constant));
8653     __ orr($dst$$Register->successor(), $dst$$Register->successor(), AsmOperand($src1$$Register, lsr, 32-$src2$$constant));
8654     __ mov($dst$$Register, AsmOperand($src1$$Register, lsl, $src2$$constant));
8655   %}
8656   ins_pipe(ialu_reg_imm);
8657 %}
8658 #endif // !AARCH64
8659 
8660 // Register Arithmetic Shift Right
8661 instruct sarI_reg_reg(iRegI dst, iRegI src1, iRegI src2) %{
8662   match(Set dst (RShiftI src1 src2));
8663   size(4);
8664 #ifdef AARCH64
8665   format %{ "ASRV   $dst,$src1,$src2\t! int" %}
8666   ins_encode %{
8667     __ asrv_w($dst$$Register, $src1$$Register, $src2$$Register);
8668   %}
8669 #else
8670   format %{ "ASR    $dst,$src1,$src2\t! int" %}
8671   ins_encode %{
8672     __ mov($dst$$Register, AsmOperand($src1$$Register, asr, $src2$$Register));
8673   %}
8674 #endif
8675   ins_pipe(ialu_reg_reg);
8676 %}
8677 
8678 // Register Arithmetic Shift Right Immediate
8679 instruct sarI_reg_imm5(iRegI dst, iRegI src1, immU5 src2) %{
8680   match(Set dst (RShiftI src1 src2));
8681 
8682   size(4);
8683 #ifdef AARCH64
8684   format %{ "ASR_w  $dst,$src1,$src2" %}
8685   ins_encode %{
8686     __ _asr_w($dst$$Register, $src1$$Register, $src2$$constant);
8687   %}
8688 #else
8689   format %{ "ASR    $dst,$src1,$src2" %}
8690   ins_encode %{
8691     __ mov($dst$$Register, AsmOperand($src1$$Register, asr, $src2$$constant));
8692   %}
8693 #endif
8694   ins_pipe(ialu_reg_imm);
8695 %}
8696 
8697 #ifndef AARCH64
8698 // Register Shift Right Arithmetic Long
8699 instruct sarL_reg_reg_merge_lo(iRegL dst, iRegL src1, iRegI src2) %{
8700   effect(USE_DEF dst, USE src1, USE src2);
8701   size(4);
8702   format %{ "OR  $dst.lo,$dst.lo,($src1.lo >> $src2)"  %}
8703   ins_encode %{
8704     __ orr($dst$$Register, $dst$$Register, AsmOperand($src1$$Register, lsr, $src2$$Register));
8705   %}
8706   ins_pipe(ialu_reg_reg);
8707 %}
8708 
8709 instruct sarL_reg_reg_merge_hi(iRegL dst, iRegL src1, iRegI src2) %{
8710   effect(USE_DEF dst, USE src1, USE src2);
8711   size(4);
8712   format %{ "ASR  $dst.hi,$src1.hi,$src2 \n\t" %}
8713   ins_encode %{
8714     __ mov($dst$$Register->successor(), AsmOperand($src1$$Register->successor(), asr, $src2$$Register));
8715   %}
8716   ins_pipe(ialu_reg_reg);
8717 %}
8718 
8719 instruct sarL_reg_reg_overlap(iRegL dst, iRegL src1, iRegI src2, flagsReg ccr) %{
8720   effect(DEF dst, USE src1, USE src2, KILL ccr);
8721   size(16);
8722   format %{ "SUBS  $dst.lo,$src2,32 \n\t"
8723             "ASRpl $dst.lo,$src1.hi,$dst.lo \n\t"
8724             "RSBmi $dst.lo,$dst.lo,0 \n\t"
8725             "LSLmi $dst.lo,$src1.hi,$dst.lo" %}
8726 
8727   ins_encode %{
8728     // $src1$$Register->successor() and $dst$$Register can't be the same
8729     __ subs($dst$$Register, $src2$$Register, 32);
8730     __ mov($dst$$Register, AsmOperand($src1$$Register->successor(), asr, $dst$$Register), pl);
8731     __ rsb($dst$$Register, $dst$$Register, 0, mi);
8732     __ mov($dst$$Register, AsmOperand($src1$$Register->successor(), lsl, $dst$$Register), mi);
8733   %}
8734   ins_pipe(ialu_reg_reg);
8735 %}
8736 #endif // !AARCH64
8737 
8738 instruct sarL_reg_reg(iRegL dst, iRegL src1, iRegI src2) %{
8739   match(Set dst (RShiftL src1 src2));
8740 
8741 #ifdef AARCH64
8742   size(4);
8743   format %{ "ASRV  $dst,$src1,$src2\t! long" %}
8744   ins_encode %{
8745     __ asrv($dst$$Register, $src1$$Register, $src2$$Register);
8746   %}
8747   ins_pipe(ialu_reg_reg);
8748 #else
8749   expand %{
8750     flagsReg ccr;
8751     sarL_reg_reg_overlap(dst, src1, src2, ccr);
8752     sarL_reg_reg_merge_lo(dst, src1, src2);
8753     sarL_reg_reg_merge_hi(dst, src1, src2);
8754   %}
8755 #endif
8756 %}
8757 
8758 // Register Shift Left Immediate
8759 #ifdef AARCH64
8760 instruct sarL_reg_imm6(iRegL dst, iRegL src1, immU6 src2) %{
8761   match(Set dst (RShiftL src1 src2));
8762 
8763   size(4);
8764   format %{ "ASR    $dst,$src1,$src2\t! long" %}
8765   ins_encode %{
8766     __ _asr($dst$$Register, $src1$$Register, $src2$$constant);
8767   %}
8768   ins_pipe(ialu_reg_imm);
8769 %}
8770 #else
8771 instruct sarL_reg_imm6(iRegL dst, iRegL src1, immU6Big src2) %{
8772   match(Set dst (RShiftL src1 src2));
8773 
8774   size(8);
8775   format %{ "ASR   $dst.lo,$src1.hi,$src2-32\t! or mov if $src2==32\n\t"
8776             "ASR   $dst.hi,$src1.hi, $src2" %}
8777   ins_encode %{
8778     if ($src2$$constant == 32) {
8779       __ mov($dst$$Register, $src1$$Register->successor());
8780     } else{
8781       __ mov($dst$$Register, AsmOperand($src1$$Register->successor(), asr, $src2$$constant-32));
8782     }
8783     __ mov($dst$$Register->successor(), AsmOperand($src1$$Register->successor(), asr, 0));
8784   %}
8785 
8786   ins_pipe(ialu_reg_imm);
8787 %}
8788 
8789 instruct sarL_reg_imm5(iRegL dst, iRegL src1, immU5 src2) %{
8790   match(Set dst (RShiftL src1 src2));
8791   size(12);
8792   format %{ "LSR   $dst.lo,$src1.lo,$src2\n\t"
8793             "OR    $dst.lo, $dst.lo, $src1.hi << 32-$src2\n\t"
8794             "ASR   $dst.hi,$src1.hi,$src2" %}
8795   ins_encode %{
8796     // The order of the following 3 instructions matters: src1.lo and
8797     // dst.hi can't overlap but src.hi and dst.hi can.
8798     __ mov($dst$$Register, AsmOperand($src1$$Register, lsr, $src2$$constant));
8799     __ orr($dst$$Register, $dst$$Register, AsmOperand($src1$$Register->successor(), lsl, 32-$src2$$constant));
8800     __ mov($dst$$Register->successor(), AsmOperand($src1$$Register->successor(), asr, $src2$$constant));
8801   %}
8802   ins_pipe(ialu_reg_imm);
8803 %}
8804 #endif
8805 
8806 // Register Shift Right
8807 instruct shrI_reg_reg(iRegI dst, iRegI src1, iRegI src2) %{
8808   match(Set dst (URShiftI src1 src2));
8809   size(4);
8810 #ifdef AARCH64
8811   format %{ "LSRV   $dst,$src1,$src2\t! int" %}
8812   ins_encode %{
8813     __ lsrv_w($dst$$Register, $src1$$Register, $src2$$Register);
8814   %}
8815 #else
8816   format %{ "LSR    $dst,$src1,$src2\t! int" %}
8817   ins_encode %{
8818     __ mov($dst$$Register, AsmOperand($src1$$Register, lsr, $src2$$Register));
8819   %}
8820 #endif
8821   ins_pipe(ialu_reg_reg);
8822 %}
8823 
8824 // Register Shift Right Immediate
8825 instruct shrI_reg_imm5(iRegI dst, iRegI src1, immU5 src2) %{
8826   match(Set dst (URShiftI src1 src2));
8827 
8828   size(4);
8829 #ifdef AARCH64
8830   format %{ "LSR_w  $dst,$src1,$src2" %}
8831   ins_encode %{
8832     __ _lsr_w($dst$$Register, $src1$$Register, $src2$$constant);
8833   %}
8834 #else
8835   format %{ "LSR    $dst,$src1,$src2" %}
8836   ins_encode %{
8837     __ mov($dst$$Register, AsmOperand($src1$$Register, lsr, $src2$$constant));
8838   %}
8839 #endif
8840   ins_pipe(ialu_reg_imm);
8841 %}
8842 
8843 #ifndef AARCH64
8844 // Register Shift Right
8845 instruct shrL_reg_reg_merge_lo(iRegL dst, iRegL src1, iRegI src2) %{
8846   effect(USE_DEF dst, USE src1, USE src2);
8847   size(4);
8848   format %{ "OR   $dst.lo,$dst,($src1.lo >>> $src2)"  %}
8849   ins_encode %{
8850     __ orr($dst$$Register, $dst$$Register, AsmOperand($src1$$Register, lsr, $src2$$Register));
8851   %}
8852   ins_pipe(ialu_reg_reg);
8853 %}
8854 
8855 instruct shrL_reg_reg_merge_hi(iRegL dst, iRegL src1, iRegI src2) %{
8856   effect(USE_DEF dst, USE src1, USE src2);
8857   size(4);
8858   format %{ "LSR  $dst.hi,$src1.hi,$src2 \n\t" %}
8859   ins_encode %{
8860     __ mov($dst$$Register->successor(), AsmOperand($src1$$Register->successor(), lsr, $src2$$Register));
8861   %}
8862   ins_pipe(ialu_reg_reg);
8863 %}
8864 
8865 instruct shrL_reg_reg_overlap(iRegL dst, iRegL src1, iRegI src2, flagsReg ccr) %{
8866   effect(DEF dst, USE src1, USE src2, KILL ccr);
8867   size(16);
8868   format %{ "SUBS  $dst,$src2,32 \n\t"
8869             "LSRpl $dst,$src1.hi,$dst \n\t"
8870             "RSBmi $dst,$dst,0 \n\t"
8871             "LSLmi $dst,$src1.hi,$dst" %}
8872 
8873   ins_encode %{
8874     // $src1$$Register->successor() and $dst$$Register can't be the same
8875     __ subs($dst$$Register, $src2$$Register, 32);
8876     __ mov($dst$$Register, AsmOperand($src1$$Register->successor(), lsr, $dst$$Register), pl);
8877     __ rsb($dst$$Register, $dst$$Register, 0, mi);
8878     __ mov($dst$$Register, AsmOperand($src1$$Register->successor(), lsl, $dst$$Register), mi);
8879   %}
8880   ins_pipe(ialu_reg_reg);
8881 %}
8882 #endif // !AARCH64
8883 
8884 instruct shrL_reg_reg(iRegL dst, iRegL src1, iRegI src2) %{
8885   match(Set dst (URShiftL src1 src2));
8886 
8887 #ifdef AARCH64
8888   size(4);
8889   format %{ "LSRV  $dst,$src1,$src2\t! long" %}
8890   ins_encode %{
8891     __ lsrv($dst$$Register, $src1$$Register, $src2$$Register);
8892   %}
8893   ins_pipe(ialu_reg_reg);
8894 #else
8895   expand %{
8896     flagsReg ccr;
8897     shrL_reg_reg_overlap(dst, src1, src2, ccr);
8898     shrL_reg_reg_merge_lo(dst, src1, src2);
8899     shrL_reg_reg_merge_hi(dst, src1, src2);
8900   %}
8901 #endif
8902 %}
8903 
8904 // Register Shift Right Immediate
8905 #ifdef AARCH64
8906 instruct shrL_reg_imm6(iRegL dst, iRegL src1, immU6 src2) %{
8907   match(Set dst (URShiftL src1 src2));
8908 
8909   size(4);
8910   format %{ "LSR    $dst,$src1,$src2" %}
8911   ins_encode %{
8912     __ _lsr($dst$$Register, $src1$$Register, $src2$$constant);
8913   %}
8914   ins_pipe(ialu_reg_imm);
8915 %}
8916 #else
8917 instruct shrL_reg_imm6(iRegL dst, iRegL src1, immU6Big src2) %{
8918   match(Set dst (URShiftL src1 src2));
8919 
8920   size(8);
8921   format %{ "LSR   $dst.lo,$src1.hi,$src2-32\t! or mov if $src2==32\n\t"
8922             "MOV   $dst.hi, 0" %}
8923   ins_encode %{
8924     if ($src2$$constant == 32) {
8925       __ mov($dst$$Register, $src1$$Register->successor());
8926     } else {
8927       __ mov($dst$$Register, AsmOperand($src1$$Register->successor(), lsr, $src2$$constant-32));
8928     }
8929     __ mov($dst$$Register->successor(), 0);
8930   %}
8931 
8932   ins_pipe(ialu_reg_imm);
8933 %}
8934 
8935 instruct shrL_reg_imm5(iRegL dst, iRegL src1, immU5 src2) %{
8936   match(Set dst (URShiftL src1 src2));
8937 
8938   size(12);
8939   format %{ "LSR   $dst.lo,$src1.lo,$src2\n\t"
8940             "OR    $dst.lo, $dst.lo, $src1.hi << 32-$src2\n\t"
8941             "LSR   $dst.hi,$src1.hi,$src2" %}
8942   ins_encode %{
8943     // The order of the following 3 instructions matters: src1.lo and
8944     // dst.hi can't overlap but src.hi and dst.hi can.
8945     __ mov($dst$$Register, AsmOperand($src1$$Register, lsr, $src2$$constant));
8946     __ orr($dst$$Register, $dst$$Register, AsmOperand($src1$$Register->successor(), lsl, 32-$src2$$constant));
8947     __ mov($dst$$Register->successor(), AsmOperand($src1$$Register->successor(), lsr, $src2$$constant));
8948   %}
8949   ins_pipe(ialu_reg_imm);
8950 %}
8951 #endif // !AARCH64
8952 
8953 
8954 instruct shrP_reg_imm5(iRegX dst, iRegP src1, immU5 src2) %{
8955   match(Set dst (URShiftI (CastP2X src1) src2));
8956   size(4);
8957   format %{ "LSR    $dst,$src1,$src2\t! Cast ptr $src1 to int and shift" %}
8958   ins_encode %{
8959     __ logical_shift_right($dst$$Register, $src1$$Register, $src2$$constant);
8960   %}
8961   ins_pipe(ialu_reg_imm);
8962 %}
8963 
8964 //----------Floating Point Arithmetic Instructions-----------------------------
8965 
8966 //  Add float single precision
8967 instruct addF_reg_reg(regF dst, regF src1, regF src2) %{
8968   match(Set dst (AddF src1 src2));
8969 
8970   size(4);
8971   format %{ "FADDS  $dst,$src1,$src2" %}
8972   ins_encode %{
8973     __ add_float($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister);
8974   %}
8975 
8976   ins_pipe(faddF_reg_reg);
8977 %}
8978 
8979 //  Add float double precision
8980 instruct addD_reg_reg(regD dst, regD src1, regD src2) %{
8981   match(Set dst (AddD src1 src2));
8982 
8983   size(4);
8984   format %{ "FADDD  $dst,$src1,$src2" %}
8985   ins_encode %{
8986     __ add_double($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister);
8987   %}
8988 
8989   ins_pipe(faddD_reg_reg);
8990 %}
8991 
8992 //  Sub float single precision
8993 instruct subF_reg_reg(regF dst, regF src1, regF src2) %{
8994   match(Set dst (SubF src1 src2));
8995 
8996   size(4);
8997   format %{ "FSUBS  $dst,$src1,$src2" %}
8998   ins_encode %{
8999     __ sub_float($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister);
9000   %}
9001   ins_pipe(faddF_reg_reg);
9002 %}
9003 
9004 //  Sub float double precision
9005 instruct subD_reg_reg(regD dst, regD src1, regD src2) %{
9006   match(Set dst (SubD src1 src2));
9007 
9008   size(4);
9009   format %{ "FSUBD  $dst,$src1,$src2" %}
9010   ins_encode %{
9011     __ sub_double($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister);
9012   %}
9013   ins_pipe(faddD_reg_reg);
9014 %}
9015 
9016 //  Mul float single precision
9017 instruct mulF_reg_reg(regF dst, regF src1, regF src2) %{
9018   match(Set dst (MulF src1 src2));
9019 
9020   size(4);
9021   format %{ "FMULS  $dst,$src1,$src2" %}
9022   ins_encode %{
9023     __ mul_float($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister);
9024   %}
9025 
9026   ins_pipe(fmulF_reg_reg);
9027 %}
9028 
9029 //  Mul float double precision
9030 instruct mulD_reg_reg(regD dst, regD src1, regD src2) %{
9031   match(Set dst (MulD src1 src2));
9032 
9033   size(4);
9034   format %{ "FMULD  $dst,$src1,$src2" %}
9035   ins_encode %{
9036     __ mul_double($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister);
9037   %}
9038 
9039   ins_pipe(fmulD_reg_reg);
9040 %}
9041 
9042 //  Div float single precision
9043 instruct divF_reg_reg(regF dst, regF src1, regF src2) %{
9044   match(Set dst (DivF src1 src2));
9045 
9046   size(4);
9047   format %{ "FDIVS  $dst,$src1,$src2" %}
9048   ins_encode %{
9049     __ div_float($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister);
9050   %}
9051 
9052   ins_pipe(fdivF_reg_reg);
9053 %}
9054 
9055 //  Div float double precision
9056 instruct divD_reg_reg(regD dst, regD src1, regD src2) %{
9057   match(Set dst (DivD src1 src2));
9058 
9059   size(4);
9060   format %{ "FDIVD  $dst,$src1,$src2" %}
9061   ins_encode %{
9062     __ div_double($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister);
9063   %}
9064 
9065   ins_pipe(fdivD_reg_reg);
9066 %}
9067 
9068 //  Absolute float double precision
9069 instruct absD_reg(regD dst, regD src) %{
9070   match(Set dst (AbsD src));
9071 
9072   size(4);
9073   format %{ "FABSd  $dst,$src" %}
9074   ins_encode %{
9075     __ abs_double($dst$$FloatRegister, $src$$FloatRegister);
9076   %}
9077   ins_pipe(faddD_reg);
9078 %}
9079 
9080 //  Absolute float single precision
9081 instruct absF_reg(regF dst, regF src) %{
9082   match(Set dst (AbsF src));
9083   format %{ "FABSs  $dst,$src" %}
9084   ins_encode %{
9085     __ abs_float($dst$$FloatRegister, $src$$FloatRegister);
9086   %}
9087   ins_pipe(faddF_reg);
9088 %}
9089 
9090 instruct negF_reg(regF dst, regF src) %{
9091   match(Set dst (NegF src));
9092 
9093   size(4);
9094   format %{ "FNEGs  $dst,$src" %}
9095   ins_encode %{
9096     __ neg_float($dst$$FloatRegister, $src$$FloatRegister);
9097   %}
9098   ins_pipe(faddF_reg);
9099 %}
9100 
9101 instruct negD_reg(regD dst, regD src) %{
9102   match(Set dst (NegD src));
9103 
9104   format %{ "FNEGd  $dst,$src" %}
9105   ins_encode %{
9106     __ neg_double($dst$$FloatRegister, $src$$FloatRegister);
9107   %}
9108   ins_pipe(faddD_reg);
9109 %}
9110 
9111 //  Sqrt float double precision
9112 instruct sqrtF_reg_reg(regF dst, regF src) %{
9113   match(Set dst (ConvD2F (SqrtD (ConvF2D src))));
9114 
9115   size(4);
9116   format %{ "FSQRTS $dst,$src" %}
9117   ins_encode %{
9118     __ sqrt_float($dst$$FloatRegister, $src$$FloatRegister);
9119   %}
9120   ins_pipe(fdivF_reg_reg);
9121 %}
9122 
9123 //  Sqrt float double precision
9124 instruct sqrtD_reg_reg(regD dst, regD src) %{
9125   match(Set dst (SqrtD src));
9126 
9127   size(4);
9128   format %{ "FSQRTD $dst,$src" %}
9129   ins_encode %{
9130     __ sqrt_double($dst$$FloatRegister, $src$$FloatRegister);
9131   %}
9132   ins_pipe(fdivD_reg_reg);
9133 %}
9134 
9135 //----------Logical Instructions-----------------------------------------------
9136 // And Instructions
9137 // Register And
9138 instruct andI_reg_reg(iRegI dst, iRegI src1, iRegI src2) %{
9139   match(Set dst (AndI src1 src2));
9140 
9141   size(4);
9142   format %{ "and_32 $dst,$src1,$src2" %}
9143   ins_encode %{
9144     __ and_32($dst$$Register, $src1$$Register, $src2$$Register);
9145   %}
9146   ins_pipe(ialu_reg_reg);
9147 %}
9148 
9149 #ifndef AARCH64
9150 instruct andshlI_reg_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3) %{
9151   match(Set dst (AndI src1 (LShiftI src2 src3)));
9152 
9153   size(4);
9154   format %{ "AND    $dst,$src1,$src2<<$src3" %}
9155   ins_encode %{
9156     __ andr($dst$$Register, $src1$$Register, AsmOperand($src2$$Register, lsl, $src3$$Register));
9157   %}
9158   ins_pipe(ialu_reg_reg);
9159 %}
9160 #endif
9161 
9162 instruct andshlI_reg_reg_imm(iRegI dst, iRegI src1, iRegI src2, immU5 src3) %{
9163   match(Set dst (AndI src1 (LShiftI src2 src3)));
9164 
9165   size(4);
9166   format %{ "and_32 $dst,$src1,$src2<<$src3" %}
9167   ins_encode %{
9168     __ and_32($dst$$Register, $src1$$Register, AsmOperand($src2$$Register, lsl, $src3$$constant));
9169   %}
9170   ins_pipe(ialu_reg_reg);
9171 %}
9172 
9173 #ifndef AARCH64
9174 instruct andsarI_reg_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3) %{
9175   match(Set dst (AndI src1 (RShiftI src2 src3)));
9176 
9177   size(4);
9178   format %{ "AND    $dst,$src1,$src2>>$src3" %}
9179   ins_encode %{
9180     __ andr($dst$$Register, $src1$$Register, AsmOperand($src2$$Register, asr, $src3$$Register));
9181   %}
9182   ins_pipe(ialu_reg_reg);
9183 %}
9184 #endif
9185 
9186 instruct andsarI_reg_reg_imm(iRegI dst, iRegI src1, iRegI src2, immU5 src3) %{
9187   match(Set dst (AndI src1 (RShiftI src2 src3)));
9188 
9189   size(4);
9190   format %{ "and_32 $dst,$src1,$src2>>$src3" %}
9191   ins_encode %{
9192     __ and_32($dst$$Register, $src1$$Register, AsmOperand($src2$$Register, asr, $src3$$constant));
9193   %}
9194   ins_pipe(ialu_reg_reg);
9195 %}
9196 
9197 #ifndef AARCH64
9198 instruct andshrI_reg_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3) %{
9199   match(Set dst (AndI src1 (URShiftI src2 src3)));
9200 
9201   size(4);
9202   format %{ "AND    $dst,$src1,$src2>>>$src3" %}
9203   ins_encode %{
9204     __ andr($dst$$Register, $src1$$Register, AsmOperand($src2$$Register, lsr, $src3$$Register));
9205   %}
9206   ins_pipe(ialu_reg_reg);
9207 %}
9208 #endif
9209 
9210 instruct andshrI_reg_reg_imm(iRegI dst, iRegI src1, iRegI src2, immU5 src3) %{
9211   match(Set dst (AndI src1 (URShiftI src2 src3)));
9212 
9213   size(4);
9214   format %{ "and_32 $dst,$src1,$src2>>>$src3" %}
9215   ins_encode %{
9216     __ and_32($dst$$Register, $src1$$Register, AsmOperand($src2$$Register, lsr, $src3$$constant));
9217   %}
9218   ins_pipe(ialu_reg_reg);
9219 %}
9220 
9221 // Immediate And
9222 instruct andI_reg_limm(iRegI dst, iRegI src1, limmI src2) %{
9223   match(Set dst (AndI src1 src2));
9224 
9225   size(4);
9226   format %{ "and_32 $dst,$src1,$src2\t! int" %}
9227   ins_encode %{
9228     __ and_32($dst$$Register, $src1$$Register, $src2$$constant);
9229   %}
9230   ins_pipe(ialu_reg_imm);
9231 %}
9232 
9233 #ifndef AARCH64
9234 instruct andI_reg_limmn(iRegI dst, iRegI src1, limmIn src2) %{
9235   match(Set dst (AndI src1 src2));
9236 
9237   size(4);
9238   format %{ "bic    $dst,$src1,~$src2\t! int" %}
9239   ins_encode %{
9240     __ bic($dst$$Register, $src1$$Register, ~$src2$$constant);
9241   %}
9242   ins_pipe(ialu_reg_imm);
9243 %}
9244 #endif
9245 
9246 // Register And Long
9247 instruct andL_reg_reg(iRegL dst, iRegL src1, iRegL src2) %{
9248   match(Set dst (AndL src1 src2));
9249 
9250   ins_cost(DEFAULT_COST);
9251 #ifdef AARCH64
9252   size(4);
9253   format %{ "AND    $dst,$src1,$src2\t! long" %}
9254   ins_encode %{
9255     __ andr($dst$$Register, $src1$$Register, $src2$$Register);
9256   %}
9257 #else
9258   size(8);
9259   format %{ "AND    $dst,$src1,$src2\t! long" %}
9260   ins_encode %{
9261     __ andr($dst$$Register, $src1$$Register, $src2$$Register);
9262     __ andr($dst$$Register->successor(), $src1$$Register->successor(), $src2$$Register->successor());
9263   %}
9264 #endif
9265   ins_pipe(ialu_reg_reg);
9266 %}
9267 
9268 #ifdef AARCH64
9269 // Immediate And
9270 instruct andL_reg_limm(iRegL dst, iRegL src1, limmL src2) %{
9271   match(Set dst (AndL src1 src2));
9272 
9273   size(4);
9274   format %{ "AND    $dst,$src1,$src2\t! long" %}
9275   ins_encode %{
9276     __ andr($dst$$Register, $src1$$Register, (uintx)$src2$$constant);
9277   %}
9278   ins_pipe(ialu_reg_imm);
9279 %}
9280 #else
9281 // TODO: try immLRot2 instead, (0, $con$$constant) becomes
9282 // (hi($con$$constant), lo($con$$constant)) becomes
9283 instruct andL_reg_immRot(iRegL dst, iRegL src1, immLlowRot con) %{
9284   match(Set dst (AndL src1 con));
9285   ins_cost(DEFAULT_COST);
9286   size(8);
9287   format %{ "AND    $dst,$src1,$con\t! long" %}
9288   ins_encode %{
9289     __ andr($dst$$Register, $src1$$Register, $con$$constant);
9290     __ andr($dst$$Register->successor(), $src1$$Register->successor(), 0);
9291   %}
9292   ins_pipe(ialu_reg_imm);
9293 %}
9294 #endif
9295 
9296 // Or Instructions
9297 // Register Or
9298 instruct orI_reg_reg(iRegI dst, iRegI src1, iRegI src2) %{
9299   match(Set dst (OrI src1 src2));
9300 
9301   size(4);
9302   format %{ "orr_32 $dst,$src1,$src2\t! int" %}
9303   ins_encode %{
9304     __ orr_32($dst$$Register, $src1$$Register, $src2$$Register);
9305   %}
9306   ins_pipe(ialu_reg_reg);
9307 %}
9308 
9309 #ifndef AARCH64
9310 instruct orshlI_reg_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3) %{
9311   match(Set dst (OrI src1 (LShiftI src2 src3)));
9312 
9313   size(4);
9314   format %{ "OR    $dst,$src1,$src2<<$src3" %}
9315   ins_encode %{
9316     __ orr($dst$$Register, $src1$$Register, AsmOperand($src2$$Register, lsl, $src3$$Register));
9317   %}
9318   ins_pipe(ialu_reg_reg);
9319 %}
9320 #endif
9321 
9322 instruct orshlI_reg_reg_imm(iRegI dst, iRegI src1, iRegI src2, immU5 src3) %{
9323   match(Set dst (OrI src1 (LShiftI src2 src3)));
9324 
9325   size(4);
9326   format %{ "orr_32 $dst,$src1,$src2<<$src3" %}
9327   ins_encode %{
9328     __ orr_32($dst$$Register, $src1$$Register, AsmOperand($src2$$Register, lsl, $src3$$constant));
9329   %}
9330   ins_pipe(ialu_reg_reg);
9331 %}
9332 
9333 #ifndef AARCH64
9334 instruct orsarI_reg_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3) %{
9335   match(Set dst (OrI src1 (RShiftI src2 src3)));
9336 
9337   size(4);
9338   format %{ "OR    $dst,$src1,$src2>>$src3" %}
9339   ins_encode %{
9340     __ orr($dst$$Register, $src1$$Register, AsmOperand($src2$$Register, asr, $src3$$Register));
9341   %}
9342   ins_pipe(ialu_reg_reg);
9343 %}
9344 #endif
9345 
9346 instruct orsarI_reg_reg_imm(iRegI dst, iRegI src1, iRegI src2, immU5 src3) %{
9347   match(Set dst (OrI src1 (RShiftI src2 src3)));
9348 
9349   size(4);
9350   format %{ "orr_32 $dst,$src1,$src2>>$src3" %}
9351   ins_encode %{
9352     __ orr_32($dst$$Register, $src1$$Register, AsmOperand($src2$$Register, asr, $src3$$constant));
9353   %}
9354   ins_pipe(ialu_reg_reg);
9355 %}
9356 
9357 #ifndef AARCH64
9358 instruct orshrI_reg_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3) %{
9359   match(Set dst (OrI src1 (URShiftI src2 src3)));
9360 
9361   size(4);
9362   format %{ "OR    $dst,$src1,$src2>>>$src3" %}
9363   ins_encode %{
9364     __ orr($dst$$Register, $src1$$Register, AsmOperand($src2$$Register, lsr, $src3$$Register));
9365   %}
9366   ins_pipe(ialu_reg_reg);
9367 %}
9368 #endif
9369 
9370 instruct orshrI_reg_reg_imm(iRegI dst, iRegI src1, iRegI src2, immU5 src3) %{
9371   match(Set dst (OrI src1 (URShiftI src2 src3)));
9372 
9373   size(4);
9374   format %{ "orr_32 $dst,$src1,$src2>>>$src3" %}
9375   ins_encode %{
9376     __ orr_32($dst$$Register, $src1$$Register, AsmOperand($src2$$Register, lsr, $src3$$constant));
9377   %}
9378   ins_pipe(ialu_reg_reg);
9379 %}
9380 
9381 // Immediate Or
9382 instruct orI_reg_limm(iRegI dst, iRegI src1, limmI src2) %{
9383   match(Set dst (OrI src1 src2));
9384 
9385   size(4);
9386   format %{ "orr_32  $dst,$src1,$src2" %}
9387   ins_encode %{
9388     __ orr_32($dst$$Register, $src1$$Register, $src2$$constant);
9389   %}
9390   ins_pipe(ialu_reg_imm);
9391 %}
9392 // TODO: orn_32 with limmIn
9393 
9394 // Register Or Long
9395 instruct orL_reg_reg(iRegL dst, iRegL src1, iRegL src2) %{
9396   match(Set dst (OrL src1 src2));
9397 
9398   ins_cost(DEFAULT_COST);
9399 #ifdef AARCH64
9400   size(4);
9401   format %{ "OR     $dst,$src1,$src2\t! long" %}
9402   ins_encode %{
9403     __ orr($dst$$Register, $src1$$Register, $src2$$Register);
9404   %}
9405 #else
9406   size(8);
9407   format %{ "OR     $dst.lo,$src1.lo,$src2.lo\t! long\n\t"
9408             "OR     $dst.hi,$src1.hi,$src2.hi" %}
9409   ins_encode %{
9410     __ orr($dst$$Register, $src1$$Register, $src2$$Register);
9411     __ orr($dst$$Register->successor(), $src1$$Register->successor(), $src2$$Register->successor());
9412   %}
9413 #endif
9414   ins_pipe(ialu_reg_reg);
9415 %}
9416 
9417 #ifdef AARCH64
9418 instruct orL_reg_limm(iRegL dst, iRegL src1, limmL src2) %{
9419   match(Set dst (OrL src1 src2));
9420 
9421   size(4);
9422   format %{ "ORR    $dst,$src1,$src2\t! long" %}
9423   ins_encode %{
9424     __ orr($dst$$Register, $src1$$Register, (uintx)$src2$$constant);
9425   %}
9426   ins_pipe(ialu_reg_imm);
9427 %}
9428 #else
9429 // TODO: try immLRot2 instead, (0, $con$$constant) becomes
9430 // (hi($con$$constant), lo($con$$constant)) becomes
9431 instruct orL_reg_immRot(iRegL dst, iRegL src1, immLlowRot con) %{
9432   match(Set dst (OrL src1 con));
9433   ins_cost(DEFAULT_COST);
9434   size(8);
9435   format %{ "OR     $dst.lo,$src1.lo,$con\t! long\n\t"
9436             "OR     $dst.hi,$src1.hi,$con" %}
9437   ins_encode %{
9438     __ orr($dst$$Register, $src1$$Register, $con$$constant);
9439     __ orr($dst$$Register->successor(), $src1$$Register->successor(), 0);
9440   %}
9441   ins_pipe(ialu_reg_imm);
9442 %}
9443 #endif
9444 
9445 #ifdef TODO
9446 // Use SPRegP to match Rthread (TLS register) without spilling.
9447 // Use store_ptr_RegP to match Rthread (TLS register) without spilling.
9448 // Use sp_ptr_RegP to match Rthread (TLS register) without spilling.
9449 instruct orI_reg_castP2X(iRegI dst, iRegI src1, sp_ptr_RegP src2) %{
9450   match(Set dst (OrI src1 (CastP2X src2)));
9451   size(4);
9452   format %{ "OR     $dst,$src1,$src2" %}
9453   ins_encode %{
9454     __ orr($dst$$Register, $src1$$Register, $src2$$Register);
9455   %}
9456   ins_pipe(ialu_reg_reg);
9457 %}
9458 #endif
9459 
9460 // Xor Instructions
9461 // Register Xor
9462 instruct xorI_reg_reg(iRegI dst, iRegI src1, iRegI src2) %{
9463   match(Set dst (XorI src1 src2));
9464 
9465   size(4);
9466   format %{ "eor_32 $dst,$src1,$src2" %}
9467   ins_encode %{
9468     __ eor_32($dst$$Register, $src1$$Register, $src2$$Register);
9469   %}
9470   ins_pipe(ialu_reg_reg);
9471 %}
9472 
9473 #ifndef AARCH64
9474 instruct xorshlI_reg_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3) %{
9475   match(Set dst (XorI src1 (LShiftI src2 src3)));
9476 
9477   size(4);
9478   format %{ "XOR    $dst,$src1,$src2<<$src3" %}
9479   ins_encode %{
9480     __ eor($dst$$Register, $src1$$Register, AsmOperand($src2$$Register, lsl, $src3$$Register));
9481   %}
9482   ins_pipe(ialu_reg_reg);
9483 %}
9484 #endif
9485 
9486 instruct xorshlI_reg_reg_imm(iRegI dst, iRegI src1, iRegI src2, immU5 src3) %{
9487   match(Set dst (XorI src1 (LShiftI src2 src3)));
9488 
9489   size(4);
9490   format %{ "eor_32 $dst,$src1,$src2<<$src3" %}
9491   ins_encode %{
9492     __ eor_32($dst$$Register, $src1$$Register, AsmOperand($src2$$Register, lsl, $src3$$constant));
9493   %}
9494   ins_pipe(ialu_reg_reg);
9495 %}
9496 
9497 #ifndef AARCH64
9498 instruct xorsarI_reg_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3) %{
9499   match(Set dst (XorI src1 (RShiftI src2 src3)));
9500 
9501   size(4);
9502   format %{ "XOR    $dst,$src1,$src2>>$src3" %}
9503   ins_encode %{
9504     __ eor($dst$$Register, $src1$$Register, AsmOperand($src2$$Register, asr, $src3$$Register));
9505   %}
9506   ins_pipe(ialu_reg_reg);
9507 %}
9508 #endif
9509 
9510 instruct xorsarI_reg_reg_imm(iRegI dst, iRegI src1, iRegI src2, immU5 src3) %{
9511   match(Set dst (XorI src1 (RShiftI src2 src3)));
9512 
9513   size(4);
9514   format %{ "eor_32 $dst,$src1,$src2>>$src3" %}
9515   ins_encode %{
9516     __ eor_32($dst$$Register, $src1$$Register, AsmOperand($src2$$Register, asr, $src3$$constant));
9517   %}
9518   ins_pipe(ialu_reg_reg);
9519 %}
9520 
9521 #ifndef AARCH64
9522 instruct xorshrI_reg_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3) %{
9523   match(Set dst (XorI src1 (URShiftI src2 src3)));
9524 
9525   size(4);
9526   format %{ "XOR    $dst,$src1,$src2>>>$src3" %}
9527   ins_encode %{
9528     __ eor($dst$$Register, $src1$$Register, AsmOperand($src2$$Register, lsr, $src3$$Register));
9529   %}
9530   ins_pipe(ialu_reg_reg);
9531 %}
9532 #endif
9533 
9534 instruct xorshrI_reg_reg_imm(iRegI dst, iRegI src1, iRegI src2, immU5 src3) %{
9535   match(Set dst (XorI src1 (URShiftI src2 src3)));
9536 
9537   size(4);
9538   format %{ "eor_32 $dst,$src1,$src2>>>$src3" %}
9539   ins_encode %{
9540     __ eor_32($dst$$Register, $src1$$Register, AsmOperand($src2$$Register, lsr, $src3$$constant));
9541   %}
9542   ins_pipe(ialu_reg_reg);
9543 %}
9544 
9545 // Immediate Xor
9546 instruct xorI_reg_imm(iRegI dst, iRegI src1, limmI src2) %{
9547   match(Set dst (XorI src1 src2));
9548 
9549   size(4);
9550   format %{ "eor_32 $dst,$src1,$src2" %}
9551   ins_encode %{
9552     __ eor_32($dst$$Register, $src1$$Register, $src2$$constant);
9553   %}
9554   ins_pipe(ialu_reg_imm);
9555 %}
9556 
9557 // Register Xor Long
9558 instruct xorL_reg_reg(iRegL dst, iRegL src1, iRegL src2) %{
9559   match(Set dst (XorL src1 src2));
9560   ins_cost(DEFAULT_COST);
9561 #ifdef AARCH64
9562   size(4);
9563   format %{ "XOR     $dst,$src1,$src2\t! long" %}
9564   ins_encode %{
9565     __ eor($dst$$Register, $src1$$Register, $src2$$Register);
9566   %}
9567 #else
9568   size(8);
9569   format %{ "XOR     $dst.hi,$src1.hi,$src2.hi\t! long\n\t"
9570             "XOR     $dst.lo,$src1.lo,$src2.lo\t! long" %}
9571   ins_encode %{
9572     __ eor($dst$$Register, $src1$$Register, $src2$$Register);
9573     __ eor($dst$$Register->successor(), $src1$$Register->successor(), $src2$$Register->successor());
9574   %}
9575 #endif
9576   ins_pipe(ialu_reg_reg);
9577 %}
9578 
9579 #ifdef AARCH64
9580 instruct xorL_reg_limmL(iRegL dst, iRegL src1, limmL con) %{
9581   match(Set dst (XorL src1 con));
9582   ins_cost(DEFAULT_COST);
9583   size(4);
9584   format %{ "EOR     $dst,$src1,$con\t! long" %}
9585   ins_encode %{
9586     __ eor($dst$$Register, $src1$$Register, (uintx)$con$$constant);
9587   %}
9588   ins_pipe(ialu_reg_imm);
9589 %}
9590 #else
9591 // TODO: try immLRot2 instead, (0, $con$$constant) becomes
9592 // (hi($con$$constant), lo($con$$constant)) becomes
9593 instruct xorL_reg_immRot(iRegL dst, iRegL src1, immLlowRot con) %{
9594   match(Set dst (XorL src1 con));
9595   ins_cost(DEFAULT_COST);
9596   size(8);
9597   format %{ "XOR     $dst.hi,$src1.hi,$con\t! long\n\t"
9598             "XOR     $dst.lo,$src1.lo,0\t! long" %}
9599   ins_encode %{
9600     __ eor($dst$$Register, $src1$$Register, $con$$constant);
9601     __ eor($dst$$Register->successor(), $src1$$Register->successor(), 0);
9602   %}
9603   ins_pipe(ialu_reg_imm);
9604 %}
9605 #endif // AARCH64
9606 
9607 //----------Convert to Boolean-------------------------------------------------
9608 instruct convI2B( iRegI dst, iRegI src, flagsReg ccr ) %{
9609   match(Set dst (Conv2B src));
9610   effect(KILL ccr);
9611 #ifdef AARCH64
9612   size(8);
9613   ins_cost(DEFAULT_COST*2);
9614   format %{ "cmp_32 $src,ZR\n\t"
9615             "cset_w $dst, ne" %}
9616   ins_encode %{
9617     __ cmp_32($src$$Register, ZR);
9618     __ cset_w($dst$$Register, ne);
9619   %}
9620 #else
9621   size(12);
9622   ins_cost(DEFAULT_COST*2);
9623   format %{ "TST    $src,$src \n\t"
9624             "MOV    $dst, 0   \n\t"
9625             "MOV.ne $dst, 1" %}
9626   ins_encode %{ // FIXME: can do better?
9627     __ tst($src$$Register, $src$$Register);
9628     __ mov($dst$$Register, 0);
9629     __ mov($dst$$Register, 1, ne);
9630   %}
9631 #endif
9632   ins_pipe(ialu_reg_ialu);
9633 %}
9634 
9635 instruct convP2B( iRegI dst, iRegP src, flagsReg ccr ) %{
9636   match(Set dst (Conv2B src));
9637   effect(KILL ccr);
9638 #ifdef AARCH64
9639   size(8);
9640   ins_cost(DEFAULT_COST*2);
9641   format %{ "CMP    $src,ZR\n\t"
9642             "cset   $dst, ne" %}
9643   ins_encode %{
9644     __ cmp($src$$Register, ZR);
9645     __ cset($dst$$Register, ne);
9646   %}
9647 #else
9648   size(12);
9649   ins_cost(DEFAULT_COST*2);
9650   format %{ "TST    $src,$src \n\t"
9651             "MOV    $dst, 0   \n\t"
9652             "MOV.ne $dst, 1" %}
9653   ins_encode %{
9654     __ tst($src$$Register, $src$$Register);
9655     __ mov($dst$$Register, 0);
9656     __ mov($dst$$Register, 1, ne);
9657   %}
9658 #endif
9659   ins_pipe(ialu_reg_ialu);
9660 %}
9661 
9662 instruct cmpLTMask_reg_reg( iRegI dst, iRegI p, iRegI q, flagsReg ccr ) %{
9663   match(Set dst (CmpLTMask p q));
9664   effect( KILL ccr );
9665 #ifdef AARCH64
9666   size(8);
9667   ins_cost(DEFAULT_COST*2);
9668   format %{ "CMP_w   $p,$q\n\t"
9669             "CSETM_w $dst, lt" %}
9670   ins_encode %{
9671     __ cmp_w($p$$Register, $q$$Register);
9672     __ csetm_w($dst$$Register, lt);
9673   %}
9674 #else
9675   ins_cost(DEFAULT_COST*3);
9676   format %{ "CMP    $p,$q\n\t"
9677             "MOV    $dst, #0\n\t"
9678             "MOV.lt $dst, #-1" %}
9679   ins_encode %{
9680     __ cmp($p$$Register, $q$$Register);
9681     __ mov($dst$$Register, 0);
9682     __ mvn($dst$$Register, 0, lt);
9683   %}
9684 #endif
9685   ins_pipe(ialu_reg_reg_ialu);
9686 %}
9687 
9688 instruct cmpLTMask_reg_imm( iRegI dst, iRegI p, aimmI q, flagsReg ccr ) %{
9689   match(Set dst (CmpLTMask p q));
9690   effect( KILL ccr );
9691 #ifdef AARCH64
9692   size(8);
9693   ins_cost(DEFAULT_COST*2);
9694   format %{ "CMP_w   $p,$q\n\t"
9695             "CSETM_w $dst, lt" %}
9696   ins_encode %{
9697     __ cmp_w($p$$Register, $q$$constant);
9698     __ csetm_w($dst$$Register, lt);
9699   %}
9700 #else
9701   ins_cost(DEFAULT_COST*3);
9702   format %{ "CMP    $p,$q\n\t"
9703             "MOV    $dst, #0\n\t"
9704             "MOV.lt $dst, #-1" %}
9705   ins_encode %{
9706     __ cmp($p$$Register, $q$$constant);
9707     __ mov($dst$$Register, 0);
9708     __ mvn($dst$$Register, 0, lt);
9709   %}
9710 #endif
9711   ins_pipe(ialu_reg_reg_ialu);
9712 %}
9713 
9714 #ifdef AARCH64
9715 instruct cadd_cmpLTMask3( iRegI dst, iRegI p, iRegI q, iRegI y, iRegI x, flagsReg ccr ) %{
9716   match(Set dst (AddI (AndI (CmpLTMask p q) y) x));
9717   effect( TEMP dst, KILL ccr );
9718   size(12);
9719   ins_cost(DEFAULT_COST*3);
9720   format %{ "CMP_w  $p,$q\n\t"
9721             "ADD_w  $dst,$y,$x\n\t"
9722             "CSEL_w $dst,$dst,$x,lt" %}
9723   ins_encode %{
9724     __ cmp_w($p$$Register, $q$$Register);
9725     __ add_w($dst$$Register, $y$$Register, $x$$Register);
9726     __ csel_w($dst$$Register, $dst$$Register, $x$$Register, lt);
9727   %}
9728   ins_pipe( cadd_cmpltmask );
9729 %}
9730 #else
9731 instruct cadd_cmpLTMask3( iRegI p, iRegI q, iRegI y, iRegI z, flagsReg ccr ) %{
9732   match(Set z (AddI (AndI (CmpLTMask p q) y) z));
9733   effect( KILL ccr );
9734   ins_cost(DEFAULT_COST*2);
9735   format %{ "CMP    $p,$q\n\t"
9736             "ADD.lt $z,$y,$z" %}
9737   ins_encode %{
9738     __ cmp($p$$Register, $q$$Register);
9739     __ add($z$$Register, $y$$Register, $z$$Register, lt);
9740   %}
9741   ins_pipe( cadd_cmpltmask );
9742 %}
9743 #endif
9744 
9745 #ifdef AARCH64
9746 instruct cadd_cmpLTMask4( iRegI dst, iRegI p, aimmI q, iRegI y, iRegI x, flagsReg ccr ) %{
9747   match(Set dst (AddI (AndI (CmpLTMask p q) y) x));
9748   effect( TEMP dst, KILL ccr );
9749   size(12);
9750   ins_cost(DEFAULT_COST*3);
9751   format %{ "CMP_w  $p,$q\n\t"
9752             "ADD_w  $dst,$y,$x\n\t"
9753             "CSEL_w $dst,$dst,$x,lt" %}
9754   ins_encode %{
9755     __ cmp_w($p$$Register, $q$$constant);
9756     __ add_w($dst$$Register, $y$$Register, $x$$Register);
9757     __ csel_w($dst$$Register, $dst$$Register, $x$$Register, lt);
9758   %}
9759   ins_pipe( cadd_cmpltmask );
9760 %}
9761 #else
9762 // FIXME: remove unused "dst"
9763 instruct cadd_cmpLTMask4( iRegI dst, iRegI p, aimmI q, iRegI y, iRegI z, flagsReg ccr ) %{
9764   match(Set z (AddI (AndI (CmpLTMask p q) y) z));
9765   effect( KILL ccr );
9766   ins_cost(DEFAULT_COST*2);
9767   format %{ "CMP    $p,$q\n\t"
9768             "ADD.lt $z,$y,$z" %}
9769   ins_encode %{
9770     __ cmp($p$$Register, $q$$constant);
9771     __ add($z$$Register, $y$$Register, $z$$Register, lt);
9772   %}
9773   ins_pipe( cadd_cmpltmask );
9774 %}
9775 #endif // !AARCH64
9776 
9777 #ifdef AARCH64
9778 instruct cadd_cmpLTMask( iRegI dst, iRegI p, iRegI q, iRegI y, flagsReg ccr ) %{
9779   match(Set dst (AddI (AndI (CmpLTMask p q) y) (SubI p q)));
9780   effect( TEMP dst, KILL ccr );
9781   size(12);
9782   ins_cost(DEFAULT_COST*3);
9783   format %{ "SUBS_w $p,$p,$q\n\t"
9784             "ADD_w  $dst,$y,$p\n\t"
9785             "CSEL_w $dst,$dst,$p,lt" %}
9786   ins_encode %{
9787     __ subs_w($p$$Register, $p$$Register, $q$$Register);
9788     __ add_w($dst$$Register, $y$$Register, $p$$Register);
9789     __ csel_w($dst$$Register, $dst$$Register, $p$$Register, lt);
9790   %}
9791   ins_pipe( cadd_cmpltmask ); // FIXME
9792 %}
9793 #else
9794 instruct cadd_cmpLTMask( iRegI p, iRegI q, iRegI y, flagsReg ccr ) %{
9795   match(Set p (AddI (AndI (CmpLTMask p q) y) (SubI p q)));
9796   effect( KILL ccr );
9797   ins_cost(DEFAULT_COST*2);
9798   format %{ "SUBS   $p,$p,$q\n\t"
9799             "ADD.lt $p,$y,$p" %}
9800   ins_encode %{
9801     __ subs($p$$Register, $p$$Register, $q$$Register);
9802     __ add($p$$Register, $y$$Register, $p$$Register, lt);
9803   %}
9804   ins_pipe( cadd_cmpltmask );
9805 %}
9806 #endif
9807 
9808 //----------Arithmetic Conversion Instructions---------------------------------
9809 // The conversions operations are all Alpha sorted.  Please keep it that way!
9810 
9811 instruct convD2F_reg(regF dst, regD src) %{
9812   match(Set dst (ConvD2F src));
9813   size(4);
9814   format %{ "FCVTSD  $dst,$src" %}
9815   ins_encode %{
9816     __ convert_d2f($dst$$FloatRegister, $src$$FloatRegister);
9817   %}
9818   ins_pipe(fcvtD2F);
9819 %}
9820 
9821 // Convert a double to an int in a float register.
9822 // If the double is a NAN, stuff a zero in instead.
9823 
9824 #ifdef AARCH64
9825 instruct convD2I_reg_reg(iRegI dst, regD src) %{
9826   match(Set dst (ConvD2I src));
9827   ins_cost(DEFAULT_COST*2 + MEMORY_REF_COST*2 + BRANCH_COST); // FIXME
9828   format %{ "FCVTZS_wd $dst, $src" %}
9829   ins_encode %{
9830     __ fcvtzs_wd($dst$$Register, $src$$FloatRegister);
9831   %}
9832   ins_pipe(fcvtD2I);
9833 %}
9834 
9835 instruct convD2L_reg_reg(iRegL dst, regD src) %{
9836   match(Set dst (ConvD2L src));
9837   ins_cost(DEFAULT_COST*2 + MEMORY_REF_COST*2 + BRANCH_COST); // FIXME
9838   format %{ "FCVTZS_xd $dst, $src" %}
9839   ins_encode %{
9840     __ fcvtzs_xd($dst$$Register, $src$$FloatRegister);
9841   %}
9842   ins_pipe(fcvtD2L);
9843 %}
9844 #else
9845 instruct convD2I_reg_reg(iRegI dst, regD src, regF tmp) %{
9846   match(Set dst (ConvD2I src));
9847   effect( TEMP tmp );
9848   ins_cost(DEFAULT_COST*2 + MEMORY_REF_COST*2 + BRANCH_COST); // FIXME
9849   format %{ "FTOSIZD  $tmp,$src\n\t"
9850             "FMRS     $dst, $tmp" %}
9851   ins_encode %{
9852     __ ftosizd($tmp$$FloatRegister, $src$$FloatRegister);
9853     __ fmrs($dst$$Register, $tmp$$FloatRegister);
9854   %}
9855   ins_pipe(fcvtD2I);
9856 %}
9857 #endif
9858 
9859 // Convert a double to a long in a double register.
9860 // If the double is a NAN, stuff a zero in instead.
9861 
9862 #ifndef AARCH64
9863 // Double to Long conversion
9864 instruct convD2L_reg(R0R1RegL dst, regD src) %{
9865   match(Set dst (ConvD2L src));
9866   effect(CALL);
9867   ins_cost(MEMORY_REF_COST); // FIXME
9868   format %{ "convD2L    $dst,$src\t ! call to SharedRuntime::d2l" %}
9869   ins_encode %{
9870 #ifndef __ABI_HARD__
9871     __ fmrrd($dst$$Register, $dst$$Register->successor(), $src$$FloatRegister);
9872 #else
9873     if ($src$$FloatRegister != D0) {
9874       __ mov_double(D0, $src$$FloatRegister);
9875     }
9876 #endif
9877     address target = CAST_FROM_FN_PTR(address, SharedRuntime::d2l);
9878     __ call(target, relocInfo::runtime_call_type);
9879   %}
9880   ins_pipe(fcvtD2L);
9881 %}
9882 #endif
9883 
9884 instruct convF2D_reg(regD dst, regF src) %{
9885   match(Set dst (ConvF2D src));
9886   size(4);
9887   format %{ "FCVTDS  $dst,$src" %}
9888   ins_encode %{
9889     __ convert_f2d($dst$$FloatRegister, $src$$FloatRegister);
9890   %}
9891   ins_pipe(fcvtF2D);
9892 %}
9893 
9894 #ifdef AARCH64
9895 instruct convF2I_reg_reg(iRegI dst, regF src) %{
9896   match(Set dst (ConvF2I src));
9897   ins_cost(DEFAULT_COST*2 + MEMORY_REF_COST*2 + BRANCH_COST); // FIXME
9898   size(4);
9899   format %{ "FCVTZS_ws $dst, $src" %}
9900   ins_encode %{
9901     __ fcvtzs_ws($dst$$Register, $src$$FloatRegister);
9902   %}
9903   ins_pipe(fcvtF2I);
9904 %}
9905 
9906 instruct convF2L_reg_reg(iRegL dst, regF src) %{
9907   match(Set dst (ConvF2L src));
9908   ins_cost(DEFAULT_COST*2 + MEMORY_REF_COST*2 + BRANCH_COST); // FIXME
9909   size(4);
9910   format %{ "FCVTZS_xs $dst, $src" %}
9911   ins_encode %{
9912     __ fcvtzs_xs($dst$$Register, $src$$FloatRegister);
9913   %}
9914   ins_pipe(fcvtF2L);
9915 %}
9916 #else
9917 instruct convF2I_reg_reg(iRegI dst, regF src, regF tmp) %{
9918   match(Set dst (ConvF2I src));
9919   effect( TEMP tmp );
9920   ins_cost(DEFAULT_COST*2 + MEMORY_REF_COST*2 + BRANCH_COST); // FIXME
9921   size(8);
9922   format %{ "FTOSIZS  $tmp,$src\n\t"
9923             "FMRS     $dst, $tmp" %}
9924   ins_encode %{
9925     __ ftosizs($tmp$$FloatRegister, $src$$FloatRegister);
9926     __ fmrs($dst$$Register, $tmp$$FloatRegister);
9927   %}
9928   ins_pipe(fcvtF2I);
9929 %}
9930 
9931 // Float to Long conversion
9932 instruct convF2L_reg(R0R1RegL dst, regF src, R0RegI arg1) %{
9933   match(Set dst (ConvF2L src));
9934   ins_cost(DEFAULT_COST*2 + MEMORY_REF_COST*2 + BRANCH_COST); // FIXME
9935   effect(CALL);
9936   format %{ "convF2L  $dst,$src\t! call to SharedRuntime::f2l" %}
9937   ins_encode %{
9938 #ifndef __ABI_HARD__
9939     __ fmrs($arg1$$Register, $src$$FloatRegister);
9940 #else
9941     if($src$$FloatRegister != S0) {
9942       __ mov_float(S0, $src$$FloatRegister);
9943     }
9944 #endif
9945     address target = CAST_FROM_FN_PTR(address, SharedRuntime::f2l);
9946     __ call(target, relocInfo::runtime_call_type);
9947   %}
9948   ins_pipe(fcvtF2L);
9949 %}
9950 #endif
9951 
9952 #ifdef AARCH64
9953 instruct convI2D_reg_reg(iRegI src, regD dst) %{
9954   match(Set dst (ConvI2D src));
9955   ins_cost(DEFAULT_COST + MEMORY_REF_COST); // FIXME
9956   size(4);
9957   format %{ "SCVTF_dw $dst,$src" %}
9958   ins_encode %{
9959       __ scvtf_dw($dst$$FloatRegister, $src$$Register);
9960   %}
9961   ins_pipe(fcvtI2D);
9962 %}
9963 #else
9964 instruct convI2D_reg_reg(iRegI src, regD_low dst) %{
9965   match(Set dst (ConvI2D src));
9966   ins_cost(DEFAULT_COST + MEMORY_REF_COST); // FIXME
9967   size(8);
9968   format %{ "FMSR     $dst,$src \n\t"
9969             "FSITOD   $dst $dst"%}
9970   ins_encode %{
9971       __ fmsr($dst$$FloatRegister, $src$$Register);
9972       __ fsitod($dst$$FloatRegister, $dst$$FloatRegister);
9973   %}
9974   ins_pipe(fcvtI2D);
9975 %}
9976 #endif
9977 
9978 instruct convI2F_reg_reg( regF dst, iRegI src ) %{
9979   match(Set dst (ConvI2F src));
9980   ins_cost(DEFAULT_COST + MEMORY_REF_COST); // FIXME
9981 #ifdef AARCH64
9982   size(4);
9983   format %{ "SCVTF_sw $dst,$src" %}
9984   ins_encode %{
9985       __ scvtf_sw($dst$$FloatRegister, $src$$Register);
9986   %}
9987 #else
9988   size(8);
9989   format %{ "FMSR     $dst,$src \n\t"
9990             "FSITOS   $dst, $dst"%}
9991   ins_encode %{
9992       __ fmsr($dst$$FloatRegister, $src$$Register);
9993       __ fsitos($dst$$FloatRegister, $dst$$FloatRegister);
9994   %}
9995 #endif
9996   ins_pipe(fcvtI2F);
9997 %}
9998 
9999 instruct convI2L_reg(iRegL dst, iRegI src) %{
10000   match(Set dst (ConvI2L src));
10001 #ifdef AARCH64
10002   size(4);
10003   format %{ "SXTW   $dst,$src\t! int->long" %}
10004   ins_encode %{
10005     __ sxtw($dst$$Register, $src$$Register);
10006   %}
10007 #else
10008   size(8);
10009   format %{ "MOV    $dst.lo, $src \n\t"
10010             "ASR    $dst.hi,$src,31\t! int->long" %}
10011   ins_encode %{
10012     __ mov($dst$$Register, $src$$Register);
10013     __ mov($dst$$Register->successor(), AsmOperand($src$$Register, asr, 31));
10014   %}
10015 #endif
10016   ins_pipe(ialu_reg_reg);
10017 %}
10018 
10019 // Zero-extend convert int to long
10020 instruct convI2L_reg_zex(iRegL dst, iRegI src, immL_32bits mask ) %{
10021   match(Set dst (AndL (ConvI2L src) mask) );
10022 #ifdef AARCH64
10023   size(4);
10024   format %{ "mov_w  $dst,$src\t! zero-extend int to long"  %}
10025   ins_encode %{
10026     __ mov_w($dst$$Register, $src$$Register);
10027   %}
10028 #else
10029   size(8);
10030   format %{ "MOV    $dst.lo,$src.lo\t! zero-extend int to long\n\t"
10031             "MOV    $dst.hi, 0"%}
10032   ins_encode %{
10033     __ mov($dst$$Register, $src$$Register);
10034     __ mov($dst$$Register->successor(), 0);
10035   %}
10036 #endif
10037   ins_pipe(ialu_reg_reg);
10038 %}
10039 
10040 // Zero-extend long
10041 instruct zerox_long(iRegL dst, iRegL src, immL_32bits mask ) %{
10042   match(Set dst (AndL src mask) );
10043 #ifdef AARCH64
10044   size(4);
10045   format %{ "mov_w  $dst,$src\t! zero-extend long"  %}
10046   ins_encode %{
10047     __ mov_w($dst$$Register, $src$$Register);
10048   %}
10049 #else
10050   size(8);
10051   format %{ "MOV    $dst.lo,$src.lo\t! zero-extend long\n\t"
10052             "MOV    $dst.hi, 0"%}
10053   ins_encode %{
10054     __ mov($dst$$Register, $src$$Register);
10055     __ mov($dst$$Register->successor(), 0);
10056   %}
10057 #endif
10058   ins_pipe(ialu_reg_reg);
10059 %}
10060 
10061 instruct MoveF2I_reg_reg(iRegI dst, regF src) %{
10062   match(Set dst (MoveF2I src));
10063   effect(DEF dst, USE src);
10064   ins_cost(MEMORY_REF_COST); // FIXME
10065 
10066   size(4);
10067   format %{ "FMRS   $dst,$src\t! MoveF2I" %}
10068   ins_encode %{
10069     __ fmrs($dst$$Register, $src$$FloatRegister);
10070   %}
10071   ins_pipe(iload_mem); // FIXME
10072 %}
10073 
10074 instruct MoveI2F_reg_reg(regF dst, iRegI src) %{
10075   match(Set dst (MoveI2F src));
10076   ins_cost(MEMORY_REF_COST); // FIXME
10077 
10078   size(4);
10079   format %{ "FMSR   $dst,$src\t! MoveI2F" %}
10080   ins_encode %{
10081     __ fmsr($dst$$FloatRegister, $src$$Register);
10082   %}
10083   ins_pipe(iload_mem); // FIXME
10084 %}
10085 
10086 instruct MoveD2L_reg_reg(iRegL dst, regD src) %{
10087   match(Set dst (MoveD2L src));
10088   effect(DEF dst, USE src);
10089   ins_cost(MEMORY_REF_COST); // FIXME
10090 
10091   size(4);
10092 #ifdef AARCH64
10093   format %{ "FMOV_xd  $dst,$src\t! MoveD2L" %}
10094   ins_encode %{
10095     __ fmov_xd($dst$$Register, $src$$FloatRegister);
10096   %}
10097 #else
10098   format %{ "FMRRD    $dst,$src\t! MoveD2L" %}
10099   ins_encode %{
10100     __ fmrrd($dst$$Register, $dst$$Register->successor(), $src$$FloatRegister);
10101   %}
10102 #endif
10103   ins_pipe(iload_mem); // FIXME
10104 %}
10105 
10106 instruct MoveL2D_reg_reg(regD dst, iRegL src) %{
10107   match(Set dst (MoveL2D src));
10108   effect(DEF dst, USE src);
10109   ins_cost(MEMORY_REF_COST); // FIXME
10110 
10111   size(4);
10112 #ifdef AARCH64
10113   format %{ "FMOV_dx $dst,$src\t! MoveL2D" %}
10114   ins_encode %{
10115     __ fmov_dx($dst$$FloatRegister, $src$$Register);
10116   %}
10117 #else
10118   format %{ "FMDRR   $dst,$src\t! MoveL2D" %}
10119   ins_encode %{
10120     __ fmdrr($dst$$FloatRegister, $src$$Register, $src$$Register->successor());
10121   %}
10122 #endif
10123   ins_pipe(ialu_reg_reg); // FIXME
10124 %}
10125 
10126 //-----------
10127 // Long to Double conversion
10128 
10129 #ifdef AARCH64
10130 instruct convL2D(regD dst, iRegL src) %{
10131   match(Set dst (ConvL2D src));
10132   ins_cost(DEFAULT_COST*2 + MEMORY_REF_COST*2 + BRANCH_COST); // FIXME
10133   size(4);
10134   format %{ "SCVTF_dx $dst, $src" %}
10135   ins_encode %{
10136     __ scvtf_dx($dst$$FloatRegister, $src$$Register);
10137   %}
10138   ins_pipe(fcvtL2D);
10139 %}
10140 
10141 instruct convL2F(regF dst, iRegL src) %{
10142   match(Set dst (ConvL2F src));
10143   ins_cost(DEFAULT_COST*2 + MEMORY_REF_COST*2 + BRANCH_COST); // FIXME
10144   size(4);
10145   format %{ "SCVTF_sx $dst, $src" %}
10146   ins_encode %{
10147     __ scvtf_sx($dst$$FloatRegister, $src$$Register);
10148   %}
10149   ins_pipe(fcvtL2F);
10150 %}
10151 #else
10152 // Magic constant, 0x43300000
10153 instruct loadConI_x43300000(iRegI dst) %{
10154   effect(DEF dst);
10155   size(8);
10156   format %{ "MOV_SLOW  $dst,0x43300000\t! 2^52" %}
10157   ins_encode %{
10158     __ mov_slow($dst$$Register, 0x43300000);
10159   %}
10160   ins_pipe(ialu_none);
10161 %}
10162 
10163 // Magic constant, 0x41f00000
10164 instruct loadConI_x41f00000(iRegI dst) %{
10165   effect(DEF dst);
10166   size(8);
10167   format %{ "MOV_SLOW  $dst, 0x41f00000\t! 2^32" %}
10168   ins_encode %{
10169     __ mov_slow($dst$$Register, 0x41f00000);
10170   %}
10171   ins_pipe(ialu_none);
10172 %}
10173 
10174 instruct loadConI_x0(iRegI dst) %{
10175   effect(DEF dst);
10176   size(4);
10177   format %{ "MOV  $dst, 0x0\t! 0" %}
10178   ins_encode %{
10179     __ mov($dst$$Register, 0);
10180   %}
10181   ins_pipe(ialu_none);
10182 %}
10183 
10184 // Construct a double from two float halves
10185 instruct regDHi_regDLo_to_regD(regD_low dst, regD_low src1, regD_low src2) %{
10186   effect(DEF dst, USE src1, USE src2);
10187   size(8);
10188   format %{ "FCPYS  $dst.hi,$src1.hi\n\t"
10189             "FCPYS  $dst.lo,$src2.lo" %}
10190   ins_encode %{
10191     __ fcpys($dst$$FloatRegister->successor(), $src1$$FloatRegister->successor());
10192     __ fcpys($dst$$FloatRegister, $src2$$FloatRegister);
10193   %}
10194   ins_pipe(faddD_reg_reg);
10195 %}
10196 
10197 #ifndef AARCH64
10198 // Convert integer in high half of a double register (in the lower half of
10199 // the double register file) to double
10200 instruct convI2D_regDHi_regD(regD dst, regD_low src) %{
10201   effect(DEF dst, USE src);
10202   size(4);
10203   format %{ "FSITOD  $dst,$src" %}
10204   ins_encode %{
10205     __ fsitod($dst$$FloatRegister, $src$$FloatRegister->successor());
10206   %}
10207   ins_pipe(fcvtLHi2D);
10208 %}
10209 #endif
10210 
10211 // Add float double precision
10212 instruct addD_regD_regD(regD dst, regD src1, regD src2) %{
10213   effect(DEF dst, USE src1, USE src2);
10214   size(4);
10215   format %{ "FADDD  $dst,$src1,$src2" %}
10216   ins_encode %{
10217     __ add_double($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister);
10218   %}
10219   ins_pipe(faddD_reg_reg);
10220 %}
10221 
10222 // Sub float double precision
10223 instruct subD_regD_regD(regD dst, regD src1, regD src2) %{
10224   effect(DEF dst, USE src1, USE src2);
10225   size(4);
10226   format %{ "FSUBD  $dst,$src1,$src2" %}
10227   ins_encode %{
10228     __ sub_double($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister);
10229   %}
10230   ins_pipe(faddD_reg_reg);
10231 %}
10232 
10233 // Mul float double precision
10234 instruct mulD_regD_regD(regD dst, regD src1, regD src2) %{
10235   effect(DEF dst, USE src1, USE src2);
10236   size(4);
10237   format %{ "FMULD  $dst,$src1,$src2" %}
10238   ins_encode %{
10239     __ mul_double($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister);
10240   %}
10241   ins_pipe(fmulD_reg_reg);
10242 %}
10243 
10244 instruct regL_to_regD(regD dst, iRegL src) %{
10245   // No match rule to avoid chain rule match.
10246   effect(DEF dst, USE src);
10247   ins_cost(MEMORY_REF_COST);
10248   size(4);
10249   format %{ "FMDRR   $dst,$src\t! regL to regD" %}
10250   ins_encode %{
10251     __ fmdrr($dst$$FloatRegister, $src$$Register, $src$$Register->successor());
10252   %}
10253   ins_pipe(ialu_reg_reg); // FIXME
10254 %}
10255 
10256 instruct regI_regI_to_regD(regD dst, iRegI src1, iRegI src2) %{
10257   // No match rule to avoid chain rule match.
10258   effect(DEF dst, USE src1, USE src2);
10259   ins_cost(MEMORY_REF_COST);
10260   size(4);
10261   format %{ "FMDRR   $dst,$src1,$src2\t! regI,regI to regD" %}
10262   ins_encode %{
10263     __ fmdrr($dst$$FloatRegister, $src1$$Register, $src2$$Register);
10264   %}
10265   ins_pipe(ialu_reg_reg); // FIXME
10266 %}
10267 
10268 instruct convL2D_reg_slow_fxtof(regD dst, iRegL src) %{
10269   match(Set dst (ConvL2D src));
10270   ins_cost(DEFAULT_COST*8 + MEMORY_REF_COST*6); // FIXME
10271 
10272   expand %{
10273     regD_low   tmpsrc;
10274     iRegI      ix43300000;
10275     iRegI      ix41f00000;
10276     iRegI      ix0;
10277     regD_low   dx43300000;
10278     regD       dx41f00000;
10279     regD       tmp1;
10280     regD_low   tmp2;
10281     regD       tmp3;
10282     regD       tmp4;
10283 
10284     regL_to_regD(tmpsrc, src);
10285 
10286     loadConI_x43300000(ix43300000);
10287     loadConI_x41f00000(ix41f00000);
10288     loadConI_x0(ix0);
10289 
10290     regI_regI_to_regD(dx43300000, ix0, ix43300000);
10291     regI_regI_to_regD(dx41f00000, ix0, ix41f00000);
10292 
10293     convI2D_regDHi_regD(tmp1, tmpsrc);
10294     regDHi_regDLo_to_regD(tmp2, dx43300000, tmpsrc);
10295     subD_regD_regD(tmp3, tmp2, dx43300000);
10296     mulD_regD_regD(tmp4, tmp1, dx41f00000);
10297     addD_regD_regD(dst, tmp3, tmp4);
10298   %}
10299 %}
10300 #endif // !AARCH64
10301 
10302 instruct convL2I_reg(iRegI dst, iRegL src) %{
10303   match(Set dst (ConvL2I src));
10304   size(4);
10305 #ifdef AARCH64
10306   format %{ "MOV_w  $dst,$src\t! long->int" %}
10307   ins_encode %{
10308     __ mov_w($dst$$Register, $src$$Register);
10309   %}
10310 #else
10311   format %{ "MOV    $dst,$src.lo\t! long->int" %}
10312   ins_encode %{
10313     __ mov($dst$$Register, $src$$Register);
10314   %}
10315 #endif
10316   ins_pipe(ialu_move_reg_I_to_L);
10317 %}
10318 
10319 #ifndef AARCH64
10320 // Register Shift Right Immediate
10321 instruct shrL_reg_imm6_L2I(iRegI dst, iRegL src, immI_32_63 cnt) %{
10322   match(Set dst (ConvL2I (RShiftL src cnt)));
10323   size(4);
10324   format %{ "ASR    $dst,$src.hi,($cnt - 32)\t! long->int or mov if $cnt==32" %}
10325   ins_encode %{
10326     if ($cnt$$constant == 32) {
10327       __ mov($dst$$Register, $src$$Register->successor());
10328     } else {
10329       __ mov($dst$$Register, AsmOperand($src$$Register->successor(), asr, $cnt$$constant - 32));
10330     }
10331   %}
10332   ins_pipe(ialu_reg_imm);
10333 %}
10334 #endif
10335 
10336 
10337 //----------Control Flow Instructions------------------------------------------
10338 // Compare Instructions
10339 // Compare Integers
10340 instruct compI_iReg(flagsReg icc, iRegI op1, iRegI op2) %{
10341   match(Set icc (CmpI op1 op2));
10342   effect( DEF icc, USE op1, USE op2 );
10343 
10344   size(4);
10345   format %{ "cmp_32 $op1,$op2\t! int" %}
10346   ins_encode %{
10347     __ cmp_32($op1$$Register, $op2$$Register);
10348   %}
10349   ins_pipe(ialu_cconly_reg_reg);
10350 %}
10351 
10352 #ifdef _LP64
10353 // Compare compressed pointers
10354 instruct compN_reg2(flagsRegU icc, iRegN op1, iRegN op2) %{
10355   match(Set icc (CmpN op1 op2));
10356   effect( DEF icc, USE op1, USE op2 );
10357 
10358   size(4);
10359   format %{ "cmp_32 $op1,$op2\t! int" %}
10360   ins_encode %{
10361     __ cmp_32($op1$$Register, $op2$$Register);
10362   %}
10363   ins_pipe(ialu_cconly_reg_reg);
10364 %}
10365 #endif
10366 
10367 instruct compU_iReg(flagsRegU icc, iRegI op1, iRegI op2) %{
10368   match(Set icc (CmpU op1 op2));
10369 
10370   size(4);
10371   format %{ "cmp_32 $op1,$op2\t! unsigned int" %}
10372   ins_encode %{
10373     __ cmp_32($op1$$Register, $op2$$Register);
10374   %}
10375   ins_pipe(ialu_cconly_reg_reg);
10376 %}
10377 
10378 instruct compI_iReg_immneg(flagsReg icc, iRegI op1, aimmIneg op2) %{
10379   match(Set icc (CmpI op1 op2));
10380   effect( DEF icc, USE op1 );
10381 
10382   size(4);
10383   format %{ "cmn_32 $op1,-$op2\t! int" %}
10384   ins_encode %{
10385     __ cmn_32($op1$$Register, -$op2$$constant);
10386   %}
10387   ins_pipe(ialu_cconly_reg_imm);
10388 %}
10389 
10390 instruct compI_iReg_imm(flagsReg icc, iRegI op1, aimmI op2) %{
10391   match(Set icc (CmpI op1 op2));
10392   effect( DEF icc, USE op1 );
10393 
10394   size(4);
10395   format %{ "cmp_32 $op1,$op2\t! int" %}
10396   ins_encode %{
10397     __ cmp_32($op1$$Register, $op2$$constant);
10398   %}
10399   ins_pipe(ialu_cconly_reg_imm);
10400 %}
10401 
10402 instruct testI_reg_reg( flagsReg_EQNELTGE icc, iRegI op1, iRegI op2, immI0 zero ) %{
10403   match(Set icc (CmpI (AndI op1 op2) zero));
10404   size(4);
10405   format %{ "tst_32 $op2,$op1" %}
10406 
10407   ins_encode %{
10408     __ tst_32($op1$$Register, $op2$$Register);
10409   %}
10410   ins_pipe(ialu_cconly_reg_reg_zero);
10411 %}
10412 
10413 #ifndef AARCH64
10414 instruct testshlI_reg_reg_reg( flagsReg_EQNELTGE icc, iRegI op1, iRegI op2, iRegI op3, immI0 zero ) %{
10415   match(Set icc (CmpI (AndI op1 (LShiftI op2 op3)) zero));
10416   size(4);
10417   format %{ "TST   $op2,$op1<<$op3" %}
10418 
10419   ins_encode %{
10420     __ tst($op1$$Register, AsmOperand($op2$$Register, lsl, $op3$$Register));
10421   %}
10422   ins_pipe(ialu_cconly_reg_reg_zero);
10423 %}
10424 #endif
10425 
10426 instruct testshlI_reg_reg_imm( flagsReg_EQNELTGE icc, iRegI op1, iRegI op2, immU5 op3, immI0 zero ) %{
10427   match(Set icc (CmpI (AndI op1 (LShiftI op2 op3)) zero));
10428   size(4);
10429   format %{ "tst_32 $op2,$op1<<$op3" %}
10430 
10431   ins_encode %{
10432     __ tst_32($op1$$Register, AsmOperand($op2$$Register, lsl, $op3$$constant));
10433   %}
10434   ins_pipe(ialu_cconly_reg_reg_zero);
10435 %}
10436 
10437 #ifndef AARCH64
10438 instruct testsarI_reg_reg_reg( flagsReg_EQNELTGE icc, iRegI op1, iRegI op2, iRegI op3, immI0 zero ) %{
10439   match(Set icc (CmpI (AndI op1 (RShiftI op2 op3)) zero));
10440   size(4);
10441   format %{ "TST   $op2,$op1<<$op3" %}
10442 
10443   ins_encode %{
10444     __ tst($op1$$Register, AsmOperand($op2$$Register, asr, $op3$$Register));
10445   %}
10446   ins_pipe(ialu_cconly_reg_reg_zero);
10447 %}
10448 #endif
10449 
10450 instruct testsarI_reg_reg_imm( flagsReg_EQNELTGE icc, iRegI op1, iRegI op2, immU5 op3, immI0 zero ) %{
10451   match(Set icc (CmpI (AndI op1 (RShiftI op2 op3)) zero));
10452   size(4);
10453   format %{ "tst_32 $op2,$op1<<$op3" %}
10454 
10455   ins_encode %{
10456     __ tst_32($op1$$Register, AsmOperand($op2$$Register, asr, $op3$$constant));
10457   %}
10458   ins_pipe(ialu_cconly_reg_reg_zero);
10459 %}
10460 
10461 #ifndef AARCH64
10462 instruct testshrI_reg_reg_reg( flagsReg_EQNELTGE icc, iRegI op1, iRegI op2, iRegI op3, immI0 zero ) %{
10463   match(Set icc (CmpI (AndI op1 (URShiftI op2 op3)) zero));
10464   size(4);
10465   format %{ "TST   $op2,$op1<<$op3" %}
10466 
10467   ins_encode %{
10468     __ tst($op1$$Register, AsmOperand($op2$$Register, lsr, $op3$$Register));
10469   %}
10470   ins_pipe(ialu_cconly_reg_reg_zero);
10471 %}
10472 #endif
10473 
10474 instruct testshrI_reg_reg_imm( flagsReg_EQNELTGE icc, iRegI op1, iRegI op2, immU5 op3, immI0 zero ) %{
10475   match(Set icc (CmpI (AndI op1 (URShiftI op2 op3)) zero));
10476   size(4);
10477   format %{ "tst_32 $op2,$op1<<$op3" %}
10478 
10479   ins_encode %{
10480     __ tst_32($op1$$Register, AsmOperand($op2$$Register, lsr, $op3$$constant));
10481   %}
10482   ins_pipe(ialu_cconly_reg_reg_zero);
10483 %}
10484 
10485 instruct testI_reg_imm( flagsReg_EQNELTGE icc, iRegI op1, limmI op2, immI0 zero ) %{
10486   match(Set icc (CmpI (AndI op1 op2) zero));
10487   size(4);
10488   format %{ "tst_32 $op2,$op1" %}
10489 
10490   ins_encode %{
10491     __ tst_32($op1$$Register, $op2$$constant);
10492   %}
10493   ins_pipe(ialu_cconly_reg_imm_zero);
10494 %}
10495 
10496 #ifdef AARCH64
10497 instruct compL_reg_reg(flagsReg xcc, iRegL op1, iRegL op2)
10498 %{
10499   match(Set xcc (CmpL op1 op2));
10500   effect( DEF xcc, USE op1, USE op2 );
10501 
10502   size(4);
10503   format %{ "CMP     $op1,$op2\t! long" %}
10504   ins_encode %{
10505     __ cmp($op1$$Register, $op2$$Register);
10506   %}
10507   ins_pipe(ialu_cconly_reg_reg);
10508 %}
10509 
10510 instruct compUL_iReg(flagsRegU xcc, iRegL op1, iRegL op2) %{
10511   match(Set xcc (CmpUL op1 op2));
10512 
10513   size(4);
10514   format %{ "CMP     $op1,$op2\t! unsigned long" %}
10515   ins_encode %{
10516     __ cmp($op1$$Register, $op2$$Register);
10517   %}
10518   ins_pipe(ialu_cconly_reg_reg);
10519 %}
10520 #else
10521 instruct compL_reg_reg_LTGE(flagsRegL_LTGE xcc, iRegL op1, iRegL op2, iRegL tmp) %{
10522   match(Set xcc (CmpL op1 op2));
10523   effect( DEF xcc, USE op1, USE op2, TEMP tmp );
10524 
10525   size(8);
10526   format %{ "SUBS    $tmp,$op1.low,$op2.low\t\t! long\n\t"
10527             "SBCS    $tmp,$op1.hi,$op2.hi" %}
10528   ins_encode %{
10529     __ subs($tmp$$Register, $op1$$Register, $op2$$Register);
10530     __ sbcs($tmp$$Register->successor(), $op1$$Register->successor(), $op2$$Register->successor());
10531   %}
10532   ins_pipe(ialu_cconly_reg_reg);
10533 %}
10534 
10535 instruct compUL_reg_reg_LTGE(flagsRegUL_LTGE xcc, iRegL op1, iRegL op2, iRegL tmp) %{
10536   match(Set xcc (CmpUL op1 op2));
10537   effect(DEF xcc, USE op1, USE op2, TEMP tmp);
10538 
10539   size(8);
10540   format %{ "SUBS    $tmp,$op1.low,$op2.low\t\t! unsigned long\n\t"
10541             "SBCS    $tmp,$op1.hi,$op2.hi" %}
10542   ins_encode %{
10543     __ subs($tmp$$Register, $op1$$Register, $op2$$Register);
10544     __ sbcs($tmp$$Register->successor(), $op1$$Register->successor(), $op2$$Register->successor());
10545   %}
10546   ins_pipe(ialu_cconly_reg_reg);
10547 %}
10548 #endif
10549 
10550 #ifdef AARCH64
10551 instruct compL_reg_con(flagsReg xcc, iRegL op1, aimmL con) %{
10552   match(Set xcc (CmpL op1 con));
10553   effect( DEF xcc, USE op1, USE con );
10554 
10555   size(8);
10556   format %{ "CMP     $op1,$con\t\t! long"  %}
10557   ins_encode %{
10558     __ cmp($op1$$Register, $con$$constant);
10559   %}
10560 
10561   ins_pipe(ialu_cconly_reg_imm);
10562 %}
10563 
10564 instruct compUL_reg_con(flagsRegU xcc, iRegL op1, aimmL con) %{
10565   match(Set xcc (CmpUL op1 con));
10566   effect(DEF xcc, USE op1, USE con);
10567 
10568   size(8);
10569   format %{ "CMP     $op1,$con\t\t! unsigned long"  %}
10570   ins_encode %{
10571     __ cmp($op1$$Register, $con$$constant);
10572   %}
10573 
10574   ins_pipe(ialu_cconly_reg_imm);
10575 %}
10576 #else
10577 instruct compL_reg_reg_EQNE(flagsRegL_EQNE xcc, iRegL op1, iRegL op2) %{
10578   match(Set xcc (CmpL op1 op2));
10579   effect( DEF xcc, USE op1, USE op2 );
10580 
10581   size(8);
10582   format %{ "TEQ    $op1.hi,$op2.hi\t\t! long\n\t"
10583             "TEQ.eq $op1.lo,$op2.lo" %}
10584   ins_encode %{
10585     __ teq($op1$$Register->successor(), $op2$$Register->successor());
10586     __ teq($op1$$Register, $op2$$Register, eq);
10587   %}
10588   ins_pipe(ialu_cconly_reg_reg);
10589 %}
10590 
10591 instruct compL_reg_reg_LEGT(flagsRegL_LEGT xcc, iRegL op1, iRegL op2, iRegL tmp) %{
10592   match(Set xcc (CmpL op1 op2));
10593   effect( DEF xcc, USE op1, USE op2, TEMP tmp );
10594 
10595   size(8);
10596   format %{ "SUBS    $tmp,$op2.low,$op1.low\t\t! long\n\t"
10597             "SBCS    $tmp,$op2.hi,$op1.hi" %}
10598   ins_encode %{
10599     __ subs($tmp$$Register, $op2$$Register, $op1$$Register);
10600     __ sbcs($tmp$$Register->successor(), $op2$$Register->successor(), $op1$$Register->successor());
10601   %}
10602   ins_pipe(ialu_cconly_reg_reg);
10603 %}
10604 
10605 // TODO: try immLRot2 instead, (0, $con$$constant) becomes
10606 // (hi($con$$constant), lo($con$$constant)) becomes
10607 instruct compL_reg_con_LTGE(flagsRegL_LTGE xcc, iRegL op1, immLlowRot con, iRegL tmp) %{
10608   match(Set xcc (CmpL op1 con));
10609   effect( DEF xcc, USE op1, USE con, TEMP tmp );
10610 
10611   size(8);
10612   format %{ "SUBS    $tmp,$op1.low,$con\t\t! long\n\t"
10613             "SBCS    $tmp,$op1.hi,0" %}
10614   ins_encode %{
10615     __ subs($tmp$$Register, $op1$$Register, $con$$constant);
10616     __ sbcs($tmp$$Register->successor(), $op1$$Register->successor(), 0);
10617   %}
10618 
10619   ins_pipe(ialu_cconly_reg_reg);
10620 %}
10621 
10622 // TODO: try immLRot2 instead, (0, $con$$constant) becomes
10623 // (hi($con$$constant), lo($con$$constant)) becomes
10624 instruct compL_reg_con_EQNE(flagsRegL_EQNE xcc, iRegL op1, immLlowRot con) %{
10625   match(Set xcc (CmpL op1 con));
10626   effect( DEF xcc, USE op1, USE con );
10627 
10628   size(8);
10629   format %{ "TEQ    $op1.hi,0\t\t! long\n\t"
10630             "TEQ.eq $op1.lo,$con" %}
10631   ins_encode %{
10632     __ teq($op1$$Register->successor(), 0);
10633     __ teq($op1$$Register, $con$$constant, eq);
10634   %}
10635 
10636   ins_pipe(ialu_cconly_reg_reg);
10637 %}
10638 
10639 // TODO: try immLRot2 instead, (0, $con$$constant) becomes
10640 // (hi($con$$constant), lo($con$$constant)) becomes
10641 instruct compL_reg_con_LEGT(flagsRegL_LEGT xcc, iRegL op1, immLlowRot con, iRegL tmp) %{
10642   match(Set xcc (CmpL op1 con));
10643   effect( DEF xcc, USE op1, USE con, TEMP tmp );
10644 
10645   size(8);
10646   format %{ "RSBS    $tmp,$op1.low,$con\t\t! long\n\t"
10647             "RSCS    $tmp,$op1.hi,0" %}
10648   ins_encode %{
10649     __ rsbs($tmp$$Register, $op1$$Register, $con$$constant);
10650     __ rscs($tmp$$Register->successor(), $op1$$Register->successor(), 0);
10651   %}
10652 
10653   ins_pipe(ialu_cconly_reg_reg);
10654 %}
10655 
10656 instruct compUL_reg_reg_EQNE(flagsRegUL_EQNE xcc, iRegL op1, iRegL op2) %{
10657   match(Set xcc (CmpUL op1 op2));
10658   effect(DEF xcc, USE op1, USE op2);
10659 
10660   size(8);
10661   format %{ "TEQ    $op1.hi,$op2.hi\t\t! unsigned long\n\t"
10662             "TEQ.eq $op1.lo,$op2.lo" %}
10663   ins_encode %{
10664     __ teq($op1$$Register->successor(), $op2$$Register->successor());
10665     __ teq($op1$$Register, $op2$$Register, eq);
10666   %}
10667   ins_pipe(ialu_cconly_reg_reg);
10668 %}
10669 
10670 instruct compUL_reg_reg_LEGT(flagsRegUL_LEGT xcc, iRegL op1, iRegL op2, iRegL tmp) %{
10671   match(Set xcc (CmpUL op1 op2));
10672   effect(DEF xcc, USE op1, USE op2, TEMP tmp);
10673 
10674   size(8);
10675   format %{ "SUBS    $tmp,$op2.low,$op1.low\t\t! unsigned long\n\t"
10676             "SBCS    $tmp,$op2.hi,$op1.hi" %}
10677   ins_encode %{
10678     __ subs($tmp$$Register, $op2$$Register, $op1$$Register);
10679     __ sbcs($tmp$$Register->successor(), $op2$$Register->successor(), $op1$$Register->successor());
10680   %}
10681   ins_pipe(ialu_cconly_reg_reg);
10682 %}
10683 
10684 // TODO: try immLRot2 instead, (0, $con$$constant) becomes
10685 // (hi($con$$constant), lo($con$$constant)) becomes
10686 instruct compUL_reg_con_LTGE(flagsRegUL_LTGE xcc, iRegL op1, immLlowRot con, iRegL tmp) %{
10687   match(Set xcc (CmpUL op1 con));
10688   effect(DEF xcc, USE op1, USE con, TEMP tmp);
10689 
10690   size(8);
10691   format %{ "SUBS    $tmp,$op1.low,$con\t\t! unsigned long\n\t"
10692             "SBCS    $tmp,$op1.hi,0" %}
10693   ins_encode %{
10694     __ subs($tmp$$Register, $op1$$Register, $con$$constant);
10695     __ sbcs($tmp$$Register->successor(), $op1$$Register->successor(), 0);
10696   %}
10697 
10698   ins_pipe(ialu_cconly_reg_reg);
10699 %}
10700 
10701 // TODO: try immLRot2 instead, (0, $con$$constant) becomes
10702 // (hi($con$$constant), lo($con$$constant)) becomes
10703 instruct compUL_reg_con_EQNE(flagsRegUL_EQNE xcc, iRegL op1, immLlowRot con) %{
10704   match(Set xcc (CmpUL op1 con));
10705   effect(DEF xcc, USE op1, USE con);
10706 
10707   size(8);
10708   format %{ "TEQ    $op1.hi,0\t\t! unsigned long\n\t"
10709             "TEQ.eq $op1.lo,$con" %}
10710   ins_encode %{
10711     __ teq($op1$$Register->successor(), 0);
10712     __ teq($op1$$Register, $con$$constant, eq);
10713   %}
10714 
10715   ins_pipe(ialu_cconly_reg_reg);
10716 %}
10717 
10718 // TODO: try immLRot2 instead, (0, $con$$constant) becomes
10719 // (hi($con$$constant), lo($con$$constant)) becomes
10720 instruct compUL_reg_con_LEGT(flagsRegUL_LEGT xcc, iRegL op1, immLlowRot con, iRegL tmp) %{
10721   match(Set xcc (CmpUL op1 con));
10722   effect(DEF xcc, USE op1, USE con, TEMP tmp);
10723 
10724   size(8);
10725   format %{ "RSBS    $tmp,$op1.low,$con\t\t! unsigned long\n\t"
10726             "RSCS    $tmp,$op1.hi,0" %}
10727   ins_encode %{
10728     __ rsbs($tmp$$Register, $op1$$Register, $con$$constant);
10729     __ rscs($tmp$$Register->successor(), $op1$$Register->successor(), 0);
10730   %}
10731 
10732   ins_pipe(ialu_cconly_reg_reg);
10733 %}
10734 #endif
10735 
10736 /* instruct testL_reg_reg(flagsRegL xcc, iRegL op1, iRegL op2, immL0 zero) %{ */
10737 /*   match(Set xcc (CmpL (AndL op1 op2) zero)); */
10738 /*   ins_encode %{ */
10739 /*     __ stop("testL_reg_reg unimplemented"); */
10740 /*   %} */
10741 /*   ins_pipe(ialu_cconly_reg_reg); */
10742 /* %} */
10743 
10744 /* // useful for checking the alignment of a pointer: */
10745 /* instruct testL_reg_con(flagsRegL xcc, iRegL op1, immLlowRot con, immL0 zero) %{ */
10746 /*   match(Set xcc (CmpL (AndL op1 con) zero)); */
10747 /*   ins_encode %{ */
10748 /*     __ stop("testL_reg_con unimplemented"); */
10749 /*   %} */
10750 /*   ins_pipe(ialu_cconly_reg_reg); */
10751 /* %} */
10752 
10753 instruct compU_iReg_imm(flagsRegU icc, iRegI op1, aimmU31 op2 ) %{
10754   match(Set icc (CmpU op1 op2));
10755 
10756   size(4);
10757   format %{ "cmp_32 $op1,$op2\t! unsigned" %}
10758   ins_encode %{
10759     __ cmp_32($op1$$Register, $op2$$constant);
10760   %}
10761   ins_pipe(ialu_cconly_reg_imm);
10762 %}
10763 
10764 // Compare Pointers
10765 instruct compP_iRegP(flagsRegP pcc, iRegP op1, iRegP op2 ) %{
10766   match(Set pcc (CmpP op1 op2));
10767 
10768   size(4);
10769   format %{ "CMP    $op1,$op2\t! ptr" %}
10770   ins_encode %{
10771     __ cmp($op1$$Register, $op2$$Register);
10772   %}
10773   ins_pipe(ialu_cconly_reg_reg);
10774 %}
10775 
10776 instruct compP_iRegP_imm(flagsRegP pcc, iRegP op1, aimmP op2 ) %{
10777   match(Set pcc (CmpP op1 op2));
10778 
10779   size(4);
10780   format %{ "CMP    $op1,$op2\t! ptr" %}
10781   ins_encode %{
10782     assert($op2$$constant == 0 || _opnds[2]->constant_reloc() == relocInfo::none, "reloc in cmp?");
10783     __ cmp($op1$$Register, $op2$$constant);
10784   %}
10785   ins_pipe(ialu_cconly_reg_imm);
10786 %}
10787 
10788 //----------Max and Min--------------------------------------------------------
10789 // Min Instructions
10790 // Conditional move for min
10791 instruct cmovI_reg_lt( iRegI op2, iRegI op1, flagsReg icc ) %{
10792   effect( USE_DEF op2, USE op1, USE icc );
10793 
10794   size(4);
10795   format %{ "MOV.lt  $op2,$op1\t! min" %}
10796   ins_encode %{
10797     __ mov($op2$$Register, $op1$$Register, lt);
10798   %}
10799   ins_pipe(ialu_reg_flags);
10800 %}
10801 
10802 // Min Register with Register.
10803 instruct minI_eReg(iRegI op1, iRegI op2) %{
10804   match(Set op2 (MinI op1 op2));
10805   ins_cost(DEFAULT_COST*2);
10806   expand %{
10807     flagsReg icc;
10808     compI_iReg(icc,op1,op2);
10809     cmovI_reg_lt(op2,op1,icc);
10810   %}
10811 %}
10812 
10813 // Max Instructions
10814 // Conditional move for max
10815 instruct cmovI_reg_gt( iRegI op2, iRegI op1, flagsReg icc ) %{
10816   effect( USE_DEF op2, USE op1, USE icc );
10817   format %{ "MOV.gt  $op2,$op1\t! max" %}
10818   ins_encode %{
10819     __ mov($op2$$Register, $op1$$Register, gt);
10820   %}
10821   ins_pipe(ialu_reg_flags);
10822 %}
10823 
10824 // Max Register with Register
10825 instruct maxI_eReg(iRegI op1, iRegI op2) %{
10826   match(Set op2 (MaxI op1 op2));
10827   ins_cost(DEFAULT_COST*2);
10828   expand %{
10829     flagsReg icc;
10830     compI_iReg(icc,op1,op2);
10831     cmovI_reg_gt(op2,op1,icc);
10832   %}
10833 %}
10834 
10835 
10836 //----------Float Compares----------------------------------------------------
10837 // Compare floating, generate condition code
10838 instruct cmpF_cc(flagsRegF fcc, flagsReg icc, regF src1, regF src2) %{
10839   match(Set icc (CmpF src1 src2));
10840   effect(KILL fcc);
10841 
10842 #ifdef AARCH64
10843   size(4);
10844   format %{ "FCMP_s  $src1,$src2" %}
10845   ins_encode %{
10846     __ fcmp_s($src1$$FloatRegister, $src2$$FloatRegister);
10847   %}
10848 #else
10849   size(8);
10850   format %{ "FCMPs  $src1,$src2\n\t"
10851             "FMSTAT" %}
10852   ins_encode %{
10853     __ fcmps($src1$$FloatRegister, $src2$$FloatRegister);
10854     __ fmstat();
10855   %}
10856 #endif
10857   ins_pipe(faddF_fcc_reg_reg_zero);
10858 %}
10859 
10860 instruct cmpF0_cc(flagsRegF fcc, flagsReg icc, regF src1, immF0 src2) %{
10861   match(Set icc (CmpF src1 src2));
10862   effect(KILL fcc);
10863 
10864 #ifdef AARCH64
10865   size(4);
10866   format %{ "FCMP0_s $src1" %}
10867   ins_encode %{
10868     __ fcmp0_s($src1$$FloatRegister);
10869   %}
10870 #else
10871   size(8);
10872   format %{ "FCMPs  $src1,$src2\n\t"
10873             "FMSTAT" %}
10874   ins_encode %{
10875     __ fcmpzs($src1$$FloatRegister);
10876     __ fmstat();
10877   %}
10878 #endif
10879   ins_pipe(faddF_fcc_reg_reg_zero);
10880 %}
10881 
10882 instruct cmpD_cc(flagsRegF fcc, flagsReg icc, regD src1, regD src2) %{
10883   match(Set icc (CmpD src1 src2));
10884   effect(KILL fcc);
10885 
10886 #ifdef AARCH64
10887   size(4);
10888   format %{ "FCMP_d $src1,$src2" %}
10889   ins_encode %{
10890     __ fcmp_d($src1$$FloatRegister, $src2$$FloatRegister);
10891   %}
10892 #else
10893   size(8);
10894   format %{ "FCMPd  $src1,$src2 \n\t"
10895             "FMSTAT" %}
10896   ins_encode %{
10897     __ fcmpd($src1$$FloatRegister, $src2$$FloatRegister);
10898     __ fmstat();
10899   %}
10900 #endif
10901   ins_pipe(faddD_fcc_reg_reg_zero);
10902 %}
10903 
10904 instruct cmpD0_cc(flagsRegF fcc, flagsReg icc, regD src1, immD0 src2) %{
10905   match(Set icc (CmpD src1 src2));
10906   effect(KILL fcc);
10907 
10908 #ifdef AARCH64
10909   size(8);
10910   format %{ "FCMP0_d $src1" %}
10911   ins_encode %{
10912     __ fcmp0_d($src1$$FloatRegister);
10913   %}
10914 #else
10915   size(8);
10916   format %{ "FCMPZd  $src1,$src2 \n\t"
10917             "FMSTAT" %}
10918   ins_encode %{
10919     __ fcmpzd($src1$$FloatRegister);
10920     __ fmstat();
10921   %}
10922 #endif
10923   ins_pipe(faddD_fcc_reg_reg_zero);
10924 %}
10925 
10926 #ifdef AARCH64
10927 // Compare floating, generate -1,0,1
10928 instruct cmpF_reg(iRegI dst, regF src1, regF src2, flagsReg icc) %{
10929   match(Set dst (CmpF3 src1 src2));
10930   // effect(KILL fcc); // nobody cares if flagsRegF is killed
10931   effect(KILL icc);
10932   ins_cost(DEFAULT_COST*3); // FIXME
10933   size(12);
10934   format %{ "FCMP_s $src1,$src2\n\t"
10935             "CSET   $dst, gt\n\t"
10936             "CSINV  $dst, $dst, ZR, ge" %}
10937   ins_encode %{
10938     Register dst = $dst$$Register;
10939     __ fcmp_s($src1$$FloatRegister, $src2$$FloatRegister);
10940     __ cset(dst, gt);            // 1 if '>', else 0
10941     __ csinv(dst, dst, ZR, ge);  // previous value if '>=', else -1
10942   %}
10943   ins_pipe( floating_cmp ); // FIXME
10944 %}
10945 
10946 // Compare floating, generate -1,0,1
10947 instruct cmpD_reg(iRegI dst, regD src1, regD src2, flagsReg icc) %{
10948   match(Set dst (CmpD3 src1 src2));
10949   // effect(KILL fcc); // nobody cares if flagsRegF is killed
10950   effect(KILL icc);
10951   ins_cost(DEFAULT_COST*3); // FIXME
10952   size(12);
10953   format %{ "FCMP_d $src1,$src2\n\t"
10954             "CSET   $dst, gt\n\t"
10955             "CSINV  $dst, $dst, ZR, ge" %}
10956   ins_encode %{
10957     Register dst = $dst$$Register;
10958     __ fcmp_d($src1$$FloatRegister, $src2$$FloatRegister);
10959     __ cset(dst, gt);            // 1 if '>', else 0
10960     __ csinv(dst, dst, ZR, ge);  // previous value if '>=', else -1
10961   %}
10962   ins_pipe( floating_cmp ); // FIXME
10963 %}
10964 
10965 // Compare floating, generate -1,0,1
10966 instruct cmpF0_reg(iRegI dst, regF src1, immF0 src2, flagsReg icc) %{
10967   match(Set dst (CmpF3 src1 src2));
10968   // effect(KILL fcc); // nobody cares if flagsRegF is killed
10969   effect(KILL icc);
10970   ins_cost(DEFAULT_COST*3); // FIXME
10971   size(12);
10972   format %{ "FCMP0_s $src1\n\t"
10973             "CSET   $dst, gt\n\t"
10974             "CSINV  $dst, $dst, ZR, ge" %}
10975   ins_encode %{
10976     Register dst = $dst$$Register;
10977     __ fcmp0_s($src1$$FloatRegister);
10978     __ cset(dst, gt);            // 1 if '>', else 0
10979     __ csinv(dst, dst, ZR, ge);  // previous value if '>=', else -1
10980   %}
10981   ins_pipe( floating_cmp ); // FIXME
10982 %}
10983 
10984 // Compare floating, generate -1,0,1
10985 instruct cmpD0_reg(iRegI dst, regD src1, immD0 src2, flagsReg icc) %{
10986   match(Set dst (CmpD3 src1 src2));
10987   // effect(KILL fcc); // nobody cares if flagsRegF is killed
10988   effect(KILL icc);
10989   ins_cost(DEFAULT_COST*3); // FIXME
10990   size(12);
10991   format %{ "FCMP0_d $src1\n\t"
10992             "CSET   $dst, gt\n\t"
10993             "CSINV  $dst, $dst, ZR, ge" %}
10994   ins_encode %{
10995     Register dst = $dst$$Register;
10996     __ fcmp0_d($src1$$FloatRegister);
10997     __ cset(dst, gt);            // 1 if '>', else 0
10998     __ csinv(dst, dst, ZR, ge);  // previous value if '>=', else -1
10999   %}
11000   ins_pipe( floating_cmp ); // FIXME
11001 %}
11002 #else
11003 // Compare floating, generate -1,0,1
11004 instruct cmpF_reg(iRegI dst, regF src1, regF src2, flagsRegF fcc) %{
11005   match(Set dst (CmpF3 src1 src2));
11006   effect(KILL fcc);
11007   ins_cost(DEFAULT_COST*3+BRANCH_COST*3); // FIXME
11008   size(20);
11009   // same number of instructions as code using conditional moves but
11010   // doesn't kill integer condition register
11011   format %{ "FCMPs  $dst,$src1,$src2 \n\t"
11012             "VMRS   $dst, FPSCR \n\t"
11013             "OR     $dst, $dst, 0x08000000 \n\t"
11014             "EOR    $dst, $dst, $dst << 3 \n\t"
11015             "MOV    $dst, $dst >> 30" %}
11016   ins_encode %{
11017     __ fcmps($src1$$FloatRegister, $src2$$FloatRegister);
11018     __ floating_cmp($dst$$Register);
11019   %}
11020   ins_pipe( floating_cmp );
11021 %}
11022 
11023 instruct cmpF0_reg(iRegI dst, regF src1, immF0 src2, flagsRegF fcc) %{
11024   match(Set dst (CmpF3 src1 src2));
11025   effect(KILL fcc);
11026   ins_cost(DEFAULT_COST*3+BRANCH_COST*3); // FIXME
11027   size(20);
11028   // same number of instructions as code using conditional moves but
11029   // doesn't kill integer condition register
11030   format %{ "FCMPZs $dst,$src1,$src2 \n\t"
11031             "VMRS   $dst, FPSCR \n\t"
11032             "OR     $dst, $dst, 0x08000000 \n\t"
11033             "EOR    $dst, $dst, $dst << 3 \n\t"
11034             "MOV    $dst, $dst >> 30" %}
11035   ins_encode %{
11036     __ fcmpzs($src1$$FloatRegister);
11037     __ floating_cmp($dst$$Register);
11038   %}
11039   ins_pipe( floating_cmp );
11040 %}
11041 
11042 instruct cmpD_reg(iRegI dst, regD src1, regD src2, flagsRegF fcc) %{
11043   match(Set dst (CmpD3 src1 src2));
11044   effect(KILL fcc);
11045   ins_cost(DEFAULT_COST*3+BRANCH_COST*3); // FIXME
11046   size(20);
11047   // same number of instructions as code using conditional moves but
11048   // doesn't kill integer condition register
11049   format %{ "FCMPd  $dst,$src1,$src2 \n\t"
11050             "VMRS   $dst, FPSCR \n\t"
11051             "OR     $dst, $dst, 0x08000000 \n\t"
11052             "EOR    $dst, $dst, $dst << 3 \n\t"
11053             "MOV    $dst, $dst >> 30" %}
11054   ins_encode %{
11055     __ fcmpd($src1$$FloatRegister, $src2$$FloatRegister);
11056     __ floating_cmp($dst$$Register);
11057   %}
11058   ins_pipe( floating_cmp );
11059 %}
11060 
11061 instruct cmpD0_reg(iRegI dst, regD src1, immD0 src2, flagsRegF fcc) %{
11062   match(Set dst (CmpD3 src1 src2));
11063   effect(KILL fcc);
11064   ins_cost(DEFAULT_COST*3+BRANCH_COST*3); // FIXME
11065   size(20);
11066   // same number of instructions as code using conditional moves but
11067   // doesn't kill integer condition register
11068   format %{ "FCMPZd $dst,$src1,$src2 \n\t"
11069             "VMRS   $dst, FPSCR \n\t"
11070             "OR     $dst, $dst, 0x08000000 \n\t"
11071             "EOR    $dst, $dst, $dst << 3 \n\t"
11072             "MOV    $dst, $dst >> 30" %}
11073   ins_encode %{
11074     __ fcmpzd($src1$$FloatRegister);
11075     __ floating_cmp($dst$$Register);
11076   %}
11077   ins_pipe( floating_cmp );
11078 %}
11079 #endif // !AARCH64
11080 
11081 //----------Branches---------------------------------------------------------
11082 // Jump
11083 // (compare 'operand indIndex' and 'instruct addP_reg_reg' above)
11084 // FIXME
11085 instruct jumpXtnd(iRegX switch_val, iRegP tmp) %{
11086   match(Jump switch_val);
11087   effect(TEMP tmp);
11088   ins_cost(350);
11089   format %{  "ADD    $tmp, $constanttablebase, $switch_val\n\t"
11090              "LDR    $tmp,[$tmp + $constantoffset]\n\t"
11091              "BX     $tmp" %}
11092   size(20);
11093   ins_encode %{
11094     Register table_reg;
11095     Register label_reg = $tmp$$Register;
11096     if (constant_offset() == 0) {
11097       table_reg = $constanttablebase;
11098       __ ldr(label_reg, Address(table_reg, $switch_val$$Register));
11099     } else {
11100       table_reg = $tmp$$Register;
11101       int offset = $constantoffset;
11102       if (is_memoryP(offset)) {
11103         __ add(table_reg, $constanttablebase, $switch_val$$Register);
11104         __ ldr(label_reg, Address(table_reg, offset));
11105       } else {
11106         __ mov_slow(table_reg, $constantoffset);
11107         __ add(table_reg, $constanttablebase, table_reg);
11108         __ ldr(label_reg, Address(table_reg, $switch_val$$Register));
11109       }
11110     }
11111     __ jump(label_reg); // ldr + b better than ldr to PC for branch predictor?
11112     //    __ ldr(PC, Address($table$$Register, $switch_val$$Register));
11113   %}
11114   ins_pipe(ialu_reg_reg);
11115 %}
11116 
11117 // // Direct Branch.
11118 instruct branch(label labl) %{
11119   match(Goto);
11120   effect(USE labl);
11121 
11122   size(4);
11123   ins_cost(BRANCH_COST);
11124   format %{ "B     $labl" %}
11125   ins_encode %{
11126     __ b(*($labl$$label));
11127   %}
11128   ins_pipe(br);
11129 %}
11130 
11131 // Conditional Direct Branch
11132 instruct branchCon(cmpOp cmp, flagsReg icc, label labl) %{
11133   match(If cmp icc);
11134   effect(USE labl);
11135 
11136   size(4);
11137   ins_cost(BRANCH_COST);
11138   format %{ "B$cmp   $icc,$labl" %}
11139   ins_encode %{
11140     __ b(*($labl$$label), (AsmCondition)($cmp$$cmpcode));
11141   %}
11142   ins_pipe(br_cc);
11143 %}
11144 
11145 #ifdef ARM
11146 instruct branchCon_EQNELTGE(cmpOp0 cmp, flagsReg_EQNELTGE icc, label labl) %{
11147   match(If cmp icc);
11148   effect(USE labl);
11149   predicate( _kids[0]->_leaf->as_Bool()->_test._test == BoolTest::eq || _kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ne || _kids[0]->_leaf->as_Bool()->_test._test == BoolTest::lt || _kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ge);
11150 
11151   size(4);
11152   ins_cost(BRANCH_COST);
11153   format %{ "B$cmp   $icc,$labl" %}
11154   ins_encode %{
11155     __ b(*($labl$$label), (AsmCondition)($cmp$$cmpcode));
11156   %}
11157   ins_pipe(br_cc);
11158 %}
11159 #endif
11160 
11161 #ifdef AARCH64
11162 instruct cbzI(cmpOp cmp, iRegI op1, immI0 op2, label labl) %{
11163   match(If cmp (CmpI op1 op2));
11164   effect(USE labl);
11165   predicate(_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::eq ||
11166             _kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ne);
11167   size(4);
11168   ins_cost(BRANCH_COST);
11169   format %{ "CB{N}Z $op1, $labl\t! int $cmp" %}
11170   ins_encode %{
11171     if ($cmp$$cmpcode == eq) {
11172       __ cbz_w($op1$$Register, *($labl$$label));
11173     } else {
11174       __ cbnz_w($op1$$Register, *($labl$$label));
11175     }
11176   %}
11177   ins_pipe(br_cc); // FIXME
11178 %}
11179 
11180 instruct cbzP(cmpOpP cmp, iRegP op1, immP0 op2, label labl) %{
11181   match(If cmp (CmpP op1 op2));
11182   effect(USE labl);
11183   predicate(_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::eq ||
11184             _kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ne);
11185   size(4);
11186   ins_cost(BRANCH_COST);
11187   format %{ "CB{N}Z $op1, $labl\t! ptr $cmp" %}
11188   ins_encode %{
11189     if ($cmp$$cmpcode == eq) {
11190       __ cbz($op1$$Register, *($labl$$label));
11191     } else {
11192       __ cbnz($op1$$Register, *($labl$$label));
11193     }
11194   %}
11195   ins_pipe(br_cc); // FIXME
11196 %}
11197 
11198 instruct cbzL(cmpOpL cmp, iRegL op1, immL0 op2, label labl) %{
11199   match(If cmp (CmpL op1 op2));
11200   effect(USE labl);
11201   predicate(_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::eq ||
11202             _kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ne);
11203   size(4);
11204   ins_cost(BRANCH_COST);
11205   format %{ "CB{N}Z $op1, $labl\t! long $cmp" %}
11206   ins_encode %{
11207     if ($cmp$$cmpcode == eq) {
11208       __ cbz($op1$$Register, *($labl$$label));
11209     } else {
11210       __ cbnz($op1$$Register, *($labl$$label));
11211     }
11212   %}
11213   ins_pipe(br_cc); // FIXME
11214 %}
11215 #endif
11216 
11217 instruct branchConU(cmpOpU cmp, flagsRegU icc, label labl) %{
11218   match(If cmp icc);
11219   effect(USE labl);
11220 
11221   size(4);
11222   ins_cost(BRANCH_COST);
11223   format %{ "B$cmp  $icc,$labl" %}
11224   ins_encode %{
11225     __ b(*($labl$$label), (AsmCondition)($cmp$$cmpcode));
11226   %}
11227   ins_pipe(br_cc);
11228 %}
11229 
11230 instruct branchConP(cmpOpP cmp, flagsRegP pcc, label labl) %{
11231   match(If cmp pcc);
11232   effect(USE labl);
11233 
11234   size(4);
11235   ins_cost(BRANCH_COST);
11236   format %{ "B$cmp  $pcc,$labl" %}
11237   ins_encode %{
11238     __ b(*($labl$$label), (AsmCondition)($cmp$$cmpcode));
11239   %}
11240   ins_pipe(br_cc);
11241 %}
11242 
11243 #ifndef AARCH64
11244 instruct branchConL_LTGE(cmpOpL cmp, flagsRegL_LTGE xcc, label labl) %{
11245   match(If cmp xcc);
11246   effect(USE labl);
11247   predicate( _kids[0]->_leaf->as_Bool()->_test._test == BoolTest::lt || _kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ge );
11248 
11249   size(4);
11250   ins_cost(BRANCH_COST);
11251   format %{ "B$cmp  $xcc,$labl" %}
11252   ins_encode %{
11253     __ b(*($labl$$label), (AsmCondition)($cmp$$cmpcode));
11254   %}
11255   ins_pipe(br_cc);
11256 %}
11257 
11258 instruct branchConL_EQNE(cmpOpL cmp, flagsRegL_EQNE xcc, label labl) %{
11259   match(If cmp xcc);
11260   effect(USE labl);
11261   predicate( _kids[0]->_leaf->as_Bool()->_test._test == BoolTest::eq || _kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ne );
11262 
11263   size(4);
11264   ins_cost(BRANCH_COST);
11265   format %{ "B$cmp  $xcc,$labl" %}
11266   ins_encode %{
11267     __ b(*($labl$$label), (AsmCondition)($cmp$$cmpcode));
11268   %}
11269   ins_pipe(br_cc);
11270 %}
11271 
11272 instruct branchConL_LEGT(cmpOpL_commute cmp, flagsRegL_LEGT xcc, label labl) %{
11273   match(If cmp xcc);
11274   effect(USE labl);
11275   predicate( _kids[0]->_leaf->as_Bool()->_test._test == BoolTest::gt || _kids[0]->_leaf->as_Bool()->_test._test == BoolTest::le );
11276 
11277   size(4);
11278   ins_cost(BRANCH_COST);
11279   format %{ "B$cmp  $xcc,$labl" %}
11280   ins_encode %{
11281     __ b(*($labl$$label), (AsmCondition)($cmp$$cmpcode));
11282   %}
11283   ins_pipe(br_cc);
11284 %}
11285 
11286 instruct branchConUL_LTGE(cmpOpUL cmp, flagsRegUL_LTGE xcc, label labl) %{
11287   match(If cmp xcc);
11288   effect(USE labl);
11289   predicate(_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::lt || _kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ge);
11290 
11291   size(4);
11292   ins_cost(BRANCH_COST);
11293   format %{ "B$cmp  $xcc,$labl" %}
11294   ins_encode %{
11295     __ b(*($labl$$label), (AsmCondition)($cmp$$cmpcode));
11296   %}
11297   ins_pipe(br_cc);
11298 %}
11299 
11300 instruct branchConUL_EQNE(cmpOpUL cmp, flagsRegUL_EQNE xcc, label labl) %{
11301   match(If cmp xcc);
11302   effect(USE labl);
11303   predicate(_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::eq || _kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ne);
11304 
11305   size(4);
11306   ins_cost(BRANCH_COST);
11307   format %{ "B$cmp  $xcc,$labl" %}
11308   ins_encode %{
11309     __ b(*($labl$$label), (AsmCondition)($cmp$$cmpcode));
11310   %}
11311   ins_pipe(br_cc);
11312 %}
11313 
11314 instruct branchConUL_LEGT(cmpOpUL_commute cmp, flagsRegUL_LEGT xcc, label labl) %{
11315   match(If cmp xcc);
11316   effect(USE labl);
11317   predicate(_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::gt || _kids[0]->_leaf->as_Bool()->_test._test == BoolTest::le);
11318 
11319   size(4);
11320   ins_cost(BRANCH_COST);
11321   format %{ "B$cmp  $xcc,$labl" %}
11322   ins_encode %{
11323     __ b(*($labl$$label), (AsmCondition)($cmp$$cmpcode));
11324   %}
11325   ins_pipe(br_cc);
11326 %}
11327 #endif
11328 
11329 instruct branchLoopEnd(cmpOp cmp, flagsReg icc, label labl) %{
11330   match(CountedLoopEnd cmp icc);
11331   effect(USE labl);
11332 
11333   size(4);
11334   ins_cost(BRANCH_COST);
11335   format %{ "B$cmp   $icc,$labl\t! Loop end" %}
11336   ins_encode %{
11337     __ b(*($labl$$label), (AsmCondition)($cmp$$cmpcode));
11338   %}
11339   ins_pipe(br_cc);
11340 %}
11341 
11342 // instruct branchLoopEndU(cmpOpU cmp, flagsRegU icc, label labl) %{
11343 //   match(CountedLoopEnd cmp icc);
11344 //   ins_pipe(br_cc);
11345 // %}
11346 
11347 // ============================================================================
11348 // Long Compare
11349 //
11350 // Currently we hold longs in 2 registers.  Comparing such values efficiently
11351 // is tricky.  The flavor of compare used depends on whether we are testing
11352 // for LT, LE, or EQ.  For a simple LT test we can check just the sign bit.
11353 // The GE test is the negated LT test.  The LE test can be had by commuting
11354 // the operands (yielding a GE test) and then negating; negate again for the
11355 // GT test.  The EQ test is done by ORcc'ing the high and low halves, and the
11356 // NE test is negated from that.
11357 
11358 // Due to a shortcoming in the ADLC, it mixes up expressions like:
11359 // (foo (CmpI (CmpL X Y) 0)) and (bar (CmpI (CmpL X 0L) 0)).  Note the
11360 // difference between 'Y' and '0L'.  The tree-matches for the CmpI sections
11361 // are collapsed internally in the ADLC's dfa-gen code.  The match for
11362 // (CmpI (CmpL X Y) 0) is silently replaced with (CmpI (CmpL X 0L) 0) and the
11363 // foo match ends up with the wrong leaf.  One fix is to not match both
11364 // reg-reg and reg-zero forms of long-compare.  This is unfortunate because
11365 // both forms beat the trinary form of long-compare and both are very useful
11366 // on Intel which has so few registers.
11367 
11368 // instruct branchCon_long(cmpOp cmp, flagsRegL xcc, label labl) %{
11369 //   match(If cmp xcc);
11370 //   ins_pipe(br_cc);
11371 // %}
11372 
11373 // Manifest a CmpL3 result in an integer register.  Very painful.
11374 // This is the test to avoid.
11375 #ifdef AARCH64
11376 instruct cmpL3_reg_reg(iRegI dst, iRegL src1, iRegL src2, flagsReg ccr) %{
11377   match(Set dst (CmpL3 src1 src2));
11378   // effect(KILL fcc); // nobody cares if flagsRegF is killed
11379   effect(KILL ccr);
11380   ins_cost(DEFAULT_COST*3); // FIXME
11381   size(12);
11382   format %{ "CMP    $src1,$src2\n\t"
11383             "CSET   $dst, gt\n\t"
11384             "CSINV  $dst, $dst, ZR, ge" %}
11385   ins_encode %{
11386     Register dst = $dst$$Register;
11387     __ cmp($src1$$Register, $src2$$Register);
11388     __ cset(dst, gt);            // 1 if '>', else 0
11389     __ csinv(dst, dst, ZR, ge);  // previous value if '>=', else -1
11390   %}
11391   ins_pipe( ialu_cconly_reg_reg ); // FIXME
11392 %}
11393 // TODO cmpL3_reg_imm
11394 #else
11395 instruct cmpL3_reg_reg(iRegI dst, iRegL src1, iRegL src2, flagsReg ccr ) %{
11396   match(Set dst (CmpL3 src1 src2) );
11397   effect( KILL ccr );
11398   ins_cost(6*DEFAULT_COST); // FIXME
11399   size(32);
11400   format %{
11401       "CMP    $src1.hi, $src2.hi\t\t! long\n"
11402     "\tMOV.gt $dst, 1\n"
11403     "\tmvn.lt $dst, 0\n"
11404     "\tB.ne   done\n"
11405     "\tSUBS   $dst, $src1.lo, $src2.lo\n"
11406     "\tMOV.hi $dst, 1\n"
11407     "\tmvn.lo $dst, 0\n"
11408     "done:"     %}
11409   ins_encode %{
11410     Label done;
11411     __ cmp($src1$$Register->successor(), $src2$$Register->successor());
11412     __ mov($dst$$Register, 1, gt);
11413     __ mvn($dst$$Register, 0, lt);
11414     __ b(done, ne);
11415     __ subs($dst$$Register, $src1$$Register, $src2$$Register);
11416     __ mov($dst$$Register, 1, hi);
11417     __ mvn($dst$$Register, 0, lo);
11418     __ bind(done);
11419   %}
11420   ins_pipe(cmpL_reg);
11421 %}
11422 #endif
11423 
11424 #ifndef AARCH64
11425 // Conditional move
11426 instruct cmovLL_reg_LTGE(cmpOpL cmp, flagsRegL_LTGE xcc, iRegL dst, iRegL src) %{
11427   match(Set dst (CMoveL (Binary cmp xcc) (Binary dst src)));
11428   predicate(_kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::lt || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ge );
11429 
11430   ins_cost(150);
11431   size(8);
11432   format %{ "MOV$cmp  $dst.lo,$src.lo\t! long\n\t"
11433             "MOV$cmp  $dst,$src.hi" %}
11434   ins_encode %{
11435     __ mov($dst$$Register, $src$$Register, (AsmCondition)($cmp$$cmpcode));
11436     __ mov($dst$$Register->successor(), $src$$Register->successor(), (AsmCondition)($cmp$$cmpcode));
11437   %}
11438   ins_pipe(ialu_reg);
11439 %}
11440 
11441 instruct cmovLL_reg_EQNE(cmpOpL cmp, flagsRegL_EQNE xcc, iRegL dst, iRegL src) %{
11442   match(Set dst (CMoveL (Binary cmp xcc) (Binary dst src)));
11443   predicate(_kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::eq || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ne );
11444 
11445   ins_cost(150);
11446   size(8);
11447   format %{ "MOV$cmp  $dst.lo,$src.lo\t! long\n\t"
11448             "MOV$cmp  $dst,$src.hi" %}
11449   ins_encode %{
11450     __ mov($dst$$Register, $src$$Register, (AsmCondition)($cmp$$cmpcode));
11451     __ mov($dst$$Register->successor(), $src$$Register->successor(), (AsmCondition)($cmp$$cmpcode));
11452   %}
11453   ins_pipe(ialu_reg);
11454 %}
11455 
11456 instruct cmovLL_reg_LEGT(cmpOpL_commute cmp, flagsRegL_LEGT xcc, iRegL dst, iRegL src) %{
11457   match(Set dst (CMoveL (Binary cmp xcc) (Binary dst src)));
11458   predicate(_kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::le || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::gt );
11459 
11460   ins_cost(150);
11461   size(8);
11462   format %{ "MOV$cmp  $dst.lo,$src.lo\t! long\n\t"
11463             "MOV$cmp  $dst,$src.hi" %}
11464   ins_encode %{
11465     __ mov($dst$$Register, $src$$Register, (AsmCondition)($cmp$$cmpcode));
11466     __ mov($dst$$Register->successor(), $src$$Register->successor(), (AsmCondition)($cmp$$cmpcode));
11467   %}
11468   ins_pipe(ialu_reg);
11469 %}
11470 
11471 instruct cmovLL_imm_LTGE(cmpOpL cmp, flagsRegL_LTGE xcc, iRegL dst, immL0 src) %{
11472   match(Set dst (CMoveL (Binary cmp xcc) (Binary dst src)));
11473   predicate(_kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::lt || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ge );
11474   ins_cost(140);
11475   size(8);
11476   format %{ "MOV$cmp  $dst.lo,0\t! long\n\t"
11477             "MOV$cmp  $dst,0" %}
11478   ins_encode %{
11479     __ mov($dst$$Register, 0, (AsmCondition)($cmp$$cmpcode));
11480     __ mov($dst$$Register->successor(), 0, (AsmCondition)($cmp$$cmpcode));
11481   %}
11482   ins_pipe(ialu_imm);
11483 %}
11484 
11485 instruct cmovLL_imm_EQNE(cmpOpL cmp, flagsRegL_EQNE xcc, iRegL dst, immL0 src) %{
11486   match(Set dst (CMoveL (Binary cmp xcc) (Binary dst src)));
11487   predicate(_kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::eq || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ne );
11488   ins_cost(140);
11489   size(8);
11490   format %{ "MOV$cmp  $dst.lo,0\t! long\n\t"
11491             "MOV$cmp  $dst,0" %}
11492   ins_encode %{
11493     __ mov($dst$$Register, 0, (AsmCondition)($cmp$$cmpcode));
11494     __ mov($dst$$Register->successor(), 0, (AsmCondition)($cmp$$cmpcode));
11495   %}
11496   ins_pipe(ialu_imm);
11497 %}
11498 
11499 instruct cmovLL_imm_LEGT(cmpOpL_commute cmp, flagsRegL_LEGT xcc, iRegL dst, immL0 src) %{
11500   match(Set dst (CMoveL (Binary cmp xcc) (Binary dst src)));
11501   predicate(_kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::le || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::gt );
11502   ins_cost(140);
11503   size(8);
11504   format %{ "MOV$cmp  $dst.lo,0\t! long\n\t"
11505             "MOV$cmp  $dst,0" %}
11506   ins_encode %{
11507     __ mov($dst$$Register, 0, (AsmCondition)($cmp$$cmpcode));
11508     __ mov($dst$$Register->successor(), 0, (AsmCondition)($cmp$$cmpcode));
11509   %}
11510   ins_pipe(ialu_imm);
11511 %}
11512 #endif // !AARCH64
11513 
11514 #ifndef AARCH64
11515 instruct cmovIL_reg_LTGE(cmpOpL cmp, flagsRegL_LTGE xcc, iRegI dst, iRegI src) %{
11516   match(Set dst (CMoveI (Binary cmp xcc) (Binary dst src)));
11517   predicate(_kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::lt || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ge );
11518 
11519   ins_cost(150);
11520   size(4);
11521   format %{ "MOV$cmp  $dst,$src" %}
11522   ins_encode %{
11523     __ mov($dst$$Register, $src$$Register, (AsmCondition)($cmp$$cmpcode));
11524   %}
11525   ins_pipe(ialu_reg);
11526 %}
11527 
11528 instruct cmovIL_reg_EQNE(cmpOpL cmp, flagsRegL_EQNE xcc, iRegI dst, iRegI src) %{
11529   match(Set dst (CMoveI (Binary cmp xcc) (Binary dst src)));
11530   predicate(_kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::eq || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ne );
11531 
11532   ins_cost(150);
11533   size(4);
11534   format %{ "MOV$cmp  $dst,$src" %}
11535   ins_encode %{
11536     __ mov($dst$$Register, $src$$Register, (AsmCondition)($cmp$$cmpcode));
11537   %}
11538   ins_pipe(ialu_reg);
11539 %}
11540 
11541 instruct cmovIL_reg_LEGT(cmpOpL_commute cmp, flagsRegL_LEGT xcc, iRegI dst, iRegI src) %{
11542   match(Set dst (CMoveI (Binary cmp xcc) (Binary dst src)));
11543   predicate(_kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::le || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::gt );
11544 
11545   ins_cost(150);
11546   size(4);
11547   format %{ "MOV$cmp  $dst,$src" %}
11548   ins_encode %{
11549     __ mov($dst$$Register, $src$$Register, (AsmCondition)($cmp$$cmpcode));
11550   %}
11551   ins_pipe(ialu_reg);
11552 %}
11553 #endif // !AARCH64
11554 
11555 #ifndef AARCH64
11556 instruct cmovIL_imm_LTGE(cmpOpL cmp, flagsRegL_LTGE xcc, iRegI dst, immI16 src) %{
11557   match(Set dst (CMoveI (Binary cmp xcc) (Binary dst src)));
11558   predicate(_kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::lt || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ge );
11559 
11560   ins_cost(140);
11561   format %{ "MOVW$cmp  $dst,$src" %}
11562   ins_encode %{
11563     __ movw($dst$$Register, $src$$constant, (AsmCondition)($cmp$$cmpcode));
11564   %}
11565   ins_pipe(ialu_imm);
11566 %}
11567 
11568 instruct cmovIL_imm_EQNE(cmpOpL cmp, flagsRegL_EQNE xcc, iRegI dst, immI16 src) %{
11569   match(Set dst (CMoveI (Binary cmp xcc) (Binary dst src)));
11570   predicate(_kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::eq || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ne );
11571 
11572   ins_cost(140);
11573   format %{ "MOVW$cmp  $dst,$src" %}
11574   ins_encode %{
11575     __ movw($dst$$Register, $src$$constant, (AsmCondition)($cmp$$cmpcode));
11576   %}
11577   ins_pipe(ialu_imm);
11578 %}
11579 
11580 instruct cmovIL_imm_LEGT(cmpOpL_commute cmp, flagsRegL_LEGT xcc, iRegI dst, immI16 src) %{
11581   match(Set dst (CMoveI (Binary cmp xcc) (Binary dst src)));
11582   predicate(_kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::le || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::gt );
11583 
11584   ins_cost(140);
11585   format %{ "MOVW$cmp  $dst,$src" %}
11586   ins_encode %{
11587     __ movw($dst$$Register, $src$$constant, (AsmCondition)($cmp$$cmpcode));
11588   %}
11589   ins_pipe(ialu_imm);
11590 %}
11591 
11592 instruct cmovPL_reg_LTGE(cmpOpL cmp, flagsRegL_LTGE xcc, iRegP dst, iRegP src) %{
11593   match(Set dst (CMoveP (Binary cmp xcc) (Binary dst src)));
11594   predicate(_kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::lt || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ge );
11595 
11596   ins_cost(150);
11597   size(4);
11598   format %{ "MOV$cmp  $dst,$src" %}
11599   ins_encode %{
11600     __ mov($dst$$Register, $src$$Register, (AsmCondition)($cmp$$cmpcode));
11601   %}
11602   ins_pipe(ialu_reg);
11603 %}
11604 
11605 instruct cmovPL_reg_EQNE(cmpOpL cmp, flagsRegL_EQNE xcc, iRegP dst, iRegP src) %{
11606   match(Set dst (CMoveP (Binary cmp xcc) (Binary dst src)));
11607   predicate(_kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::eq || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ne );
11608 
11609   ins_cost(150);
11610   size(4);
11611   format %{ "MOV$cmp  $dst,$src" %}
11612   ins_encode %{
11613     __ mov($dst$$Register, $src$$Register, (AsmCondition)($cmp$$cmpcode));
11614   %}
11615   ins_pipe(ialu_reg);
11616 %}
11617 
11618 instruct cmovPL_reg_LEGT(cmpOpL_commute cmp, flagsRegL_LEGT xcc, iRegP dst, iRegP src) %{
11619   match(Set dst (CMoveP (Binary cmp xcc) (Binary dst src)));
11620   predicate(_kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::le || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::gt );
11621 
11622   ins_cost(150);
11623   size(4);
11624   format %{ "MOV$cmp  $dst,$src" %}
11625   ins_encode %{
11626     __ mov($dst$$Register, $src$$Register, (AsmCondition)($cmp$$cmpcode));
11627   %}
11628   ins_pipe(ialu_reg);
11629 %}
11630 
11631 instruct cmovPL_imm_LTGE(cmpOpL cmp, flagsRegL_LTGE xcc, iRegP dst, immP0 src) %{
11632   match(Set dst (CMoveP (Binary cmp xcc) (Binary dst src)));
11633   predicate(_kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::lt || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ge );
11634 
11635   ins_cost(140);
11636   format %{ "MOVW$cmp  $dst,$src" %}
11637   ins_encode %{
11638     __ movw($dst$$Register, $src$$constant, (AsmCondition)($cmp$$cmpcode));
11639   %}
11640   ins_pipe(ialu_imm);
11641 %}
11642 
11643 instruct cmovPL_imm_EQNE(cmpOpL cmp, flagsRegL_EQNE xcc, iRegP dst, immP0 src) %{
11644   match(Set dst (CMoveP (Binary cmp xcc) (Binary dst src)));
11645   predicate(_kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::eq || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ne );
11646 
11647   ins_cost(140);
11648   format %{ "MOVW$cmp  $dst,$src" %}
11649   ins_encode %{
11650     __ movw($dst$$Register, $src$$constant, (AsmCondition)($cmp$$cmpcode));
11651   %}
11652   ins_pipe(ialu_imm);
11653 %}
11654 
11655 instruct cmovPL_imm_LEGT(cmpOpL_commute cmp, flagsRegL_LEGT xcc, iRegP dst, immP0 src) %{
11656   match(Set dst (CMoveP (Binary cmp xcc) (Binary dst src)));
11657   predicate(_kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::le || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::gt );
11658 
11659   ins_cost(140);
11660   format %{ "MOVW$cmp  $dst,$src" %}
11661   ins_encode %{
11662     __ movw($dst$$Register, $src$$constant, (AsmCondition)($cmp$$cmpcode));
11663   %}
11664   ins_pipe(ialu_imm);
11665 %}
11666 
11667 instruct cmovFL_reg_LTGE(cmpOpL cmp, flagsRegL_LTGE xcc, regF dst, regF src) %{
11668   match(Set dst (CMoveF (Binary cmp xcc) (Binary dst src)));
11669   predicate(_kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::lt || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ge );
11670   ins_cost(150);
11671   size(4);
11672   format %{ "FCPYS$cmp $dst,$src" %}
11673   ins_encode %{
11674     __ fcpys($dst$$FloatRegister, $src$$FloatRegister, (AsmCondition)($cmp$$cmpcode));
11675   %}
11676   ins_pipe(int_conditional_float_move);
11677 %}
11678 
11679 instruct cmovFL_reg_EQNE(cmpOpL cmp, flagsRegL_EQNE xcc, regF dst, regF src) %{
11680   match(Set dst (CMoveF (Binary cmp xcc) (Binary dst src)));
11681   predicate(_kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::eq || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ne );
11682   ins_cost(150);
11683   size(4);
11684   format %{ "FCPYS$cmp $dst,$src" %}
11685   ins_encode %{
11686     __ fcpys($dst$$FloatRegister, $src$$FloatRegister, (AsmCondition)($cmp$$cmpcode));
11687   %}
11688   ins_pipe(int_conditional_float_move);
11689 %}
11690 
11691 instruct cmovFL_reg_LEGT(cmpOpL_commute cmp, flagsRegL_LEGT xcc, regF dst, regF src) %{
11692   match(Set dst (CMoveF (Binary cmp xcc) (Binary dst src)));
11693   predicate(_kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::le || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::gt );
11694   ins_cost(150);
11695   size(4);
11696   format %{ "FCPYS$cmp $dst,$src" %}
11697   ins_encode %{
11698     __ fcpys($dst$$FloatRegister, $src$$FloatRegister, (AsmCondition)($cmp$$cmpcode));
11699   %}
11700   ins_pipe(int_conditional_float_move);
11701 %}
11702 
11703 instruct cmovDL_reg_LTGE(cmpOpL cmp, flagsRegL_LTGE xcc, regD dst, regD src) %{
11704   match(Set dst (CMoveD (Binary cmp xcc) (Binary dst src)));
11705   predicate(_kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::lt || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ge );
11706 
11707   ins_cost(150);
11708   size(4);
11709   format %{ "FCPYD$cmp $dst,$src" %}
11710   ins_encode %{
11711     __ fcpyd($dst$$FloatRegister, $src$$FloatRegister, (AsmCondition)($cmp$$cmpcode));
11712   %}
11713   ins_pipe(int_conditional_float_move);
11714 %}
11715 
11716 instruct cmovDL_reg_EQNE(cmpOpL cmp, flagsRegL_EQNE xcc, regD dst, regD src) %{
11717   match(Set dst (CMoveD (Binary cmp xcc) (Binary dst src)));
11718   predicate(_kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::eq || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ne );
11719 
11720   ins_cost(150);
11721   size(4);
11722   format %{ "FCPYD$cmp $dst,$src" %}
11723   ins_encode %{
11724     __ fcpyd($dst$$FloatRegister, $src$$FloatRegister, (AsmCondition)($cmp$$cmpcode));
11725   %}
11726   ins_pipe(int_conditional_float_move);
11727 %}
11728 
11729 instruct cmovDL_reg_LEGT(cmpOpL_commute cmp, flagsRegL_LEGT xcc, regD dst, regD src) %{
11730   match(Set dst (CMoveD (Binary cmp xcc) (Binary dst src)));
11731   predicate(_kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::le || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::gt );
11732 
11733   ins_cost(150);
11734   size(4);
11735   format %{ "FCPYD$cmp $dst,$src" %}
11736   ins_encode %{
11737     __ fcpyd($dst$$FloatRegister, $src$$FloatRegister, (AsmCondition)($cmp$$cmpcode));
11738   %}
11739   ins_pipe(int_conditional_float_move);
11740 %}
11741 #endif // !AARCH64
11742 
11743 // ============================================================================
11744 // Safepoint Instruction
11745 #ifdef AARCH64
11746 instruct safePoint_poll(iRegP poll, flagsReg icc, RtempRegP tmp) %{
11747   match(SafePoint poll);
11748   // The handler stub kills Rtemp
11749   effect(USE poll, KILL tmp, KILL icc);
11750 
11751   size(4);
11752   format %{ "LDR   ZR,[$poll]\t! Safepoint: poll for GC" %}
11753   ins_encode %{
11754     __ relocate(relocInfo::poll_type);
11755     __ ldr(ZR, Address($poll$$Register));
11756   %}
11757   ins_pipe(loadPollP);
11758 %}
11759 #else
11760 // rather than KILL R12, it would be better to use any reg as
11761 // TEMP. Can't do that at this point because it crashes the compiler
11762 instruct safePoint_poll(iRegP poll, R12RegI tmp, flagsReg icc) %{
11763   match(SafePoint poll);
11764   effect(USE poll, KILL tmp, KILL icc);
11765 
11766   size(4);
11767   format %{ "LDR   $tmp,[$poll]\t! Safepoint: poll for GC" %}
11768   ins_encode %{
11769     __ relocate(relocInfo::poll_type);
11770     __ ldr($tmp$$Register, Address($poll$$Register));
11771   %}
11772   ins_pipe(loadPollP);
11773 %}
11774 #endif
11775 
11776 
11777 // ============================================================================
11778 // Call Instructions
11779 // Call Java Static Instruction
11780 instruct CallStaticJavaDirect( method meth ) %{
11781   match(CallStaticJava);
11782   predicate(! ((CallStaticJavaNode*)n)->is_method_handle_invoke());
11783   effect(USE meth);
11784 
11785   ins_cost(CALL_COST);
11786   format %{ "CALL,static ==> " %}
11787   ins_encode( Java_Static_Call( meth ), call_epilog );
11788   ins_pipe(simple_call);
11789 %}
11790 
11791 // Call Java Static Instruction (method handle version)
11792 instruct CallStaticJavaHandle( method meth ) %{
11793   match(CallStaticJava);
11794   predicate(((CallStaticJavaNode*)n)->is_method_handle_invoke());
11795   effect(USE meth);
11796   // FP is saved by all callees (for interpreter stack correction).
11797   // We use it here for a similar purpose, in {preserve,restore}_FP.
11798 
11799   ins_cost(CALL_COST);
11800   format %{ "CALL,static/MethodHandle ==> " %}
11801   ins_encode( preserve_SP, Java_Static_Call( meth ), restore_SP, call_epilog );
11802   ins_pipe(simple_call);
11803 %}
11804 
11805 // Call Java Dynamic Instruction
11806 instruct CallDynamicJavaDirect( method meth ) %{
11807   match(CallDynamicJava);
11808   effect(USE meth);
11809 
11810   ins_cost(CALL_COST);
11811   format %{ "MOV_OOP    (empty),R_R8\n\t"
11812             "CALL,dynamic  ; NOP ==> " %}
11813   ins_encode( Java_Dynamic_Call( meth ), call_epilog );
11814   ins_pipe(call);
11815 %}
11816 
11817 // Call Runtime Instruction
11818 instruct CallRuntimeDirect(method meth) %{
11819   match(CallRuntime);
11820   effect(USE meth);
11821   ins_cost(CALL_COST);
11822   format %{ "CALL,runtime" %}
11823 #ifdef AARCH64
11824   ins_encode( save_last_PC, Java_To_Runtime( meth ),
11825               call_epilog );
11826 #else
11827   ins_encode( Java_To_Runtime( meth ),
11828               call_epilog );
11829 #endif
11830   ins_pipe(simple_call);
11831 %}
11832 
11833 // Call runtime without safepoint - same as CallRuntime
11834 instruct CallLeafDirect(method meth) %{
11835   match(CallLeaf);
11836   effect(USE meth);
11837   ins_cost(CALL_COST);
11838   format %{ "CALL,runtime leaf" %}
11839   // TODO: ned save_last_PC here?
11840   ins_encode( Java_To_Runtime( meth ),
11841               call_epilog );
11842   ins_pipe(simple_call);
11843 %}
11844 
11845 // Call runtime without safepoint - same as CallLeaf
11846 instruct CallLeafNoFPDirect(method meth) %{
11847   match(CallLeafNoFP);
11848   effect(USE meth);
11849   ins_cost(CALL_COST);
11850   format %{ "CALL,runtime leaf nofp" %}
11851   // TODO: ned save_last_PC here?
11852   ins_encode( Java_To_Runtime( meth ),
11853               call_epilog );
11854   ins_pipe(simple_call);
11855 %}
11856 
11857 // Tail Call; Jump from runtime stub to Java code.
11858 // Also known as an 'interprocedural jump'.
11859 // Target of jump will eventually return to caller.
11860 // TailJump below removes the return address.
11861 instruct TailCalljmpInd(IPRegP jump_target, inline_cache_regP method_oop) %{
11862   match(TailCall jump_target method_oop );
11863 
11864   ins_cost(CALL_COST);
11865   format %{ "MOV    Rexception_pc, LR\n\t"
11866             "jump   $jump_target  \t! $method_oop holds method oop" %}
11867   ins_encode %{
11868     __ mov(Rexception_pc, LR);   // this is used only to call
11869                                  // StubRoutines::forward_exception_entry()
11870                                  // which expects PC of exception in
11871                                  // R5. FIXME?
11872     __ jump($jump_target$$Register);
11873   %}
11874   ins_pipe(tail_call);
11875 %}
11876 
11877 
11878 // Return Instruction
11879 instruct Ret() %{
11880   match(Return);
11881 
11882   format %{ "ret LR" %}
11883 
11884   ins_encode %{
11885     __ ret(LR);
11886   %}
11887 
11888   ins_pipe(br);
11889 %}
11890 
11891 
11892 // Tail Jump; remove the return address; jump to target.
11893 // TailCall above leaves the return address around.
11894 // TailJump is used in only one place, the rethrow_Java stub (fancy_jump=2).
11895 // ex_oop (Exception Oop) is needed in %o0 at the jump. As there would be a
11896 // "restore" before this instruction (in Epilogue), we need to materialize it
11897 // in %i0.
11898 instruct tailjmpInd(IPRegP jump_target, RExceptionRegP ex_oop) %{
11899   match( TailJump jump_target ex_oop );
11900   ins_cost(CALL_COST);
11901   format %{ "MOV    Rexception_pc, LR\n\t"
11902             "jump   $jump_target \t! $ex_oop holds exc. oop" %}
11903   ins_encode %{
11904     __ mov(Rexception_pc, LR);
11905     __ jump($jump_target$$Register);
11906   %}
11907   ins_pipe(tail_call);
11908 %}
11909 
11910 // Create exception oop: created by stack-crawling runtime code.
11911 // Created exception is now available to this handler, and is setup
11912 // just prior to jumping to this handler.  No code emitted.
11913 instruct CreateException( RExceptionRegP ex_oop )
11914 %{
11915   match(Set ex_oop (CreateEx));
11916   ins_cost(0);
11917 
11918   size(0);
11919   // use the following format syntax
11920   format %{ "! exception oop is in Rexception_obj; no code emitted" %}
11921   ins_encode();
11922   ins_pipe(empty);
11923 %}
11924 
11925 
11926 // Rethrow exception:
11927 // The exception oop will come in the first argument position.
11928 // Then JUMP (not call) to the rethrow stub code.
11929 instruct RethrowException()
11930 %{
11931   match(Rethrow);
11932   ins_cost(CALL_COST);
11933 
11934   // use the following format syntax
11935   format %{ "b    rethrow_stub" %}
11936   ins_encode %{
11937     Register scratch = R1_tmp;
11938     assert_different_registers(scratch, c_rarg0, LR);
11939     __ jump(OptoRuntime::rethrow_stub(), relocInfo::runtime_call_type, scratch);
11940   %}
11941   ins_pipe(tail_call);
11942 %}
11943 
11944 
11945 // Die now
11946 instruct ShouldNotReachHere( )
11947 %{
11948   match(Halt);
11949   ins_cost(CALL_COST);
11950 
11951   size(4);
11952   // Use the following format syntax
11953   format %{ "ShouldNotReachHere" %}
11954   ins_encode %{
11955 #ifdef AARCH64
11956     __ dpcs1(0xdead);
11957 #else
11958     __ udf(0xdead);
11959 #endif
11960   %}
11961   ins_pipe(tail_call);
11962 %}
11963 
11964 // ============================================================================
11965 // The 2nd slow-half of a subtype check.  Scan the subklass's 2ndary superklass
11966 // array for an instance of the superklass.  Set a hidden internal cache on a
11967 // hit (cache is checked with exposed code in gen_subtype_check()).  Return
11968 // not zero for a miss or zero for a hit.  The encoding ALSO sets flags.
11969 instruct partialSubtypeCheck( R0RegP index, R1RegP sub, R2RegP super, flagsRegP pcc, LRRegP lr ) %{
11970   match(Set index (PartialSubtypeCheck sub super));
11971   effect( KILL pcc, KILL lr );
11972   ins_cost(DEFAULT_COST*10);
11973   format %{ "CALL   PartialSubtypeCheck" %}
11974   ins_encode %{
11975     __ call(StubRoutines::Arm::partial_subtype_check(), relocInfo::runtime_call_type);
11976   %}
11977   ins_pipe(partial_subtype_check_pipe);
11978 %}
11979 
11980 /* instruct partialSubtypeCheck_vs_zero( flagsRegP pcc, o1RegP sub, o2RegP super, immP0 zero, o0RegP idx, o7RegP o7 ) %{ */
11981 /*   match(Set pcc (CmpP (PartialSubtypeCheck sub super) zero)); */
11982 /*   ins_pipe(partial_subtype_check_pipe); */
11983 /* %} */
11984 
11985 
11986 // ============================================================================
11987 // inlined locking and unlocking
11988 
11989 #ifdef AARCH64
11990 instruct cmpFastLock(flagsRegP pcc, iRegP object, iRegP box, iRegP scratch2, iRegP scratch, iRegP scratch3 )
11991 #else
11992 instruct cmpFastLock(flagsRegP pcc, iRegP object, iRegP box, iRegP scratch2, iRegP scratch )
11993 #endif
11994 %{
11995   match(Set pcc (FastLock object box));
11996 
11997 #ifdef AARCH64
11998   effect(TEMP scratch, TEMP scratch2, TEMP scratch3);
11999 #else
12000   effect(TEMP scratch, TEMP scratch2);
12001 #endif
12002   ins_cost(100);
12003 
12004 #ifdef AARCH64
12005   format %{ "FASTLOCK  $object, $box; KILL $scratch, $scratch2, $scratch3" %}
12006   ins_encode %{
12007     __ fast_lock($object$$Register, $box$$Register, $scratch$$Register, $scratch2$$Register, $scratch3$$Register);
12008   %}
12009 #else
12010   format %{ "FASTLOCK  $object, $box; KILL $scratch, $scratch2" %}
12011   ins_encode %{
12012     __ fast_lock($object$$Register, $box$$Register, $scratch$$Register, $scratch2$$Register);
12013   %}
12014 #endif
12015   ins_pipe(long_memory_op);
12016 %}
12017 
12018 
12019 #ifdef AARCH64
12020 instruct cmpFastUnlock(flagsRegP pcc, iRegP object, iRegP box, iRegP scratch2, iRegP scratch, iRegP scratch3 ) %{
12021   match(Set pcc (FastUnlock object box));
12022   effect(TEMP scratch, TEMP scratch2, TEMP scratch3);
12023   ins_cost(100);
12024 
12025   format %{ "FASTUNLOCK  $object, $box; KILL $scratch, $scratch2, $scratch3" %}
12026   ins_encode %{
12027     __ fast_unlock($object$$Register, $box$$Register, $scratch$$Register, $scratch2$$Register, $scratch3$$Register);
12028   %}
12029   ins_pipe(long_memory_op);
12030 %}
12031 #else
12032 instruct cmpFastUnlock(flagsRegP pcc, iRegP object, iRegP box, iRegP scratch2, iRegP scratch ) %{
12033   match(Set pcc (FastUnlock object box));
12034   effect(TEMP scratch, TEMP scratch2);
12035   ins_cost(100);
12036 
12037   format %{ "FASTUNLOCK  $object, $box; KILL $scratch, $scratch2" %}
12038   ins_encode %{
12039     __ fast_unlock($object$$Register, $box$$Register, $scratch$$Register, $scratch2$$Register);
12040   %}
12041   ins_pipe(long_memory_op);
12042 %}
12043 #endif
12044 
12045 #ifdef AARCH64
12046 // TODO: add version that takes immI cnt?
12047 instruct clear_array(iRegX cnt, iRegP base, iRegP ptr, iRegX temp, Universe dummy, flagsReg cpsr) %{
12048   match(Set dummy (ClearArray cnt base));
12049   effect(TEMP temp, TEMP ptr, KILL cpsr);
12050   ins_cost(300);
12051   format %{
12052       "        MOV    $temp,$cnt\n"
12053       "        ADD    $ptr,$base,$cnt\n"
12054       "        SUBS   $temp,$temp,16\t! Count down dword pair in bytes\n"
12055       "        B.lt   done16\n"
12056       "loop:   STP    ZR,ZR,[$ptr,-16]!\n"
12057       "        SUBS   $temp,$temp,16\t! Count down dword pair in bytes\n"
12058       "        B.ge   loop\t! Clearing loop\n"
12059       "done16: ADDS   $temp,$temp,8\t! Room for 1 more long?\n"
12060       "        B.lt   done\n"
12061       "        STR    ZR,[$base+$temp]\n"
12062       "done:"
12063   %}
12064   ins_encode %{
12065     // TODO: preload?
12066     __ mov($temp$$Register, $cnt$$Register);
12067     __ add($ptr$$Register, $base$$Register, $cnt$$Register);
12068     Label loop, done, done16;
12069     __ subs($temp$$Register, $temp$$Register, 16);
12070     __ b(done16, lt);
12071     __ bind(loop);
12072     __ stp(ZR, ZR, Address($ptr$$Register, -16, pre_indexed));
12073     __ subs($temp$$Register, $temp$$Register, 16);
12074     __ b(loop, ge);
12075     __ bind(done16);
12076     __ adds($temp$$Register, $temp$$Register, 8);
12077     __ b(done, lt);
12078     // $temp should be 0 here
12079     __ str(ZR, Address($base$$Register, $temp$$Register));
12080     __ bind(done);
12081   %}
12082   ins_pipe(long_memory_op);
12083 %}
12084 #else
12085 // Count and Base registers are fixed because the allocator cannot
12086 // kill unknown registers.  The encodings are generic.
12087 instruct clear_array(iRegX cnt, iRegP base, iRegI temp, iRegX zero, Universe dummy, flagsReg cpsr) %{
12088   match(Set dummy (ClearArray cnt base));
12089   effect(TEMP temp, TEMP zero, KILL cpsr);
12090   ins_cost(300);
12091   format %{ "MOV    $zero,0\n"
12092       "        MOV    $temp,$cnt\n"
12093       "loop:   SUBS   $temp,$temp,4\t! Count down a dword of bytes\n"
12094       "        STR.ge $zero,[$base+$temp]\t! delay slot"
12095       "        B.gt   loop\t\t! Clearing loop\n" %}
12096   ins_encode %{
12097     __ mov($zero$$Register, 0);
12098     __ mov($temp$$Register, $cnt$$Register);
12099     Label(loop);
12100     __ bind(loop);
12101     __ subs($temp$$Register, $temp$$Register, 4);
12102     __ str($zero$$Register, Address($base$$Register, $temp$$Register), ge);
12103     __ b(loop, gt);
12104   %}
12105   ins_pipe(long_memory_op);
12106 %}
12107 #endif
12108 
12109 #ifdef XXX
12110 // FIXME: Why R0/R1/R2/R3?
12111 instruct string_compare(R0RegP str1, R1RegP str2, R2RegI cnt1, R3RegI cnt2, iRegI result,
12112                         iRegI tmp1, iRegI tmp2, flagsReg ccr) %{
12113   predicate(!CompactStrings);
12114   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
12115   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL ccr, TEMP tmp1, TEMP tmp2);
12116   ins_cost(300);
12117   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   // TEMP $tmp1, $tmp2" %}
12118   ins_encode( enc_String_Compare(str1, str2, cnt1, cnt2, result, tmp1, tmp2) );
12119 
12120   ins_pipe(long_memory_op);
12121 %}
12122 
12123 // FIXME: Why R0/R1/R2?
12124 instruct string_equals(R0RegP str1, R1RegP str2, R2RegI cnt, iRegI result, iRegI tmp1, iRegI tmp2,
12125                        flagsReg ccr) %{
12126   predicate(!CompactStrings);
12127   match(Set result (StrEquals (Binary str1 str2) cnt));
12128   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt, TEMP tmp1, TEMP tmp2, TEMP result, KILL ccr);
12129 
12130   ins_cost(300);
12131   format %{ "String Equals $str1,$str2,$cnt -> $result   // TEMP $tmp1, $tmp2" %}
12132   ins_encode( enc_String_Equals(str1, str2, cnt, result, tmp1, tmp2) );
12133   ins_pipe(long_memory_op);
12134 %}
12135 
12136 // FIXME: Why R0/R1?
12137 instruct array_equals(R0RegP ary1, R1RegP ary2, iRegI tmp1, iRegI tmp2, iRegI tmp3, iRegI result,
12138                       flagsReg ccr) %{
12139   predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
12140   match(Set result (AryEq ary1 ary2));
12141   effect(USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP result, KILL ccr);
12142 
12143   ins_cost(300);
12144   format %{ "Array Equals $ary1,$ary2 -> $result   // TEMP $tmp1,$tmp2,$tmp3" %}
12145   ins_encode( enc_Array_Equals(ary1, ary2, tmp1, tmp2, tmp3, result));
12146   ins_pipe(long_memory_op);
12147 %}
12148 #endif
12149 
12150 //---------- Zeros Count Instructions ------------------------------------------
12151 
12152 instruct countLeadingZerosI(iRegI dst, iRegI src) %{
12153   match(Set dst (CountLeadingZerosI src));
12154   size(4);
12155   format %{ "CLZ_32 $dst,$src" %}
12156   ins_encode %{
12157     __ clz_32($dst$$Register, $src$$Register);
12158   %}
12159   ins_pipe(ialu_reg);
12160 %}
12161 
12162 #ifdef AARCH64
12163 instruct countLeadingZerosL(iRegI dst, iRegL src) %{
12164   match(Set dst (CountLeadingZerosL src));
12165   size(4);
12166   format %{ "CLZ $dst,$src" %}
12167   ins_encode %{
12168     __ clz($dst$$Register, $src$$Register);
12169   %}
12170   ins_pipe(ialu_reg);
12171 %}
12172 #else
12173 instruct countLeadingZerosL(iRegI dst, iRegL src, iRegI tmp, flagsReg ccr) %{
12174   match(Set dst (CountLeadingZerosL src));
12175   effect(TEMP tmp, TEMP dst, KILL ccr);
12176   size(16);
12177   format %{ "CLZ    $dst,$src.hi\n\t"
12178             "TEQ    $dst,32\n\t"
12179             "CLZ.eq $tmp,$src.lo\n\t"
12180             "ADD.eq $dst, $dst, $tmp\n\t" %}
12181   ins_encode %{
12182     __ clz($dst$$Register, $src$$Register->successor());
12183     __ teq($dst$$Register, 32);
12184     __ clz($tmp$$Register, $src$$Register, eq);
12185     __ add($dst$$Register, $dst$$Register, $tmp$$Register, eq);
12186   %}
12187   ins_pipe(ialu_reg);
12188 %}
12189 #endif
12190 
12191 instruct countTrailingZerosI(iRegI dst, iRegI src, iRegI tmp) %{
12192   match(Set dst (CountTrailingZerosI src));
12193   effect(TEMP tmp);
12194   size(8);
12195   format %{ "RBIT_32 $tmp, $src\n\t"
12196             "CLZ_32  $dst,$tmp" %}
12197   ins_encode %{
12198     __ rbit_32($tmp$$Register, $src$$Register);
12199     __ clz_32($dst$$Register, $tmp$$Register);
12200   %}
12201   ins_pipe(ialu_reg);
12202 %}
12203 
12204 #ifdef AARCH64
12205 instruct countTrailingZerosL(iRegI dst, iRegL src, iRegL tmp) %{
12206   match(Set dst (CountTrailingZerosL src));
12207   effect(TEMP tmp);
12208   size(8);
12209   format %{ "RBIT $tmp, $src\n\t"
12210             "CLZ  $dst,$tmp" %}
12211   ins_encode %{
12212     __ rbit($tmp$$Register, $src$$Register);
12213     __ clz($dst$$Register, $tmp$$Register);
12214   %}
12215   ins_pipe(ialu_reg);
12216 %}
12217 #else
12218 instruct countTrailingZerosL(iRegI dst, iRegL src, iRegI tmp, flagsReg ccr) %{
12219   match(Set dst (CountTrailingZerosL src));
12220   effect(TEMP tmp, TEMP dst, KILL ccr);
12221   size(24);
12222   format %{ "RBIT   $tmp,$src.lo\n\t"
12223             "CLZ    $dst,$tmp\n\t"
12224             "TEQ    $dst,32\n\t"
12225             "RBIT   $tmp,$src.hi\n\t"
12226             "CLZ.eq $tmp,$tmp\n\t"
12227             "ADD.eq $dst,$dst,$tmp\n\t" %}
12228   ins_encode %{
12229     __ rbit($tmp$$Register, $src$$Register);
12230     __ clz($dst$$Register, $tmp$$Register);
12231     __ teq($dst$$Register, 32);
12232     __ rbit($tmp$$Register, $src$$Register->successor());
12233     __ clz($tmp$$Register, $tmp$$Register, eq);
12234     __ add($dst$$Register, $dst$$Register, $tmp$$Register, eq);
12235   %}
12236   ins_pipe(ialu_reg);
12237 %}
12238 #endif
12239 
12240 
12241 //---------- Population Count Instructions -------------------------------------
12242 
12243 #ifdef AARCH64
12244 instruct popCountI(iRegI dst, iRegI src, regD_low tmp) %{
12245   predicate(UsePopCountInstruction);
12246   match(Set dst (PopCountI src));
12247   effect(TEMP tmp);
12248   size(20);
12249 
12250   format %{ "MOV_W      $dst,$src\n\t"
12251             "FMOV_dx    $tmp,$dst\n\t"
12252             "VCNT       $tmp.8B,$tmp.8B\n\t"
12253             "ADDV       $tmp.B,$tmp.8B\n\t"
12254             "FMRS       $dst,$tmp" %}
12255 
12256   ins_encode %{
12257     __ mov_w($dst$$Register, $src$$Register);
12258     __ fmov_dx($tmp$$FloatRegister, $dst$$Register);
12259     int quad = 0;
12260     int cnt_size = 0; // VELEM_SIZE_8
12261     __ vcnt($tmp$$FloatRegister, $tmp$$FloatRegister, quad, cnt_size);
12262     int add_size = 0; // VELEM_SIZE_8
12263     __ addv($tmp$$FloatRegister, $tmp$$FloatRegister, quad, add_size);
12264     __ fmrs($dst$$Register, $tmp$$FloatRegister);
12265   %}
12266   ins_pipe(ialu_reg); // FIXME
12267 %}
12268 #else
12269 instruct popCountI(iRegI dst, iRegI src, regD_low tmp) %{
12270   predicate(UsePopCountInstruction);
12271   match(Set dst (PopCountI src));
12272   effect(TEMP tmp);
12273 
12274   format %{ "FMSR       $tmp,$src\n\t"
12275             "VCNT.8     $tmp,$tmp\n\t"
12276             "VPADDL.U8  $tmp,$tmp\n\t"
12277             "VPADDL.U16 $tmp,$tmp\n\t"
12278             "FMRS       $dst,$tmp" %}
12279   size(20);
12280 
12281   ins_encode %{
12282     __ fmsr($tmp$$FloatRegister, $src$$Register);
12283     __ vcnt($tmp$$FloatRegister, $tmp$$FloatRegister);
12284     __ vpaddl($tmp$$FloatRegister, $tmp$$FloatRegister, 8, 0);
12285     __ vpaddl($tmp$$FloatRegister, $tmp$$FloatRegister, 16, 0);
12286     __ fmrs($dst$$Register, $tmp$$FloatRegister);
12287   %}
12288   ins_pipe(ialu_reg); // FIXME
12289 %}
12290 #endif
12291 
12292 #ifdef AARCH64
12293 instruct popCountL(iRegI dst, iRegL src, regD tmp) %{
12294   predicate(UsePopCountInstruction);
12295   match(Set dst (PopCountL src));
12296   effect(TEMP tmp);
12297   size(16);
12298 
12299   format %{ "FMOV_dx    $tmp,$src\n\t"
12300             "VCNT       $tmp.8B,$tmp.8B\n\t"
12301             "ADDV       $tmp.B,$tmp.8B\n\t"
12302             "FMOV_ws    $dst,$tmp" %}
12303 
12304   ins_encode %{
12305     __ fmov_dx($tmp$$FloatRegister, $src$$Register);
12306     int quad = 0;
12307     int cnt_size = 0;
12308     __ vcnt($tmp$$FloatRegister, $tmp$$FloatRegister, quad, cnt_size);
12309     int add_size = 0;
12310     __ addv($tmp$$FloatRegister, $tmp$$FloatRegister, quad, add_size);
12311     __ fmov_ws($dst$$Register, $tmp$$FloatRegister);
12312   %}
12313   ins_pipe(ialu_reg); // FIXME
12314 %}
12315 #else
12316 // Note: Long.bitCount(long) returns an int.
12317 instruct popCountL(iRegI dst, iRegL src, regD_low tmp) %{
12318   predicate(UsePopCountInstruction);
12319   match(Set dst (PopCountL src));
12320   effect(TEMP tmp);
12321 
12322   format %{ "FMDRR       $tmp,$src.lo,$src.hi\n\t"
12323             "VCNT.8      $tmp,$tmp\n\t"
12324             "VPADDL.U8   $tmp,$tmp\n\t"
12325             "VPADDL.U16  $tmp,$tmp\n\t"
12326             "VPADDL.U32  $tmp,$tmp\n\t"
12327             "FMRS        $dst,$tmp" %}
12328 
12329   size(32);
12330 
12331   ins_encode %{
12332     __ fmdrr($tmp$$FloatRegister, $src$$Register, $src$$Register->successor());
12333     __ vcnt($tmp$$FloatRegister, $tmp$$FloatRegister);
12334     __ vpaddl($tmp$$FloatRegister, $tmp$$FloatRegister, 8, 0);
12335     __ vpaddl($tmp$$FloatRegister, $tmp$$FloatRegister, 16, 0);
12336     __ vpaddl($tmp$$FloatRegister, $tmp$$FloatRegister, 32, 0);
12337     __ fmrs($dst$$Register, $tmp$$FloatRegister);
12338   %}
12339   ins_pipe(ialu_reg);
12340 %}
12341 #endif
12342 
12343 
12344 // ============================================================================
12345 //------------Bytes reverse--------------------------------------------------
12346 
12347 instruct bytes_reverse_int(iRegI dst, iRegI src) %{
12348   match(Set dst (ReverseBytesI src));
12349 
12350   size(4);
12351   format %{ "REV32 $dst,$src" %}
12352   ins_encode %{
12353 #ifdef AARCH64
12354     __ rev_w($dst$$Register, $src$$Register);
12355     // high 32 bits zeroed, not sign extended
12356 #else
12357     __ rev($dst$$Register, $src$$Register);
12358 #endif
12359   %}
12360   ins_pipe( iload_mem ); // FIXME
12361 %}
12362 
12363 instruct bytes_reverse_long(iRegL dst, iRegL src) %{
12364   match(Set dst (ReverseBytesL src));
12365 #ifdef AARCH64
12366 //size(4);
12367   format %{ "REV $dst,$src"  %}
12368   ins_encode %{
12369     __ rev($dst$$Register, $src$$Register);
12370   %}
12371   ins_pipe(ialu_reg_reg); // FIXME
12372 #else
12373   effect(TEMP dst);
12374   size(8);
12375   format %{ "REV $dst.lo,$src.lo\n\t"
12376             "REV $dst.hi,$src.hi" %}
12377   ins_encode %{
12378     __ rev($dst$$Register, $src$$Register->successor());
12379     __ rev($dst$$Register->successor(), $src$$Register);
12380   %}
12381   ins_pipe( iload_mem ); // FIXME
12382 #endif
12383 %}
12384 
12385 instruct bytes_reverse_unsigned_short(iRegI dst, iRegI src) %{
12386   match(Set dst (ReverseBytesUS src));
12387 #ifdef AARCH64
12388   size(4);
12389   format %{ "REV16_W $dst,$src" %}
12390   ins_encode %{
12391     __ rev16_w($dst$$Register, $src$$Register);
12392     // high 32 bits zeroed
12393   %}
12394 #else
12395   size(4);
12396   format %{ "REV16 $dst,$src" %}
12397   ins_encode %{
12398     __ rev16($dst$$Register, $src$$Register);
12399   %}
12400 #endif
12401   ins_pipe( iload_mem ); // FIXME
12402 %}
12403 
12404 instruct bytes_reverse_short(iRegI dst, iRegI src) %{
12405   match(Set dst (ReverseBytesS src));
12406 #ifdef AARCH64
12407   size(8);
12408   format %{ "REV16_W $dst,$src\n\t"
12409             "SIGN_EXT16 $dst" %}
12410   ins_encode %{
12411     __ rev16_w($dst$$Register, $src$$Register);
12412     __ sign_extend($dst$$Register, $dst$$Register, 16);
12413   %}
12414 #else
12415   size(4);
12416   format %{ "REVSH $dst,$src" %}
12417   ins_encode %{
12418     __ revsh($dst$$Register, $src$$Register);
12419   %}
12420 #endif
12421   ins_pipe( iload_mem ); // FIXME
12422 %}
12423 
12424 
12425 // ====================VECTOR INSTRUCTIONS=====================================
12426 
12427 // Load Aligned Packed values into a Double Register
12428 instruct loadV8(vecD dst, memoryD mem) %{
12429   predicate(n->as_LoadVector()->memory_size() == 8);
12430   match(Set dst (LoadVector mem));
12431   ins_cost(MEMORY_REF_COST);
12432   size(4);
12433   format %{ "FLDD   $mem,$dst\t! load vector (8 bytes)" %}
12434   ins_encode %{
12435     __ ldr_double($dst$$FloatRegister, $mem$$Address);
12436   %}
12437   ins_pipe(floadD_mem);
12438 %}
12439 
12440 // Load Aligned Packed values into a Double Register Pair
12441 instruct loadV16(vecX dst, memoryvld mem) %{
12442   predicate(n->as_LoadVector()->memory_size() == 16);
12443   match(Set dst (LoadVector mem));
12444   ins_cost(MEMORY_REF_COST);
12445   size(4);
12446   format %{ "VLD1   $mem,$dst.Q\t! load vector (16 bytes)" %}
12447   ins_encode %{
12448     __ vld1($dst$$FloatRegister, $mem$$Address, MacroAssembler::VELEM_SIZE_16, 128);
12449   %}
12450   ins_pipe(floadD_mem); // FIXME
12451 %}
12452 
12453 // Store Vector in Double register to memory
12454 instruct storeV8(memoryD mem, vecD src) %{
12455   predicate(n->as_StoreVector()->memory_size() == 8);
12456   match(Set mem (StoreVector mem src));
12457   ins_cost(MEMORY_REF_COST);
12458   size(4);
12459   format %{ "FSTD   $src,$mem\t! store vector (8 bytes)" %}
12460   ins_encode %{
12461     __ str_double($src$$FloatRegister, $mem$$Address);
12462   %}
12463   ins_pipe(fstoreD_mem_reg);
12464 %}
12465 
12466 // Store Vector in Double Register Pair to memory
12467 instruct storeV16(memoryvld mem, vecX src) %{
12468   predicate(n->as_StoreVector()->memory_size() == 16);
12469   match(Set mem (StoreVector mem src));
12470   ins_cost(MEMORY_REF_COST);
12471   size(4);
12472   format %{ "VST1   $src,$mem\t! store vector (16 bytes)" %}
12473   ins_encode %{
12474     __ vst1($src$$FloatRegister, $mem$$Address, MacroAssembler::VELEM_SIZE_16, 128);
12475   %}
12476   ins_pipe(fstoreD_mem_reg); // FIXME
12477 %}
12478 
12479 #ifndef AARCH64
12480 // Replicate scalar to packed byte values in Double register
12481 instruct Repl8B_reg(vecD dst, iRegI src, iRegI tmp) %{
12482   predicate(n->as_Vector()->length() == 8);
12483   match(Set dst (ReplicateB src));
12484   ins_cost(DEFAULT_COST*4);
12485   effect(TEMP tmp);
12486   size(16);
12487 
12488   // FIXME: could use PKH instruction instead?
12489   format %{ "LSL      $tmp, $src, 24 \n\t"
12490             "OR       $tmp, $tmp, ($tmp >> 8) \n\t"
12491             "OR       $tmp, $tmp, ($tmp >> 16) \n\t"
12492             "FMDRR    $dst,$tmp,$tmp\t" %}
12493   ins_encode %{
12494     __ mov($tmp$$Register, AsmOperand($src$$Register, lsl, 24));
12495     __ orr($tmp$$Register, $tmp$$Register, AsmOperand($tmp$$Register, lsr, 8));
12496     __ orr($tmp$$Register, $tmp$$Register, AsmOperand($tmp$$Register, lsr, 16));
12497     __ fmdrr($dst$$FloatRegister, $tmp$$Register, $tmp$$Register);
12498   %}
12499   ins_pipe(ialu_reg); // FIXME
12500 %}
12501 #endif /* !AARCH64 */
12502 
12503 // Replicate scalar to packed byte values in Double register
12504 instruct Repl8B_reg_simd(vecD dst, iRegI src) %{
12505   predicate(n->as_Vector()->length_in_bytes() == 8 && VM_Version::has_simd());
12506   match(Set dst (ReplicateB src));
12507   size(4);
12508 
12509   format %{ "VDUP.8 $dst,$src\t" %}
12510   ins_encode %{
12511     bool quad = false;
12512     __ vdupI($dst$$FloatRegister, $src$$Register,
12513              MacroAssembler::VELEM_SIZE_8, quad);
12514   %}
12515   ins_pipe(ialu_reg); // FIXME
12516 %}
12517 
12518 // Replicate scalar to packed byte values in Double register pair
12519 instruct Repl16B_reg(vecX dst, iRegI src) %{
12520   predicate(n->as_Vector()->length_in_bytes() == 16);
12521   match(Set dst (ReplicateB src));
12522   size(4);
12523 
12524   format %{ "VDUP.8 $dst.Q,$src\t" %}
12525   ins_encode %{
12526     bool quad = true;
12527     __ vdupI($dst$$FloatRegister, $src$$Register,
12528              MacroAssembler::VELEM_SIZE_8, quad);
12529   %}
12530   ins_pipe(ialu_reg); // FIXME
12531 %}
12532 
12533 #ifndef AARCH64
12534 // Replicate scalar constant to packed byte values in Double register
12535 instruct Repl8B_immI(vecD dst, immI src, iRegI tmp) %{
12536   predicate(n->as_Vector()->length() == 8);
12537   match(Set dst (ReplicateB src));
12538   ins_cost(DEFAULT_COST*2);
12539   effect(TEMP tmp);
12540   size(12);
12541 
12542   format %{ "MOV      $tmp, Repl4($src))\n\t"
12543             "FMDRR    $dst,$tmp,$tmp\t" %}
12544   ins_encode( LdReplImmI(src, dst, tmp, (4), (1)) );
12545   ins_pipe(loadConFD); // FIXME
12546 %}
12547 #endif /* !AARCH64 */
12548 
12549 // Replicate scalar constant to packed byte values in Double register
12550 // TODO: support negative constants with MVNI?
12551 instruct Repl8B_immU8(vecD dst, immU8 src) %{
12552   predicate(n->as_Vector()->length_in_bytes() == 8 && VM_Version::has_simd());
12553   match(Set dst (ReplicateB src));
12554   size(4);
12555 
12556   format %{ "VMOV.U8  $dst,$src" %}
12557   ins_encode %{
12558     bool quad = false;
12559     __ vmovI($dst$$FloatRegister, $src$$constant,
12560              MacroAssembler::VELEM_SIZE_8, quad);
12561   %}
12562   ins_pipe(loadConFD); // FIXME
12563 %}
12564 
12565 // Replicate scalar constant to packed byte values in Double register pair
12566 instruct Repl16B_immU8(vecX dst, immU8 src) %{
12567   predicate(n->as_Vector()->length_in_bytes() == 16 && VM_Version::has_simd());
12568   match(Set dst (ReplicateB src));
12569   size(4);
12570 
12571   format %{ "VMOV.U8  $dst.Q,$src" %}
12572   ins_encode %{
12573     bool quad = true;
12574     __ vmovI($dst$$FloatRegister, $src$$constant,
12575              MacroAssembler::VELEM_SIZE_8, quad);
12576   %}
12577   ins_pipe(loadConFD); // FIXME
12578 %}
12579 
12580 #ifndef AARCH64
12581 // Replicate scalar to packed short/char values into Double register
12582 instruct Repl4S_reg(vecD dst, iRegI src, iRegI tmp) %{
12583   predicate(n->as_Vector()->length() == 4);
12584   match(Set dst (ReplicateS src));
12585   ins_cost(DEFAULT_COST*3);
12586   effect(TEMP tmp);
12587   size(12);
12588 
12589   // FIXME: could use PKH instruction instead?
12590   format %{ "LSL      $tmp, $src, 16 \n\t"
12591             "OR       $tmp, $tmp, ($tmp >> 16) \n\t"
12592             "FMDRR    $dst,$tmp,$tmp\t" %}
12593   ins_encode %{
12594     __ mov($tmp$$Register, AsmOperand($src$$Register, lsl, 16));
12595     __ orr($tmp$$Register, $tmp$$Register, AsmOperand($tmp$$Register, lsr, 16));
12596     __ fmdrr($dst$$FloatRegister, $tmp$$Register, $tmp$$Register);
12597   %}
12598   ins_pipe(ialu_reg); // FIXME
12599 %}
12600 #endif /* !AARCH64 */
12601 
12602 // Replicate scalar to packed byte values in Double register
12603 instruct Repl4S_reg_simd(vecD dst, iRegI src) %{
12604   predicate(n->as_Vector()->length_in_bytes() == 8 && VM_Version::has_simd());
12605   match(Set dst (ReplicateS src));
12606   size(4);
12607 
12608   format %{ "VDUP.16 $dst,$src\t" %}
12609   ins_encode %{
12610     bool quad = false;
12611     __ vdupI($dst$$FloatRegister, $src$$Register,
12612              MacroAssembler::VELEM_SIZE_16, quad);
12613   %}
12614   ins_pipe(ialu_reg); // FIXME
12615 %}
12616 
12617 // Replicate scalar to packed byte values in Double register pair
12618 instruct Repl8S_reg(vecX dst, iRegI src) %{
12619   predicate(n->as_Vector()->length_in_bytes() == 16 && VM_Version::has_simd());
12620   match(Set dst (ReplicateS src));
12621   size(4);
12622 
12623   format %{ "VDUP.16 $dst.Q,$src\t" %}
12624   ins_encode %{
12625     bool quad = true;
12626     __ vdupI($dst$$FloatRegister, $src$$Register,
12627              MacroAssembler::VELEM_SIZE_16, quad);
12628   %}
12629   ins_pipe(ialu_reg); // FIXME
12630 %}
12631 
12632 
12633 #ifndef AARCH64
12634 // Replicate scalar constant to packed short/char values in Double register
12635 instruct Repl4S_immI(vecD dst, immI src, iRegP tmp) %{
12636   predicate(n->as_Vector()->length() == 4);
12637   match(Set dst (ReplicateS src));
12638   effect(TEMP tmp);
12639   size(12);
12640   ins_cost(DEFAULT_COST*4); // FIXME
12641 
12642   format %{ "MOV      $tmp, Repl2($src))\n\t"
12643             "FMDRR    $dst,$tmp,$tmp\t" %}
12644   ins_encode( LdReplImmI(src, dst, tmp, (2), (2)) );
12645   ins_pipe(loadConFD); // FIXME
12646 %}
12647 #endif /* !AARCH64 */
12648 
12649 // Replicate scalar constant to packed byte values in Double register
12650 instruct Repl4S_immU8(vecD dst, immU8 src) %{
12651   predicate(n->as_Vector()->length_in_bytes() == 8 && VM_Version::has_simd());
12652   match(Set dst (ReplicateS src));
12653   size(4);
12654 
12655   format %{ "VMOV.U16  $dst,$src" %}
12656   ins_encode %{
12657     bool quad = false;
12658     __ vmovI($dst$$FloatRegister, $src$$constant,
12659              MacroAssembler::VELEM_SIZE_16, quad);
12660   %}
12661   ins_pipe(loadConFD); // FIXME
12662 %}
12663 
12664 // Replicate scalar constant to packed byte values in Double register pair
12665 instruct Repl8S_immU8(vecX dst, immU8 src) %{
12666   predicate(n->as_Vector()->length_in_bytes() == 16 && VM_Version::has_simd());
12667   match(Set dst (ReplicateS src));
12668   size(4);
12669 
12670   format %{ "VMOV.U16  $dst.Q,$src" %}
12671   ins_encode %{
12672     bool quad = true;
12673     __ vmovI($dst$$FloatRegister, $src$$constant,
12674              MacroAssembler::VELEM_SIZE_16, quad);
12675   %}
12676   ins_pipe(loadConFD); // FIXME
12677 %}
12678 
12679 #ifndef AARCH64
12680 // Replicate scalar to packed int values in Double register
12681 instruct Repl2I_reg(vecD dst, iRegI src) %{
12682   predicate(n->as_Vector()->length() == 2);
12683   match(Set dst (ReplicateI src));
12684   size(4);
12685 
12686   format %{ "FMDRR    $dst,$src,$src\t" %}
12687   ins_encode %{
12688     __ fmdrr($dst$$FloatRegister, $src$$Register, $src$$Register);
12689   %}
12690   ins_pipe(ialu_reg); // FIXME
12691 %}
12692 
12693 // Replicate scalar to packed int values in Double register pair
12694 instruct Repl4I_reg(vecX dst, iRegI src) %{
12695   predicate(n->as_Vector()->length() == 4);
12696   match(Set dst (ReplicateI src));
12697   ins_cost(DEFAULT_COST*2);
12698   size(8);
12699 
12700   format %{ "FMDRR    $dst.lo,$src,$src\n\t"
12701             "FMDRR    $dst.hi,$src,$src" %}
12702 
12703   ins_encode %{
12704     __ fmdrr($dst$$FloatRegister, $src$$Register, $src$$Register);
12705     __ fmdrr($dst$$FloatRegister->successor()->successor(),
12706              $src$$Register, $src$$Register);
12707   %}
12708   ins_pipe(ialu_reg); // FIXME
12709 %}
12710 #endif /* !AARCH64 */
12711 
12712 // Replicate scalar to packed int values in Double register
12713 instruct Repl2I_reg_simd(vecD dst, iRegI src) %{
12714   predicate(n->as_Vector()->length_in_bytes() == 8 && VM_Version::has_simd());
12715   match(Set dst (ReplicateI src));
12716   size(4);
12717 
12718   format %{ "VDUP.32 $dst.D,$src\t" %}
12719   ins_encode %{
12720     bool quad = false;
12721     __ vdupI($dst$$FloatRegister, $src$$Register,
12722              MacroAssembler::VELEM_SIZE_32, quad);
12723   %}
12724   ins_pipe(ialu_reg); // FIXME
12725 %}
12726 
12727 // Replicate scalar to packed int values in Double register pair
12728 instruct Repl4I_reg_simd(vecX dst, iRegI src) %{
12729   predicate(n->as_Vector()->length_in_bytes() == 16 && VM_Version::has_simd());
12730   match(Set dst (ReplicateI src));
12731   size(4);
12732 
12733   format %{ "VDUP.32 $dst.Q,$src\t" %}
12734   ins_encode %{
12735     bool quad = true;
12736     __ vdupI($dst$$FloatRegister, $src$$Register,
12737              MacroAssembler::VELEM_SIZE_32, quad);
12738   %}
12739   ins_pipe(ialu_reg); // FIXME
12740 %}
12741 
12742 
12743 #ifndef AARCH64
12744 // Replicate scalar zero constant to packed int values in Double register
12745 instruct Repl2I_immI(vecD dst, immI src, iRegI tmp) %{
12746   predicate(n->as_Vector()->length() == 2);
12747   match(Set dst (ReplicateI src));
12748   effect(TEMP tmp);
12749   size(12);
12750   ins_cost(DEFAULT_COST*4); // FIXME
12751 
12752   format %{ "MOV      $tmp, Repl1($src))\n\t"
12753             "FMDRR    $dst,$tmp,$tmp\t" %}
12754   ins_encode( LdReplImmI(src, dst, tmp, (1), (4)) );
12755   ins_pipe(loadConFD); // FIXME
12756 %}
12757 #endif /* !AARCH64 */
12758 
12759 // Replicate scalar constant to packed byte values in Double register
12760 instruct Repl2I_immU8(vecD dst, immU8 src) %{
12761   predicate(n->as_Vector()->length_in_bytes() == 8 && VM_Version::has_simd());
12762   match(Set dst (ReplicateI src));
12763   size(4);
12764 
12765   format %{ "VMOV.I32  $dst.D,$src" %}
12766   ins_encode %{
12767     bool quad = false;
12768     __ vmovI($dst$$FloatRegister, $src$$constant,
12769              MacroAssembler::VELEM_SIZE_32, quad);
12770   %}
12771   ins_pipe(loadConFD); // FIXME
12772 %}
12773 
12774 // Replicate scalar constant to packed byte values in Double register pair
12775 instruct Repl4I_immU8(vecX dst, immU8 src) %{
12776   predicate(n->as_Vector()->length_in_bytes() == 16 && VM_Version::has_simd());
12777   match(Set dst (ReplicateI src));
12778   size(4);
12779 
12780   format %{ "VMOV.I32  $dst.Q,$src" %}
12781   ins_encode %{
12782     bool quad = true;
12783     __ vmovI($dst$$FloatRegister, $src$$constant,
12784              MacroAssembler::VELEM_SIZE_32, quad);
12785   %}
12786   ins_pipe(loadConFD); // FIXME
12787 %}
12788 
12789 #ifdef AARCH64
12790 // Replicate scalar to packed byte values in Double register pair
12791 instruct Repl2L_reg(vecX dst, iRegL src) %{
12792   predicate(n->as_Vector()->length() == 2);
12793   match(Set dst (ReplicateL src));
12794   size(4*1);
12795   ins_cost(DEFAULT_COST*1); // FIXME
12796 
12797   format %{ "VDUP.2D $dst.Q,$src\t" %}
12798   ins_encode %{
12799     bool quad = true;
12800     __ vdupI($dst$$FloatRegister, $src$$Register,
12801              MacroAssembler::VELEM_SIZE_64, quad);
12802   %}
12803   ins_pipe(ialu_reg); // FIXME
12804 %}
12805 #else /* !AARCH64 */
12806 // Replicate scalar to packed byte values in Double register pair
12807 instruct Repl2L_reg(vecX dst, iRegL src) %{
12808   predicate(n->as_Vector()->length() == 2);
12809   match(Set dst (ReplicateL src));
12810   size(8);
12811   ins_cost(DEFAULT_COST*2); // FIXME
12812 
12813   format %{ "FMDRR $dst.D,$src.lo,$src.hi\t\n"
12814             "FMDRR $dst.D.next,$src.lo,$src.hi" %}
12815   ins_encode %{
12816     __ fmdrr($dst$$FloatRegister, $src$$Register, $src$$Register->successor());
12817     __ fmdrr($dst$$FloatRegister->successor()->successor(),
12818              $src$$Register, $src$$Register->successor());
12819   %}
12820   ins_pipe(ialu_reg); // FIXME
12821 %}
12822 
12823 
12824 // Replicate scalar to packed float values in Double register
12825 instruct Repl2F_regI(vecD dst, iRegI src) %{
12826   predicate(n->as_Vector()->length() == 2);
12827   match(Set dst (ReplicateF src));
12828   size(4);
12829 
12830   format %{ "FMDRR    $dst.D,$src,$src\t" %}
12831   ins_encode %{
12832     __ fmdrr($dst$$FloatRegister, $src$$Register, $src$$Register);
12833   %}
12834   ins_pipe(ialu_reg); // FIXME
12835 %}
12836 
12837 // Replicate scalar to packed float values in Double register
12838 instruct Repl2F_reg_vfp(vecD dst, regF src) %{
12839   predicate(n->as_Vector()->length() == 2);
12840   match(Set dst (ReplicateF src));
12841   size(4*2);
12842   ins_cost(DEFAULT_COST*2); // FIXME
12843 
12844   expand %{
12845     iRegI tmp;
12846     MoveF2I_reg_reg(tmp, src);
12847     Repl2F_regI(dst,tmp);
12848   %}
12849 %}
12850 #endif /* !AARCH64 */
12851 
12852 // Replicate scalar to packed float values in Double register
12853 instruct Repl2F_reg_simd(vecD dst, regF src) %{
12854   predicate(n->as_Vector()->length_in_bytes() == 8 && VM_Version::has_simd());
12855   match(Set dst (ReplicateF src));
12856   size(4);
12857   ins_cost(DEFAULT_COST); // FIXME
12858 
12859   format %{ "VDUP.32  $dst.D,$src.D\t" %}
12860   ins_encode %{
12861     bool quad = false;
12862     __ vdupF($dst$$FloatRegister, $src$$FloatRegister, quad);
12863   %}
12864   ins_pipe(ialu_reg); // FIXME
12865 %}
12866 
12867 #ifndef AARCH64
12868 // Replicate scalar to packed float values in Double register pair
12869 instruct Repl4F_reg(vecX dst, regF src, iRegI tmp) %{
12870   predicate(n->as_Vector()->length() == 4);
12871   match(Set dst (ReplicateF src));
12872   effect(TEMP tmp);
12873   size(4*3);
12874   ins_cost(DEFAULT_COST*3); // FIXME
12875 
12876   format %{ "FMRS     $tmp,$src\n\t"
12877             "FMDRR    $dst.D,$tmp,$tmp\n\t"
12878             "FMDRR    $dst.D.next,$tmp,$tmp\t" %}
12879   ins_encode %{
12880     __ fmrs($tmp$$Register, $src$$FloatRegister);
12881     __ fmdrr($dst$$FloatRegister, $tmp$$Register, $tmp$$Register);
12882     __ fmdrr($dst$$FloatRegister->successor()->successor(),
12883              $tmp$$Register, $tmp$$Register);
12884   %}
12885   ins_pipe(ialu_reg); // FIXME
12886 %}
12887 #endif /* !AARCH64 */
12888 
12889 // Replicate scalar to packed float values in Double register pair
12890 instruct Repl4F_reg_simd(vecX dst, regF src) %{
12891   predicate(n->as_Vector()->length_in_bytes() == 16 && VM_Version::has_simd());
12892   match(Set dst (ReplicateF src));
12893   size(4);
12894   ins_cost(DEFAULT_COST); // FIXME
12895 
12896   format %{ "VDUP.32  $dst.Q,$src.D\t" %}
12897   ins_encode %{
12898     bool quad = true;
12899     __ vdupF($dst$$FloatRegister, $src$$FloatRegister, quad);
12900   %}
12901   ins_pipe(ialu_reg); // FIXME
12902 %}
12903 
12904 #ifndef AARCH64
12905 // Replicate scalar zero constant to packed float values in Double register
12906 instruct Repl2F_immI(vecD dst, immF src, iRegI tmp) %{
12907   predicate(n->as_Vector()->length() == 2);
12908   match(Set dst (ReplicateF src));
12909   effect(TEMP tmp);
12910   size(12);
12911   ins_cost(DEFAULT_COST*4); // FIXME
12912 
12913   format %{ "MOV      $tmp, Repl1($src))\n\t"
12914             "FMDRR    $dst,$tmp,$tmp\t" %}
12915   ins_encode( LdReplImmF(src, dst, tmp) );
12916   ins_pipe(loadConFD); // FIXME
12917 %}
12918 #endif /* !AAARCH64 */
12919 
12920 // Replicate scalar to packed double float values in Double register pair
12921 instruct Repl2D_reg(vecX dst, regD src) %{
12922 #ifdef AARCH64
12923   predicate(n->as_Vector()->length() == 2 && VM_Version::has_simd());
12924   match(Set dst (ReplicateD src));
12925   size(4*1);
12926   ins_cost(DEFAULT_COST*1); // FIXME
12927 
12928   format %{ "VDUP     $dst.2D,$src\t" %}
12929   ins_encode %{
12930     bool quad = true;
12931     __ vdupD($dst$$FloatRegister, $src$$FloatRegister, quad);
12932   %}
12933 #else
12934   predicate(n->as_Vector()->length() == 2);
12935   match(Set dst (ReplicateD src));
12936   size(4*2);
12937   ins_cost(DEFAULT_COST*2); // FIXME
12938 
12939   format %{ "FCPYD    $dst.D.a,$src\n\t"
12940             "FCPYD    $dst.D.b,$src\t" %}
12941   ins_encode %{
12942     FloatRegister dsta = $dst$$FloatRegister;
12943     FloatRegister src = $src$$FloatRegister;
12944     __ fcpyd(dsta, src);
12945     FloatRegister dstb = dsta->successor()->successor();
12946     __ fcpyd(dstb, src);
12947   %}
12948 #endif
12949   ins_pipe(ialu_reg); // FIXME
12950 %}
12951 
12952 // ====================VECTOR ARITHMETIC=======================================
12953 
12954 // --------------------------------- ADD --------------------------------------
12955 
12956 // Bytes vector add
12957 instruct vadd8B_reg(vecD dst, vecD src1, vecD src2) %{
12958   predicate(n->as_Vector()->length() == 8);
12959   match(Set dst (AddVB src1 src2));
12960   format %{ "VADD.I8 $dst,$src1,$src2\t! add packed8B" %}
12961   size(4);
12962   ins_encode %{
12963     bool quad = false;
12964     __ vaddI($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister,
12965              MacroAssembler::VELEM_SIZE_8, quad);
12966   %}
12967   ins_pipe( ialu_reg_reg ); // FIXME
12968 %}
12969 
12970 instruct vadd16B_reg(vecX dst, vecX src1, vecX src2) %{
12971   predicate(n->as_Vector()->length() == 16);
12972   match(Set dst (AddVB src1 src2));
12973   size(4);
12974   format %{ "VADD.I8 $dst.Q,$src1.Q,$src2.Q\t! add packed16B" %}
12975   ins_encode %{
12976     bool quad = true;
12977     __ vaddI($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister,
12978              MacroAssembler::VELEM_SIZE_8, quad);
12979   %}
12980   ins_pipe( ialu_reg_reg ); // FIXME
12981 %}
12982 
12983 // Shorts/Chars vector add
12984 instruct vadd4S_reg(vecD dst, vecD src1, vecD src2) %{
12985   predicate(n->as_Vector()->length() == 4);
12986   match(Set dst (AddVS src1 src2));
12987   size(4);
12988   format %{ "VADD.I16 $dst,$src1,$src2\t! add packed4S" %}
12989   ins_encode %{
12990     bool quad = false;
12991     __ vaddI($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister,
12992              MacroAssembler::VELEM_SIZE_16, quad);
12993   %}
12994   ins_pipe( ialu_reg_reg ); // FIXME
12995 %}
12996 
12997 instruct vadd8S_reg(vecX dst, vecX src1, vecX src2) %{
12998   predicate(n->as_Vector()->length() == 8);
12999   match(Set dst (AddVS src1 src2));
13000   size(4);
13001   format %{ "VADD.I16 $dst.Q,$src1.Q,$src2.Q\t! add packed8S" %}
13002   ins_encode %{
13003     bool quad = true;
13004     __ vaddI($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister,
13005              MacroAssembler::VELEM_SIZE_16, quad);
13006   %}
13007   ins_pipe( ialu_reg_reg ); // FIXME
13008 %}
13009 
13010 // Integers vector add
13011 instruct vadd2I_reg(vecD dst, vecD src1, vecD src2) %{
13012   predicate(n->as_Vector()->length() == 2);
13013   match(Set dst (AddVI src1 src2));
13014   size(4);
13015   format %{ "VADD.I32 $dst.D,$src1.D,$src2.D\t! add packed2I" %}
13016   ins_encode %{
13017     bool quad = false;
13018     __ vaddI($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister,
13019              MacroAssembler::VELEM_SIZE_32, quad);
13020   %}
13021   ins_pipe( ialu_reg_reg ); // FIXME
13022 %}
13023 
13024 instruct vadd4I_reg(vecX dst, vecX src1, vecX src2) %{
13025   predicate(n->as_Vector()->length() == 4);
13026   match(Set dst (AddVI src1 src2));
13027   size(4);
13028   format %{ "VADD.I32 $dst.Q,$src1.Q,$src2.Q\t! add packed4I" %}
13029   ins_encode %{
13030     bool quad = true;
13031     __ vaddI($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister,
13032              MacroAssembler::VELEM_SIZE_32, quad);
13033   %}
13034   ins_pipe( ialu_reg_reg ); // FIXME
13035 %}
13036 
13037 // Longs vector add
13038 instruct vadd2L_reg(vecX dst, vecX src1, vecX src2) %{
13039   predicate(n->as_Vector()->length() == 2);
13040   match(Set dst (AddVL src1 src2));
13041   size(4);
13042   format %{ "VADD.I64 $dst.Q,$src1.Q,$src2.Q\t! add packed2L" %}
13043   ins_encode %{
13044     bool quad = true;
13045     __ vaddI($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister,
13046              MacroAssembler::VELEM_SIZE_64, quad);
13047   %}
13048   ins_pipe( ialu_reg_reg ); // FIXME
13049 %}
13050 
13051 // Floats vector add
13052 instruct vadd2F_reg(vecD dst, vecD src1, vecD src2) %{
13053   predicate(n->as_Vector()->length() == 2 && VM_Version::simd_math_is_compliant());
13054   match(Set dst (AddVF src1 src2));
13055   size(4);
13056   format %{ "VADD.F32 $dst,$src1,$src2\t! add packed2F" %}
13057   ins_encode %{
13058     bool quad = false;
13059     __ vaddF($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister,
13060              MacroAssembler::VFA_SIZE_F32, quad);
13061   %}
13062   ins_pipe( faddD_reg_reg ); // FIXME
13063 %}
13064 
13065 #ifndef AARCH64
13066 instruct vadd2F_reg_vfp(vecD dst, vecD src1, vecD src2) %{
13067   predicate(n->as_Vector()->length() == 2 && !VM_Version::simd_math_is_compliant());
13068   match(Set dst (AddVF src1 src2));
13069   ins_cost(DEFAULT_COST*2); // FIXME
13070 
13071   size(4*2);
13072   format %{ "FADDS  $dst.a,$src1.a,$src2.a\n\t"
13073             "FADDS  $dst.b,$src1.b,$src2.b" %}
13074   ins_encode %{
13075     __ add_float($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister);
13076     __ add_float($dst$$FloatRegister->successor(),
13077              $src1$$FloatRegister->successor(),
13078              $src2$$FloatRegister->successor());
13079   %}
13080 
13081   ins_pipe(faddF_reg_reg); // FIXME
13082 %}
13083 #endif
13084 
13085 instruct vadd4F_reg_simd(vecX dst, vecX src1, vecX src2) %{
13086   predicate(n->as_Vector()->length() == 4 && VM_Version::simd_math_is_compliant());
13087   match(Set dst (AddVF src1 src2));
13088   size(4);
13089   format %{ "VADD.F32 $dst.Q,$src1.Q,$src2.Q\t! add packed4F" %}
13090   ins_encode %{
13091     bool quad = true;
13092     __ vaddF($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister,
13093              MacroAssembler::VFA_SIZE_F32, quad);
13094   %}
13095   ins_pipe( faddD_reg_reg ); // FIXME
13096 %}
13097 
13098 #ifdef AARCH64
13099 instruct vadd2D_reg_simd(vecX dst, vecX src1, vecX src2) %{
13100   predicate(n->as_Vector()->length() == 2 && VM_Version::simd_math_is_compliant());
13101   match(Set dst (AddVD src1 src2));
13102   size(4);
13103   format %{ "VADD.F64 $dst.Q,$src1.Q,$src2.Q\t! add packed2D" %}
13104   ins_encode %{
13105     bool quad = true;
13106     __ vaddF($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister,
13107              MacroAssembler::VFA_SIZE_F64, quad);
13108   %}
13109   ins_pipe( faddD_reg_reg ); // FIXME
13110 %}
13111 #else
13112 instruct vadd4F_reg_vfp(vecX dst, vecX src1, vecX src2) %{
13113   predicate(n->as_Vector()->length() == 4 && !VM_Version::simd_math_is_compliant());
13114   match(Set dst (AddVF src1 src2));
13115   size(4*4);
13116   ins_cost(DEFAULT_COST*4); // FIXME
13117 
13118   format %{ "FADDS  $dst.a,$src1.a,$src2.a\n\t"
13119             "FADDS  $dst.b,$src1.b,$src2.b\n\t"
13120             "FADDS  $dst.c,$src1.c,$src2.c\n\t"
13121             "FADDS  $dst.d,$src1.d,$src2.d" %}
13122 
13123   ins_encode %{
13124     FloatRegister dsta = $dst$$FloatRegister;
13125     FloatRegister src1a = $src1$$FloatRegister;
13126     FloatRegister src2a = $src2$$FloatRegister;
13127     __ add_float(dsta, src1a, src2a);
13128     FloatRegister dstb = dsta->successor();
13129     FloatRegister src1b = src1a->successor();
13130     FloatRegister src2b = src2a->successor();
13131     __ add_float(dstb, src1b, src2b);
13132     FloatRegister dstc = dstb->successor();
13133     FloatRegister src1c = src1b->successor();
13134     FloatRegister src2c = src2b->successor();
13135     __ add_float(dstc, src1c, src2c);
13136     FloatRegister dstd = dstc->successor();
13137     FloatRegister src1d = src1c->successor();
13138     FloatRegister src2d = src2c->successor();
13139     __ add_float(dstd, src1d, src2d);
13140   %}
13141 
13142   ins_pipe(faddF_reg_reg); // FIXME
13143 %}
13144 
13145 instruct vadd2D_reg_vfp(vecX dst, vecX src1, vecX src2) %{
13146   predicate(n->as_Vector()->length() == 2);
13147   match(Set dst (AddVD src1 src2));
13148   size(4*2);
13149   ins_cost(DEFAULT_COST*2); // FIXME
13150 
13151   format %{ "FADDD  $dst.a,$src1.a,$src2.a\n\t"
13152             "FADDD  $dst.b,$src1.b,$src2.b" %}
13153 
13154   ins_encode %{
13155     FloatRegister dsta = $dst$$FloatRegister;
13156     FloatRegister src1a = $src1$$FloatRegister;
13157     FloatRegister src2a = $src2$$FloatRegister;
13158     __ add_double(dsta, src1a, src2a);
13159     FloatRegister dstb = dsta->successor()->successor();
13160     FloatRegister src1b = src1a->successor()->successor();
13161     FloatRegister src2b = src2a->successor()->successor();
13162     __ add_double(dstb, src1b, src2b);
13163   %}
13164 
13165   ins_pipe(faddF_reg_reg); // FIXME
13166 %}
13167 #endif
13168 
13169 
13170 // Bytes vector sub
13171 instruct vsub8B_reg(vecD dst, vecD src1, vecD src2) %{
13172   predicate(n->as_Vector()->length() == 8);
13173   match(Set dst (SubVB src1 src2));
13174   size(4);
13175   format %{ "VSUB.I8 $dst,$src1,$src2\t! sub packed8B" %}
13176   ins_encode %{
13177     bool quad = false;
13178     __ vsubI($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister,
13179              MacroAssembler::VELEM_SIZE_8, quad);
13180   %}
13181   ins_pipe( ialu_reg_reg ); // FIXME
13182 %}
13183 
13184 instruct vsub16B_reg(vecX dst, vecX src1, vecX src2) %{
13185   predicate(n->as_Vector()->length() == 16);
13186   match(Set dst (SubVB src1 src2));
13187   size(4);
13188   format %{ "VSUB.I8 $dst.Q,$src1.Q,$src2.Q\t! sub packed16B" %}
13189   ins_encode %{
13190     bool quad = true;
13191     __ vsubI($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister,
13192              MacroAssembler::VELEM_SIZE_8, quad);
13193   %}
13194   ins_pipe( ialu_reg_reg ); // FIXME
13195 %}
13196 
13197 // Shorts/Chars vector sub
13198 instruct vsub4S_reg(vecD dst, vecD src1, vecD src2) %{
13199   predicate(n->as_Vector()->length() == 4);
13200   match(Set dst (SubVS src1 src2));
13201   size(4);
13202   format %{ "VSUB.I16 $dst,$src1,$src2\t! sub packed4S" %}
13203   ins_encode %{
13204     bool quad = false;
13205     __ vsubI($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister,
13206              MacroAssembler::VELEM_SIZE_16, quad);
13207   %}
13208   ins_pipe( ialu_reg_reg ); // FIXME
13209 %}
13210 
13211 instruct vsub16S_reg(vecX dst, vecX src1, vecX src2) %{
13212   predicate(n->as_Vector()->length() == 8);
13213   match(Set dst (SubVS src1 src2));
13214   size(4);
13215   format %{ "VSUB.I16 $dst.Q,$src1.Q,$src2.Q\t! sub packed8S" %}
13216   ins_encode %{
13217     bool quad = true;
13218     __ vsubI($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister,
13219              MacroAssembler::VELEM_SIZE_16, quad);
13220   %}
13221   ins_pipe( ialu_reg_reg ); // FIXME
13222 %}
13223 
13224 // Integers vector sub
13225 instruct vsub2I_reg(vecD dst, vecD src1, vecD src2) %{
13226   predicate(n->as_Vector()->length() == 2);
13227   match(Set dst (SubVI src1 src2));
13228   size(4);
13229   format %{ "VSUB.I32 $dst,$src1,$src2\t! sub packed2I" %}
13230   ins_encode %{
13231     bool quad = false;
13232     __ vsubI($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister,
13233              MacroAssembler::VELEM_SIZE_32, quad);
13234   %}
13235   ins_pipe( ialu_reg_reg ); // FIXME
13236 %}
13237 
13238 instruct vsub4I_reg(vecX dst, vecX src1, vecX src2) %{
13239   predicate(n->as_Vector()->length() == 4);
13240   match(Set dst (SubVI src1 src2));
13241   size(4);
13242   format %{ "VSUB.I32 $dst.Q,$src1.Q,$src2.Q\t! sub packed4I" %}
13243   ins_encode %{
13244     bool quad = true;
13245     __ vsubI($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister,
13246              MacroAssembler::VELEM_SIZE_32, quad);
13247   %}
13248   ins_pipe( ialu_reg_reg ); // FIXME
13249 %}
13250 
13251 // Longs vector sub
13252 instruct vsub2L_reg(vecX dst, vecX src1, vecX src2) %{
13253   predicate(n->as_Vector()->length() == 2);
13254   match(Set dst (SubVL src1 src2));
13255   size(4);
13256   format %{ "VSUB.I64 $dst.Q,$src1.Q,$src2.Q\t! sub packed2L" %}
13257   ins_encode %{
13258     bool quad = true;
13259     __ vsubI($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister,
13260              MacroAssembler::VELEM_SIZE_64, quad);
13261   %}
13262   ins_pipe( ialu_reg_reg ); // FIXME
13263 %}
13264 
13265 // Floats vector sub
13266 instruct vsub2F_reg(vecD dst, vecD src1, vecD src2) %{
13267   predicate(n->as_Vector()->length() == 2 && VM_Version::simd_math_is_compliant());
13268   match(Set dst (SubVF src1 src2));
13269   size(4);
13270   format %{ "VSUB.F32 $dst,$src1,$src2\t! sub packed2F" %}
13271   ins_encode %{
13272     bool quad = false;
13273     __ vsubF($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister,
13274              MacroAssembler::VFA_SIZE_F32, quad);
13275   %}
13276   ins_pipe( faddF_reg_reg ); // FIXME
13277 %}
13278 
13279 #ifndef AARCH64
13280 instruct vsub2F_reg_vfp(vecD dst, vecD src1, vecD src2) %{
13281   predicate(n->as_Vector()->length() == 2 && !VM_Version::simd_math_is_compliant());
13282   match(Set dst (SubVF src1 src2));
13283   size(4*2);
13284   ins_cost(DEFAULT_COST*2); // FIXME
13285 
13286   format %{ "FSUBS  $dst.a,$src1.a,$src2.a\n\t"
13287             "FSUBS  $dst.b,$src1.b,$src2.b" %}
13288 
13289   ins_encode %{
13290     FloatRegister dsta = $dst$$FloatRegister;
13291     FloatRegister src1a = $src1$$FloatRegister;
13292     FloatRegister src2a = $src2$$FloatRegister;
13293     __ sub_float(dsta, src1a, src2a);
13294     FloatRegister dstb = dsta->successor();
13295     FloatRegister src1b = src1a->successor();
13296     FloatRegister src2b = src2a->successor();
13297     __ sub_float(dstb, src1b, src2b);
13298   %}
13299 
13300   ins_pipe(faddF_reg_reg); // FIXME
13301 %}
13302 #endif
13303 
13304 
13305 instruct vsub4F_reg(vecX dst, vecX src1, vecX src2) %{
13306   predicate(n->as_Vector()->length() == 4 && VM_Version::simd_math_is_compliant());
13307   match(Set dst (SubVF src1 src2));
13308   size(4);
13309   format %{ "VSUB.F32 $dst.Q,$src1.Q,$src2.Q\t! sub packed4F" %}
13310   ins_encode %{
13311     bool quad = true;
13312     __ vsubF($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister,
13313              MacroAssembler::VFA_SIZE_F32, quad);
13314   %}
13315   ins_pipe( faddF_reg_reg ); // FIXME
13316 %}
13317 
13318 #ifdef AARCH64
13319 instruct vsub2D_reg_simd(vecX dst, vecX src1, vecX src2) %{
13320   predicate(n->as_Vector()->length() == 2 && VM_Version::simd_math_is_compliant());
13321   match(Set dst (SubVD src1 src2));
13322   size(4);
13323   format %{ "VSUB.F64 $dst.Q,$src1.Q,$src2.Q\t! add packed2D" %}
13324   ins_encode %{
13325     bool quad = true;
13326     __ vsubF($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister,
13327              MacroAssembler::VFA_SIZE_F64, quad);
13328   %}
13329   ins_pipe( faddD_reg_reg ); // FIXME
13330 %}
13331 #else
13332 instruct vsub4F_reg_vfp(vecX dst, vecX src1, vecX src2) %{
13333   predicate(n->as_Vector()->length() == 4 && !VM_Version::simd_math_is_compliant());
13334   match(Set dst (SubVF src1 src2));
13335   size(4*4);
13336   ins_cost(DEFAULT_COST*4); // FIXME
13337 
13338   format %{ "FSUBS  $dst.a,$src1.a,$src2.a\n\t"
13339             "FSUBS  $dst.b,$src1.b,$src2.b\n\t"
13340             "FSUBS  $dst.c,$src1.c,$src2.c\n\t"
13341             "FSUBS  $dst.d,$src1.d,$src2.d" %}
13342 
13343   ins_encode %{
13344     FloatRegister dsta = $dst$$FloatRegister;
13345     FloatRegister src1a = $src1$$FloatRegister;
13346     FloatRegister src2a = $src2$$FloatRegister;
13347     __ sub_float(dsta, src1a, src2a);
13348     FloatRegister dstb = dsta->successor();
13349     FloatRegister src1b = src1a->successor();
13350     FloatRegister src2b = src2a->successor();
13351     __ sub_float(dstb, src1b, src2b);
13352     FloatRegister dstc = dstb->successor();
13353     FloatRegister src1c = src1b->successor();
13354     FloatRegister src2c = src2b->successor();
13355     __ sub_float(dstc, src1c, src2c);
13356     FloatRegister dstd = dstc->successor();
13357     FloatRegister src1d = src1c->successor();
13358     FloatRegister src2d = src2c->successor();
13359     __ sub_float(dstd, src1d, src2d);
13360   %}
13361 
13362   ins_pipe(faddF_reg_reg); // FIXME
13363 %}
13364 
13365 instruct vsub2D_reg_vfp(vecX dst, vecX src1, vecX src2) %{
13366   predicate(n->as_Vector()->length() == 2);
13367   match(Set dst (SubVD src1 src2));
13368   size(4*2);
13369   ins_cost(DEFAULT_COST*2); // FIXME
13370 
13371   format %{ "FSUBD  $dst.a,$src1.a,$src2.a\n\t"
13372             "FSUBD  $dst.b,$src1.b,$src2.b" %}
13373 
13374   ins_encode %{
13375     FloatRegister dsta = $dst$$FloatRegister;
13376     FloatRegister src1a = $src1$$FloatRegister;
13377     FloatRegister src2a = $src2$$FloatRegister;
13378     __ sub_double(dsta, src1a, src2a);
13379     FloatRegister dstb = dsta->successor()->successor();
13380     FloatRegister src1b = src1a->successor()->successor();
13381     FloatRegister src2b = src2a->successor()->successor();
13382     __ sub_double(dstb, src1b, src2b);
13383   %}
13384 
13385   ins_pipe(faddF_reg_reg); // FIXME
13386 %}
13387 #endif
13388 
13389 // Shorts/Chars vector mul
13390 instruct vmul4S_reg(vecD dst, vecD src1, vecD src2) %{
13391   predicate(n->as_Vector()->length() == 4);
13392   match(Set dst (MulVS src1 src2));
13393   size(4);
13394   format %{ "VMUL.I16 $dst,$src1,$src2\t! mul packed4S" %}
13395   ins_encode %{
13396     __ vmulI($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister,
13397              MacroAssembler::VELEM_SIZE_16, 0);
13398   %}
13399   ins_pipe( ialu_reg_reg ); // FIXME
13400 %}
13401 
13402 instruct vmul8S_reg(vecX dst, vecX src1, vecX src2) %{
13403   predicate(n->as_Vector()->length() == 8);
13404   match(Set dst (MulVS src1 src2));
13405   size(4);
13406   format %{ "VMUL.I16 $dst.Q,$src1.Q,$src2.Q\t! mul packed8S" %}
13407   ins_encode %{
13408     __ vmulI($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister,
13409              MacroAssembler::VELEM_SIZE_16, 1);
13410   %}
13411   ins_pipe( ialu_reg_reg ); // FIXME
13412 %}
13413 
13414 // Integers vector mul
13415 instruct vmul2I_reg(vecD dst, vecD src1, vecD src2) %{
13416   predicate(n->as_Vector()->length() == 2);
13417   match(Set dst (MulVI src1 src2));
13418   size(4);
13419   format %{ "VMUL.I32 $dst,$src1,$src2\t! mul packed2I" %}
13420   ins_encode %{
13421     __ vmulI($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister,
13422              MacroAssembler::VELEM_SIZE_32, 0);
13423   %}
13424   ins_pipe( ialu_reg_reg ); // FIXME
13425 %}
13426 
13427 instruct vmul4I_reg(vecX dst, vecX src1, vecX src2) %{
13428   predicate(n->as_Vector()->length() == 4);
13429   match(Set dst (MulVI src1 src2));
13430   size(4);
13431   format %{ "VMUL.I32 $dst.Q,$src1.Q,$src2.Q\t! mul packed4I" %}
13432   ins_encode %{
13433     __ vmulI($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister,
13434              MacroAssembler::VELEM_SIZE_32, 1);
13435   %}
13436   ins_pipe( ialu_reg_reg ); // FIXME
13437 %}
13438 
13439 // Floats vector mul
13440 instruct vmul2F_reg(vecD dst, vecD src1, vecD src2) %{
13441   predicate(n->as_Vector()->length() == 2 && VM_Version::simd_math_is_compliant());
13442   match(Set dst (MulVF src1 src2));
13443   size(4);
13444   format %{ "VMUL.F32 $dst,$src1,$src2\t! mul packed2F" %}
13445   ins_encode %{
13446     __ vmulF($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister,
13447              MacroAssembler::VFA_SIZE_F32, 0);
13448   %}
13449   ins_pipe( fmulF_reg_reg ); // FIXME
13450 %}
13451 
13452 #ifndef AARCH64
13453 instruct vmul2F_reg_vfp(vecD dst, vecD src1, vecD src2) %{
13454   predicate(n->as_Vector()->length() == 2 && !VM_Version::simd_math_is_compliant());
13455   match(Set dst (MulVF src1 src2));
13456   size(4*2);
13457   ins_cost(DEFAULT_COST*2); // FIXME
13458 
13459   format %{ "FMULS  $dst.a,$src1.a,$src2.a\n\t"
13460             "FMULS  $dst.b,$src1.b,$src2.b" %}
13461   ins_encode %{
13462     __ mul_float($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister);
13463     __ mul_float($dst$$FloatRegister->successor(),
13464              $src1$$FloatRegister->successor(),
13465              $src2$$FloatRegister->successor());
13466   %}
13467 
13468   ins_pipe(fmulF_reg_reg); // FIXME
13469 %}
13470 #endif
13471 
13472 instruct vmul4F_reg(vecX dst, vecX src1, vecX src2) %{
13473   predicate(n->as_Vector()->length() == 4 && VM_Version::simd_math_is_compliant());
13474   match(Set dst (MulVF src1 src2));
13475   size(4);
13476   format %{ "VMUL.F32 $dst.Q,$src1.Q,$src2.Q\t! mul packed4F" %}
13477   ins_encode %{
13478     __ vmulF($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister,
13479              MacroAssembler::VFA_SIZE_F32, 1);
13480   %}
13481   ins_pipe( fmulF_reg_reg ); // FIXME
13482 %}
13483 
13484 #ifndef AARCH64
13485 instruct vmul4F_reg_vfp(vecX dst, vecX src1, vecX src2) %{
13486   predicate(n->as_Vector()->length() == 4 && !VM_Version::simd_math_is_compliant());
13487   match(Set dst (MulVF src1 src2));
13488   size(4*4);
13489   ins_cost(DEFAULT_COST*4); // FIXME
13490 
13491   format %{ "FMULS  $dst.a,$src1.a,$src2.a\n\t"
13492             "FMULS  $dst.b,$src1.b,$src2.b\n\t"
13493             "FMULS  $dst.c,$src1.c,$src2.c\n\t"
13494             "FMULS  $dst.d,$src1.d,$src2.d" %}
13495 
13496   ins_encode %{
13497     FloatRegister dsta = $dst$$FloatRegister;
13498     FloatRegister src1a = $src1$$FloatRegister;
13499     FloatRegister src2a = $src2$$FloatRegister;
13500     __ mul_float(dsta, src1a, src2a);
13501     FloatRegister dstb = dsta->successor();
13502     FloatRegister src1b = src1a->successor();
13503     FloatRegister src2b = src2a->successor();
13504     __ mul_float(dstb, src1b, src2b);
13505     FloatRegister dstc = dstb->successor();
13506     FloatRegister src1c = src1b->successor();
13507     FloatRegister src2c = src2b->successor();
13508     __ mul_float(dstc, src1c, src2c);
13509     FloatRegister dstd = dstc->successor();
13510     FloatRegister src1d = src1c->successor();
13511     FloatRegister src2d = src2c->successor();
13512     __ mul_float(dstd, src1d, src2d);
13513   %}
13514 
13515   ins_pipe(fmulF_reg_reg); // FIXME
13516 %}
13517 #endif
13518 
13519 #ifdef AARCH64
13520 instruct vmul2D_reg(vecX dst, vecX src1, vecX src2) %{
13521   predicate(n->as_Vector()->length() == 2 && VM_Version::has_simd());
13522   match(Set dst (MulVD src1 src2));
13523   size(4*1);
13524   ins_cost(DEFAULT_COST*1); // FIXME
13525 
13526   format %{ "FMUL.2D $dst,$src1,$src2\t! double[2]" %}
13527   ins_encode %{
13528     int quad = 1;
13529     __ vmulF($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister,
13530              MacroAssembler::VFA_SIZE_F64, quad);
13531   %}
13532 
13533   ins_pipe(fdivF_reg_reg); // FIXME
13534 %}
13535 #else
13536 instruct vmul2D_reg_vfp(vecX dst, vecX src1, vecX src2) %{
13537   predicate(n->as_Vector()->length() == 2);
13538   match(Set dst (MulVD src1 src2));
13539   size(4*2);
13540   ins_cost(DEFAULT_COST*2); // FIXME
13541 
13542   format %{ "FMULD  $dst.D.a,$src1.D.a,$src2.D.a\n\t"
13543             "FMULD  $dst.D.b,$src1.D.b,$src2.D.b" %}
13544   ins_encode %{
13545     FloatRegister dsta = $dst$$FloatRegister;
13546     FloatRegister src1a = $src1$$FloatRegister;
13547     FloatRegister src2a = $src2$$FloatRegister;
13548     __ mul_double(dsta, src1a, src2a);
13549     FloatRegister dstb = dsta->successor()->successor();
13550     FloatRegister src1b = src1a->successor()->successor();
13551     FloatRegister src2b = src2a->successor()->successor();
13552     __ mul_double(dstb, src1b, src2b);
13553   %}
13554 
13555   ins_pipe(fmulD_reg_reg); // FIXME
13556 %}
13557 #endif
13558 
13559 
13560 // Floats vector div
13561 instruct vdiv2F_reg_vfp(vecD dst, vecD src1, vecD src2) %{
13562   predicate(n->as_Vector()->length() == 2);
13563   match(Set dst (DivVF src1 src2));
13564 #ifdef AARCH64
13565   size(4*1);
13566   ins_cost(DEFAULT_COST*1); // FIXME
13567 
13568   format %{ "FDIV.2S $dst,$src1,$src2\t! float[2]" %}
13569   ins_encode %{
13570     int quad = 0;
13571     __ vdivF($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister,
13572              MacroAssembler::VFA_SIZE_F32, quad);
13573   %}
13574 
13575   ins_pipe(fdivF_reg_reg); // FIXME
13576 #else
13577   size(4*2);
13578   ins_cost(DEFAULT_COST*2); // FIXME
13579 
13580   format %{ "FDIVS  $dst.a,$src1.a,$src2.a\n\t"
13581             "FDIVS  $dst.b,$src1.b,$src2.b" %}
13582   ins_encode %{
13583     __ div_float($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister);
13584     __ div_float($dst$$FloatRegister->successor(),
13585              $src1$$FloatRegister->successor(),
13586              $src2$$FloatRegister->successor());
13587   %}
13588 
13589   ins_pipe(fdivF_reg_reg); // FIXME
13590 #endif
13591 %}
13592 
13593 instruct vdiv4F_reg_vfp(vecX dst, vecX src1, vecX src2) %{
13594   predicate(n->as_Vector()->length() == 4);
13595   match(Set dst (DivVF src1 src2));
13596 #ifdef AARCH64
13597   size(4*1);
13598   ins_cost(DEFAULT_COST*1); // FIXME
13599 
13600   format %{ "FDIV.4S $dst,$src1,$src2\t! float[4]" %}
13601   ins_encode %{
13602     int quad = 1;
13603     __ vdivF($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister,
13604              MacroAssembler::VFA_SIZE_F32, quad);
13605   %}
13606 
13607   ins_pipe(fdivF_reg_reg); // FIXME
13608 #else
13609   size(4*4);
13610   ins_cost(DEFAULT_COST*4); // FIXME
13611 
13612   format %{ "FDIVS  $dst.a,$src1.a,$src2.a\n\t"
13613             "FDIVS  $dst.b,$src1.b,$src2.b\n\t"
13614             "FDIVS  $dst.c,$src1.c,$src2.c\n\t"
13615             "FDIVS  $dst.d,$src1.d,$src2.d" %}
13616 
13617   ins_encode %{
13618     FloatRegister dsta = $dst$$FloatRegister;
13619     FloatRegister src1a = $src1$$FloatRegister;
13620     FloatRegister src2a = $src2$$FloatRegister;
13621     __ div_float(dsta, src1a, src2a);
13622     FloatRegister dstb = dsta->successor();
13623     FloatRegister src1b = src1a->successor();
13624     FloatRegister src2b = src2a->successor();
13625     __ div_float(dstb, src1b, src2b);
13626     FloatRegister dstc = dstb->successor();
13627     FloatRegister src1c = src1b->successor();
13628     FloatRegister src2c = src2b->successor();
13629     __ div_float(dstc, src1c, src2c);
13630     FloatRegister dstd = dstc->successor();
13631     FloatRegister src1d = src1c->successor();
13632     FloatRegister src2d = src2c->successor();
13633     __ div_float(dstd, src1d, src2d);
13634   %}
13635 
13636   ins_pipe(fdivF_reg_reg); // FIXME
13637 #endif
13638 %}
13639 
13640 #ifdef AARCH64
13641 instruct vdiv2D_reg(vecX dst, vecX src1, vecX src2) %{
13642   predicate(n->as_Vector()->length() == 2 && VM_Version::has_simd());
13643   match(Set dst (DivVD src1 src2));
13644   size(4*1);
13645   ins_cost(DEFAULT_COST*1); // FIXME
13646 
13647   format %{ "FDIV.2D $dst,$src1,$src2\t! double[2]" %}
13648   ins_encode %{
13649     int quad = 1;
13650     __ vdivF($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister,
13651              MacroAssembler::VFA_SIZE_F64, quad);
13652   %}
13653 
13654   ins_pipe(fdivF_reg_reg); // FIXME
13655 %}
13656 #else
13657 instruct vdiv2D_reg_vfp(vecX dst, vecX src1, vecX src2) %{
13658   predicate(n->as_Vector()->length() == 2);
13659   match(Set dst (DivVD src1 src2));
13660   size(4*2);
13661   ins_cost(DEFAULT_COST*2); // FIXME
13662 
13663   format %{ "FDIVD  $dst.D.a,$src1.D.a,$src2.D.a\n\t"
13664             "FDIVD  $dst.D.b,$src1.D.b,$src2.D.b" %}
13665   ins_encode %{
13666     FloatRegister dsta = $dst$$FloatRegister;
13667     FloatRegister src1a = $src1$$FloatRegister;
13668     FloatRegister src2a = $src2$$FloatRegister;
13669     __ div_double(dsta, src1a, src2a);
13670     FloatRegister dstb = dsta->successor()->successor();
13671     FloatRegister src1b = src1a->successor()->successor();
13672     FloatRegister src2b = src2a->successor()->successor();
13673     __ div_double(dstb, src1b, src2b);
13674   %}
13675 
13676   ins_pipe(fdivD_reg_reg); // FIXME
13677 %}
13678 #endif
13679 
13680 // --------------------------------- NEG --------------------------------------
13681 
13682 instruct vneg8B_reg(vecD dst, vecD src) %{
13683   predicate(n->as_Vector()->length_in_bytes() == 8);
13684   effect(DEF dst, USE src);
13685   size(4);
13686   ins_cost(DEFAULT_COST); // FIXME
13687   format %{ "VNEG.S8 $dst.D,$src.D\t! neg packed8B" %}
13688   ins_encode %{
13689     bool quad = false;
13690     __ vnegI($dst$$FloatRegister, $src$$FloatRegister,
13691               MacroAssembler::VELEM_SIZE_8, quad);
13692   %}
13693   ins_pipe( ialu_reg_reg ); // FIXME
13694 %}
13695 
13696 instruct vneg16B_reg(vecX dst, vecX src) %{
13697   predicate(n->as_Vector()->length_in_bytes() == 16);
13698   effect(DEF dst, USE src);
13699   size(4);
13700   ins_cost(DEFAULT_COST); // FIXME
13701   format %{ "VNEG.S8 $dst.Q,$src.Q\t! neg0 packed16B" %}
13702   ins_encode %{
13703     bool _float = false;
13704     bool quad = true;
13705     __ vnegI($dst$$FloatRegister, $src$$FloatRegister,
13706               MacroAssembler::VELEM_SIZE_8, quad);
13707   %}
13708   ins_pipe( ialu_reg_reg ); // FIXME
13709 %}
13710 
13711 // ------------------------------ Shift ---------------------------------------
13712 
13713 instruct vslcntD(vecD dst, iRegI cnt) %{
13714   predicate(n->as_Vector()->length_in_bytes() == 8 && VM_Version::has_simd());
13715   match(Set dst (LShiftCntV cnt));
13716   size(4);
13717   ins_cost(DEFAULT_COST); // FIXME
13718   expand %{
13719     Repl8B_reg_simd(dst, cnt);
13720   %}
13721 %}
13722 
13723 instruct vslcntX(vecX dst, iRegI cnt) %{
13724   predicate(n->as_Vector()->length_in_bytes() == 16 && VM_Version::has_simd());
13725   match(Set dst (LShiftCntV cnt));
13726   size(4);
13727   ins_cost(DEFAULT_COST); // FIXME
13728   expand %{
13729     Repl16B_reg(dst, cnt);
13730   %}
13731 %}
13732 
13733 // Low bits of vector "shift" elements are used, so it
13734 // doesn't matter if we treat it as ints or bytes here.
13735 instruct vsrcntD(vecD dst, iRegI cnt) %{
13736   predicate(n->as_Vector()->length_in_bytes() == 8 && VM_Version::has_simd());
13737   match(Set dst (RShiftCntV cnt));
13738   size(4*2);
13739   ins_cost(DEFAULT_COST*2); // FIXME
13740 
13741   format %{ "VDUP.8 $dst.D,$cnt\n\t"
13742             "VNEG.S8 $dst.D,$dst.D\t! neg packed8B" %}
13743   ins_encode %{
13744     bool quad = false;
13745     __ vdupI($dst$$FloatRegister, $cnt$$Register,
13746              MacroAssembler::VELEM_SIZE_8, quad);
13747     __ vnegI($dst$$FloatRegister, $dst$$FloatRegister,
13748               MacroAssembler::VELEM_SIZE_8, quad);
13749   %}
13750   ins_pipe( ialu_reg_reg ); // FIXME
13751 %}
13752 
13753 instruct vsrcntX(vecX dst, iRegI cnt) %{
13754   predicate(n->as_Vector()->length_in_bytes() == 16 && VM_Version::has_simd());
13755   match(Set dst (RShiftCntV cnt));
13756   size(4*2);
13757   ins_cost(DEFAULT_COST*2); // FIXME
13758   format %{ "VDUP.8 $dst.Q,$cnt\n\t"
13759             "VNEG.S8 $dst.Q,$dst.Q\t! neg packed16B" %}
13760   ins_encode %{
13761     bool quad = true;
13762     __ vdupI($dst$$FloatRegister, $cnt$$Register,
13763              MacroAssembler::VELEM_SIZE_8, quad);
13764     __ vnegI($dst$$FloatRegister, $dst$$FloatRegister,
13765               MacroAssembler::VELEM_SIZE_8, quad);
13766   %}
13767   ins_pipe( ialu_reg_reg ); // FIXME
13768 %}
13769 
13770 // Byte vector logical left/right shift based on sign
13771 instruct vsh8B_reg(vecD dst, vecD src, vecD shift) %{
13772   predicate(n->as_Vector()->length() == 8);
13773   effect(DEF dst, USE src, USE shift);
13774   size(4);
13775   ins_cost(DEFAULT_COST); // FIXME
13776   format %{
13777     "VSHL.U8 $dst.D,$src.D,$shift.D\t! logical left/right shift packed8B"
13778   %}
13779   ins_encode %{
13780     bool quad = false;
13781     __ vshlUI($dst$$FloatRegister, $shift$$FloatRegister, $src$$FloatRegister,
13782               MacroAssembler::VELEM_SIZE_8, quad);
13783   %}
13784   ins_pipe( ialu_reg_reg ); // FIXME
13785 %}
13786 
13787 instruct vsh16B_reg(vecX dst, vecX src, vecX shift) %{
13788   predicate(n->as_Vector()->length() == 16);
13789   effect(DEF dst, USE src, USE shift);
13790   size(4);
13791   ins_cost(DEFAULT_COST); // FIXME
13792   format %{
13793     "VSHL.U8 $dst.Q,$src.Q,$shift.Q\t! logical left/right shift packed16B"
13794   %}
13795   ins_encode %{
13796     bool quad = true;
13797     __ vshlUI($dst$$FloatRegister, $shift$$FloatRegister, $src$$FloatRegister,
13798               MacroAssembler::VELEM_SIZE_8, quad);
13799   %}
13800   ins_pipe( ialu_reg_reg ); // FIXME
13801 %}
13802 
13803 // Shorts/Char vector logical left/right shift based on sign
13804 instruct vsh4S_reg(vecD dst, vecD src, vecD shift) %{
13805   predicate(n->as_Vector()->length() == 4);
13806   effect(DEF dst, USE src, USE shift);
13807   size(4);
13808   ins_cost(DEFAULT_COST); // FIXME
13809   format %{
13810     "VSHL.U16 $dst.D,$src.D,$shift.D\t! logical left/right shift packed4S"
13811   %}
13812   ins_encode %{
13813     bool quad = false;
13814     __ vshlUI($dst$$FloatRegister, $shift$$FloatRegister, $src$$FloatRegister,
13815               MacroAssembler::VELEM_SIZE_16, quad);
13816   %}
13817   ins_pipe( ialu_reg_reg ); // FIXME
13818 %}
13819 
13820 instruct vsh8S_reg(vecX dst, vecX src, vecX shift) %{
13821   predicate(n->as_Vector()->length() == 8);
13822   effect(DEF dst, USE src, USE shift);
13823   size(4);
13824   ins_cost(DEFAULT_COST); // FIXME
13825   format %{
13826     "VSHL.U16 $dst.Q,$src.Q,$shift.Q\t! logical left/right shift packed8S"
13827   %}
13828   ins_encode %{
13829     bool quad = true;
13830     __ vshlUI($dst$$FloatRegister, $shift$$FloatRegister, $src$$FloatRegister,
13831               MacroAssembler::VELEM_SIZE_16, quad);
13832   %}
13833   ins_pipe( ialu_reg_reg ); // FIXME
13834 %}
13835 
13836 // Integers vector logical left/right shift based on sign
13837 instruct vsh2I_reg(vecD dst, vecD src, vecD shift) %{
13838   predicate(n->as_Vector()->length() == 2);
13839   effect(DEF dst, USE src, USE shift);
13840   size(4);
13841   ins_cost(DEFAULT_COST); // FIXME
13842   format %{
13843     "VSHL.U32 $dst.D,$src.D,$shift.D\t! logical left/right shift packed2I"
13844   %}
13845   ins_encode %{
13846     bool quad = false;
13847     __ vshlUI($dst$$FloatRegister, $shift$$FloatRegister, $src$$FloatRegister,
13848               MacroAssembler::VELEM_SIZE_32, quad);
13849   %}
13850   ins_pipe( ialu_reg_reg ); // FIXME
13851 %}
13852 
13853 instruct vsh4I_reg(vecX dst, vecX src, vecX shift) %{
13854   predicate(n->as_Vector()->length() == 4);
13855   effect(DEF dst, USE src, USE shift);
13856   size(4);
13857   ins_cost(DEFAULT_COST); // FIXME
13858   format %{
13859     "VSHL.U32 $dst.Q,$src.Q,$shift.Q\t! logical left/right shift packed4I"
13860   %}
13861   ins_encode %{
13862     bool quad = true;
13863     __ vshlUI($dst$$FloatRegister, $shift$$FloatRegister, $src$$FloatRegister,
13864               MacroAssembler::VELEM_SIZE_32, quad);
13865   %}
13866   ins_pipe( ialu_reg_reg ); // FIXME
13867 %}
13868 
13869 // Longs vector logical left/right shift based on sign
13870 instruct vsh2L_reg(vecX dst, vecX src, vecX shift) %{
13871   predicate(n->as_Vector()->length() == 2);
13872   effect(DEF dst, USE src, USE shift);
13873   size(4);
13874   ins_cost(DEFAULT_COST); // FIXME
13875   format %{
13876     "VSHL.U64 $dst.Q,$src.Q,$shift.Q\t! logical left/right shift packed2L"
13877   %}
13878   ins_encode %{
13879     bool quad = true;
13880     __ vshlUI($dst$$FloatRegister, $shift$$FloatRegister, $src$$FloatRegister,
13881               MacroAssembler::VELEM_SIZE_64, quad);
13882   %}
13883   ins_pipe( ialu_reg_reg ); // FIXME
13884 %}
13885 
13886 // ------------------------------ LeftShift -----------------------------------
13887 
13888 // Byte vector left shift
13889 instruct vsl8B_reg(vecD dst, vecD src, vecD shift) %{
13890   predicate(n->as_Vector()->length() == 8);
13891   match(Set dst (LShiftVB src shift));
13892   size(4*1);
13893   ins_cost(DEFAULT_COST*1); // FIXME
13894   expand %{
13895     vsh8B_reg(dst, src, shift);
13896   %}
13897 %}
13898 
13899 instruct vsl16B_reg(vecX dst, vecX src, vecX shift) %{
13900   predicate(n->as_Vector()->length() == 16);
13901   match(Set dst (LShiftVB src shift));
13902   size(4*1);
13903   ins_cost(DEFAULT_COST*1); // FIXME
13904   expand %{
13905     vsh16B_reg(dst, src, shift);
13906   %}
13907 %}
13908 
13909 instruct vsl8B_immI(vecD dst, vecD src, immI shift) %{
13910   predicate(n->as_Vector()->length() == 8);
13911   match(Set dst (LShiftVB src shift));
13912   size(4);
13913   ins_cost(DEFAULT_COST); // FIXME
13914   format %{
13915     "VSHL.I8 $dst.D,$src.D,$shift\t! logical left shift packed8B"
13916   %}
13917   ins_encode %{
13918     bool quad = false;
13919     __ vshli($dst$$FloatRegister, $src$$FloatRegister, 8, $shift$$constant,
13920              quad);
13921   %}
13922   ins_pipe( ialu_reg_reg ); // FIXME
13923 %}
13924 
13925 instruct vsl16B_immI(vecX dst, vecX src, immI shift) %{
13926   predicate(n->as_Vector()->length() == 16);
13927   match(Set dst (LShiftVB src shift));
13928   size(4);
13929   ins_cost(DEFAULT_COST); // FIXME
13930   format %{
13931     "VSHL.I8 $dst.Q,$src.Q,$shift\t! logical left shift packed16B"
13932   %}
13933   ins_encode %{
13934     bool quad = true;
13935     __ vshli($dst$$FloatRegister, $src$$FloatRegister, 8, $shift$$constant,
13936              quad);
13937   %}
13938   ins_pipe( ialu_reg_reg ); // FIXME
13939 %}
13940 
13941 // Shorts/Chars vector logical left/right shift
13942 instruct vsl4S_reg(vecD dst, vecD src, vecD shift) %{
13943   predicate(n->as_Vector()->length() == 4);
13944   match(Set dst (LShiftVS src shift));
13945   match(Set dst (URShiftVS src shift));
13946   size(4*1);
13947   ins_cost(DEFAULT_COST*1); // FIXME
13948   expand %{
13949     vsh4S_reg(dst, src, shift);
13950   %}
13951 %}
13952 
13953 instruct vsl8S_reg(vecX dst, vecX src, vecX shift) %{
13954   predicate(n->as_Vector()->length() == 8);
13955   match(Set dst (LShiftVS src shift));
13956   match(Set dst (URShiftVS src shift));
13957   size(4*1);
13958   ins_cost(DEFAULT_COST*1); // FIXME
13959   expand %{
13960     vsh8S_reg(dst, src, shift);
13961   %}
13962 %}
13963 
13964 instruct vsl4S_immI(vecD dst, vecD src, immI shift) %{
13965   predicate(n->as_Vector()->length() == 4);
13966   match(Set dst (LShiftVS src shift));
13967   size(4);
13968   ins_cost(DEFAULT_COST); // FIXME
13969   format %{
13970     "VSHL.I16 $dst.D,$src.D,$shift\t! logical left shift packed4S"
13971   %}
13972   ins_encode %{
13973     bool quad = false;
13974     __ vshli($dst$$FloatRegister, $src$$FloatRegister, 16, $shift$$constant,
13975              quad);
13976   %}
13977   ins_pipe( ialu_reg_reg ); // FIXME
13978 %}
13979 
13980 instruct vsl8S_immI(vecX dst, vecX src, immI shift) %{
13981   predicate(n->as_Vector()->length() == 8);
13982   match(Set dst (LShiftVS src shift));
13983   size(4);
13984   ins_cost(DEFAULT_COST); // FIXME
13985   format %{
13986     "VSHL.I16 $dst.Q,$src.Q,$shift\t! logical left shift packed8S"
13987   %}
13988   ins_encode %{
13989     bool quad = true;
13990     __ vshli($dst$$FloatRegister, $src$$FloatRegister, 16, $shift$$constant,
13991              quad);
13992   %}
13993   ins_pipe( ialu_reg_reg ); // FIXME
13994 %}
13995 
13996 // Integers vector logical left/right shift
13997 instruct vsl2I_reg(vecD dst, vecD src, vecD shift) %{
13998   predicate(n->as_Vector()->length() == 2 && VM_Version::has_simd());
13999   match(Set dst (LShiftVI src shift));
14000   match(Set dst (URShiftVI src shift));
14001   size(4*1);
14002   ins_cost(DEFAULT_COST*1); // FIXME
14003   expand %{
14004     vsh2I_reg(dst, src, shift);
14005   %}
14006 %}
14007 
14008 instruct vsl4I_reg(vecX dst, vecX src, vecX shift) %{
14009   predicate(n->as_Vector()->length() == 4 && VM_Version::has_simd());
14010   match(Set dst (LShiftVI src shift));
14011   match(Set dst (URShiftVI src shift));
14012   size(4*1);
14013   ins_cost(DEFAULT_COST*1); // FIXME
14014   expand %{
14015     vsh4I_reg(dst, src, shift);
14016   %}
14017 %}
14018 
14019 instruct vsl2I_immI(vecD dst, vecD src, immI shift) %{
14020   predicate(n->as_Vector()->length() == 2 && VM_Version::has_simd());
14021   match(Set dst (LShiftVI src shift));
14022   size(4);
14023   ins_cost(DEFAULT_COST); // FIXME
14024   format %{
14025     "VSHL.I32 $dst.D,$src.D,$shift\t! logical left shift packed2I"
14026   %}
14027   ins_encode %{
14028     bool quad = false;
14029     __ vshli($dst$$FloatRegister, $src$$FloatRegister, 32, $shift$$constant,
14030              quad);
14031   %}
14032   ins_pipe( ialu_reg_reg ); // FIXME
14033 %}
14034 
14035 instruct vsl4I_immI(vecX dst, vecX src, immI shift) %{
14036   predicate(n->as_Vector()->length() == 4 && VM_Version::has_simd());
14037   match(Set dst (LShiftVI src shift));
14038   size(4);
14039   ins_cost(DEFAULT_COST); // FIXME
14040   format %{
14041     "VSHL.I32 $dst.Q,$src.Q,$shift\t! logical left shift packed4I"
14042   %}
14043   ins_encode %{
14044     bool quad = true;
14045     __ vshli($dst$$FloatRegister, $src$$FloatRegister, 32, $shift$$constant,
14046              quad);
14047   %}
14048   ins_pipe( ialu_reg_reg ); // FIXME
14049 %}
14050 
14051 // Longs vector logical left/right shift
14052 instruct vsl2L_reg(vecX dst, vecX src, vecX shift) %{
14053   predicate(n->as_Vector()->length() == 2);
14054   match(Set dst (LShiftVL src shift));
14055   match(Set dst (URShiftVL src shift));
14056   size(4*1);
14057   ins_cost(DEFAULT_COST*1); // FIXME
14058   expand %{
14059     vsh2L_reg(dst, src, shift);
14060   %}
14061 %}
14062 
14063 instruct vsl2L_immI(vecX dst, vecX src, immI shift) %{
14064   predicate(n->as_Vector()->length() == 2);
14065   match(Set dst (LShiftVL src shift));
14066   size(4);
14067   ins_cost(DEFAULT_COST); // FIXME
14068   format %{
14069     "VSHL.I64 $dst.Q,$src.Q,$shift\t! logical left shift packed2L"
14070   %}
14071   ins_encode %{
14072     bool quad = true;
14073     __ vshli($dst$$FloatRegister, $src$$FloatRegister, 64, $shift$$constant,
14074              quad);
14075   %}
14076   ins_pipe( ialu_reg_reg ); // FIXME
14077 %}
14078 
14079 // ----------------------- LogicalRightShift -----------------------------------
14080 
14081 // Bytes/Shorts vector logical right shift produces incorrect Java result
14082 // for negative data because java code convert short value into int with
14083 // sign extension before a shift.
14084 
14085 // Chars vector logical right shift
14086 instruct vsrl4S_immI(vecD dst, vecD src, immI shift) %{
14087   predicate(n->as_Vector()->length() == 4);
14088   match(Set dst (URShiftVS src shift));
14089   size(4);
14090   ins_cost(DEFAULT_COST); // FIXME
14091   format %{
14092     "VSHR.U16 $dst.D,$src.D,$shift\t! logical right shift packed4S"
14093   %}
14094   ins_encode %{
14095     bool quad = false;
14096     __ vshrUI($dst$$FloatRegister, $src$$FloatRegister, 16, $shift$$constant,
14097              quad);
14098   %}
14099   ins_pipe( ialu_reg_reg ); // FIXME
14100 %}
14101 
14102 instruct vsrl8S_immI(vecX dst, vecX src, immI shift) %{
14103   predicate(n->as_Vector()->length() == 8);
14104   match(Set dst (URShiftVS src shift));
14105   size(4);
14106   ins_cost(DEFAULT_COST); // FIXME
14107   format %{
14108     "VSHR.U16 $dst.Q,$src.Q,$shift\t! logical right shift packed8S"
14109   %}
14110   ins_encode %{
14111     bool quad = true;
14112     __ vshrUI($dst$$FloatRegister, $src$$FloatRegister, 16, $shift$$constant,
14113              quad);
14114   %}
14115   ins_pipe( ialu_reg_reg ); // FIXME
14116 %}
14117 
14118 // Integers vector logical right shift
14119 instruct vsrl2I_immI(vecD dst, vecD src, immI shift) %{
14120   predicate(n->as_Vector()->length() == 2 && VM_Version::has_simd());
14121   match(Set dst (URShiftVI src shift));
14122   size(4);
14123   ins_cost(DEFAULT_COST); // FIXME
14124   format %{
14125     "VSHR.U32 $dst.D,$src.D,$shift\t! logical right shift packed2I"
14126   %}
14127   ins_encode %{
14128     bool quad = false;
14129     __ vshrUI($dst$$FloatRegister, $src$$FloatRegister, 32, $shift$$constant,
14130              quad);
14131   %}
14132   ins_pipe( ialu_reg_reg ); // FIXME
14133 %}
14134 
14135 instruct vsrl4I_immI(vecX dst, vecX src, immI shift) %{
14136   predicate(n->as_Vector()->length() == 4 && VM_Version::has_simd());
14137   match(Set dst (URShiftVI src shift));
14138   size(4);
14139   ins_cost(DEFAULT_COST); // FIXME
14140   format %{
14141     "VSHR.U32 $dst.Q,$src.Q,$shift\t! logical right shift packed4I"
14142   %}
14143   ins_encode %{
14144     bool quad = true;
14145     __ vshrUI($dst$$FloatRegister, $src$$FloatRegister, 32, $shift$$constant,
14146              quad);
14147   %}
14148   ins_pipe( ialu_reg_reg ); // FIXME
14149 %}
14150 
14151 // Longs vector logical right shift
14152 instruct vsrl2L_immI(vecX dst, vecX src, immI shift) %{
14153   predicate(n->as_Vector()->length() == 2);
14154   match(Set dst (URShiftVL src shift));
14155   size(4);
14156   ins_cost(DEFAULT_COST); // FIXME
14157   format %{
14158     "VSHR.U64 $dst.Q,$src.Q,$shift\t! logical right shift packed2L"
14159   %}
14160   ins_encode %{
14161     bool quad = true;
14162     __ vshrUI($dst$$FloatRegister, $src$$FloatRegister, 64, $shift$$constant,
14163              quad);
14164   %}
14165   ins_pipe( ialu_reg_reg ); // FIXME
14166 %}
14167 
14168 // ------------------- ArithmeticRightShift -----------------------------------
14169 
14170 // Bytes vector arithmetic left/right shift based on sign
14171 instruct vsha8B_reg(vecD dst, vecD src, vecD shift) %{
14172   predicate(n->as_Vector()->length() == 8);
14173   effect(DEF dst, USE src, USE shift);
14174   size(4);
14175   ins_cost(DEFAULT_COST); // FIXME
14176   format %{
14177     "VSHL.S8 $dst.D,$src.D,$shift.D\t! arithmetic right shift packed8B"
14178   %}
14179   ins_encode %{
14180     bool quad = false;
14181     __ vshlSI($dst$$FloatRegister, $shift$$FloatRegister, $src$$FloatRegister,
14182               MacroAssembler::VELEM_SIZE_8, quad);
14183   %}
14184   ins_pipe( ialu_reg_reg ); // FIXME
14185 %}
14186 
14187 instruct vsha16B_reg(vecX dst, vecX src, vecX shift) %{
14188   predicate(n->as_Vector()->length() == 16);
14189   effect(DEF dst, USE src, USE shift);
14190   size(4);
14191   ins_cost(DEFAULT_COST); // FIXME
14192   format %{
14193     "VSHL.S8 $dst.Q,$src.Q,$shift.Q\t! arithmetic right shift packed16B"
14194   %}
14195   ins_encode %{
14196     bool quad = true;
14197     __ vshlSI($dst$$FloatRegister, $shift$$FloatRegister, $src$$FloatRegister,
14198               MacroAssembler::VELEM_SIZE_8, quad);
14199   %}
14200   ins_pipe( ialu_reg_reg ); // FIXME
14201 %}
14202 
14203 // Shorts vector arithmetic left/right shift based on sign
14204 instruct vsha4S_reg(vecD dst, vecD src, vecD shift) %{
14205   predicate(n->as_Vector()->length() == 4);
14206   effect(DEF dst, USE src, USE shift);
14207   size(4);
14208   ins_cost(DEFAULT_COST); // FIXME
14209   format %{
14210     "VSHL.S16 $dst.D,$src.D,$shift.D\t! arithmetic right shift packed4S"
14211   %}
14212   ins_encode %{
14213     bool quad = false;
14214     __ vshlSI($dst$$FloatRegister, $shift$$FloatRegister, $src$$FloatRegister,
14215               MacroAssembler::VELEM_SIZE_16, quad);
14216   %}
14217   ins_pipe( ialu_reg_reg ); // FIXME
14218 %}
14219 
14220 instruct vsha8S_reg(vecX dst, vecX src, vecX shift) %{
14221   predicate(n->as_Vector()->length() == 8);
14222   effect(DEF dst, USE src, USE shift);
14223   size(4);
14224   ins_cost(DEFAULT_COST); // FIXME
14225   format %{
14226     "VSHL.S16 $dst.Q,$src.Q,$shift.Q\t! arithmetic right shift packed8S"
14227   %}
14228   ins_encode %{
14229     bool quad = true;
14230     __ vshlSI($dst$$FloatRegister, $shift$$FloatRegister, $src$$FloatRegister,
14231               MacroAssembler::VELEM_SIZE_16, quad);
14232   %}
14233   ins_pipe( ialu_reg_reg ); // FIXME
14234 %}
14235 
14236 // Integers vector arithmetic left/right shift based on sign
14237 instruct vsha2I_reg(vecD dst, vecD src, vecD shift) %{
14238   predicate(n->as_Vector()->length() == 2);
14239   effect(DEF dst, USE src, USE shift);
14240   size(4);
14241   ins_cost(DEFAULT_COST); // FIXME
14242   format %{
14243     "VSHL.S32 $dst.D,$src.D,$shift.D\t! arithmetic right shift packed2I"
14244   %}
14245   ins_encode %{
14246     bool quad = false;
14247     __ vshlSI($dst$$FloatRegister, $shift$$FloatRegister, $src$$FloatRegister,
14248               MacroAssembler::VELEM_SIZE_32, quad);
14249   %}
14250   ins_pipe( ialu_reg_reg ); // FIXME
14251 %}
14252 
14253 instruct vsha4I_reg(vecX dst, vecX src, vecX shift) %{
14254   predicate(n->as_Vector()->length() == 4);
14255   effect(DEF dst, USE src, USE shift);
14256   size(4);
14257   ins_cost(DEFAULT_COST); // FIXME
14258   format %{
14259     "VSHL.S32 $dst.Q,$src.Q,$shift.Q\t! arithmetic right shift packed4I"
14260   %}
14261   ins_encode %{
14262     bool quad = true;
14263     __ vshlSI($dst$$FloatRegister, $shift$$FloatRegister, $src$$FloatRegister,
14264               MacroAssembler::VELEM_SIZE_32, quad);
14265   %}
14266   ins_pipe( ialu_reg_reg ); // FIXME
14267 %}
14268 
14269 // Longs vector arithmetic left/right shift based on sign
14270 instruct vsha2L_reg(vecX dst, vecX src, vecX shift) %{
14271   predicate(n->as_Vector()->length() == 2);
14272   effect(DEF dst, USE src, USE shift);
14273   size(4);
14274   ins_cost(DEFAULT_COST); // FIXME
14275   format %{
14276     "VSHL.S64 $dst.Q,$src.Q,$shift.Q\t! arithmetic right shift packed2L"
14277   %}
14278   ins_encode %{
14279     bool quad = true;
14280     __ vshlSI($dst$$FloatRegister, $shift$$FloatRegister, $src$$FloatRegister,
14281               MacroAssembler::VELEM_SIZE_64, quad);
14282   %}
14283   ins_pipe( ialu_reg_reg ); // FIXME
14284 %}
14285 
14286 // Byte vector arithmetic right shift
14287 
14288 instruct vsra8B_reg(vecD dst, vecD src, vecD shift) %{
14289   predicate(n->as_Vector()->length() == 8);
14290   match(Set dst (RShiftVB src shift));
14291   size(4);
14292   ins_cost(DEFAULT_COST); // FIXME
14293   expand %{
14294     vsha8B_reg(dst, src, shift);
14295   %}
14296 %}
14297 
14298 instruct vsrl16B_reg(vecX dst, vecX src, vecX shift) %{
14299   predicate(n->as_Vector()->length() == 16);
14300   match(Set dst (RShiftVB src shift));
14301   size(4);
14302   ins_cost(DEFAULT_COST); // FIXME
14303   expand %{
14304     vsha16B_reg(dst, src, shift);
14305   %}
14306 %}
14307 
14308 instruct vsrl8B_immI(vecD dst, vecD src, immI shift) %{
14309   predicate(n->as_Vector()->length() == 8);
14310   match(Set dst (RShiftVB src shift));
14311   size(4);
14312   ins_cost(DEFAULT_COST); // FIXME
14313   format %{
14314     "VSHR.S8 $dst.D,$src.D,$shift\t! logical right shift packed8B"
14315   %}
14316   ins_encode %{
14317     bool quad = false;
14318     __ vshrSI($dst$$FloatRegister, $src$$FloatRegister, 8, $shift$$constant,
14319              quad);
14320   %}
14321   ins_pipe( ialu_reg_reg ); // FIXME
14322 %}
14323 
14324 instruct vsrl16B_immI(vecX dst, vecX src, immI shift) %{
14325   predicate(n->as_Vector()->length() == 16);
14326   match(Set dst (RShiftVB src shift));
14327   size(4);
14328   ins_cost(DEFAULT_COST); // FIXME
14329   format %{
14330     "VSHR.S8 $dst.Q,$src.Q,$shift\t! logical right shift packed16B"
14331   %}
14332   ins_encode %{
14333     bool quad = true;
14334     __ vshrSI($dst$$FloatRegister, $src$$FloatRegister, 8, $shift$$constant,
14335              quad);
14336   %}
14337   ins_pipe( ialu_reg_reg ); // FIXME
14338 %}
14339 
14340 // Shorts vector arithmetic right shift
14341 instruct vsra4S_reg(vecD dst, vecD src, vecD shift) %{
14342   predicate(n->as_Vector()->length() == 4);
14343   match(Set dst (RShiftVS src shift));
14344   size(4);
14345   ins_cost(DEFAULT_COST); // FIXME
14346   expand %{
14347     vsha4S_reg(dst, src, shift);
14348   %}
14349 %}
14350 
14351 instruct vsra8S_reg(vecX dst, vecX src, vecX shift) %{
14352   predicate(n->as_Vector()->length() == 8);
14353   match(Set dst (RShiftVS src shift));
14354   size(4);
14355   ins_cost(DEFAULT_COST); // FIXME
14356   expand %{
14357     vsha8S_reg(dst, src, shift);
14358   %}
14359 %}
14360 
14361 instruct vsra4S_immI(vecD dst, vecD src, immI shift) %{
14362   predicate(n->as_Vector()->length() == 4);
14363   match(Set dst (RShiftVS src shift));
14364   size(4);
14365   ins_cost(DEFAULT_COST); // FIXME
14366   format %{
14367     "VSHR.S16 $dst.D,$src.D,$shift\t! logical right shift packed4S"
14368   %}
14369   ins_encode %{
14370     bool quad = false;
14371     __ vshrSI($dst$$FloatRegister, $src$$FloatRegister, 16, $shift$$constant,
14372              quad);
14373   %}
14374   ins_pipe( ialu_reg_reg ); // FIXME
14375 %}
14376 
14377 instruct vsra8S_immI(vecX dst, vecX src, immI shift) %{
14378   predicate(n->as_Vector()->length() == 8);
14379   match(Set dst (RShiftVS src shift));
14380   size(4);
14381   ins_cost(DEFAULT_COST); // FIXME
14382   format %{
14383     "VSHR.S16 $dst.Q,$src.Q,$shift\t! logical right shift packed8S"
14384   %}
14385   ins_encode %{
14386     bool quad = true;
14387     __ vshrSI($dst$$FloatRegister, $src$$FloatRegister, 16, $shift$$constant,
14388              quad);
14389   %}
14390   ins_pipe( ialu_reg_reg ); // FIXME
14391 %}
14392 
14393 // Integers vector arithmetic right shift
14394 instruct vsra2I_reg(vecD dst, vecD src, vecD shift) %{
14395   predicate(n->as_Vector()->length() == 2);
14396   match(Set dst (RShiftVI src shift));
14397   size(4);
14398   ins_cost(DEFAULT_COST); // FIXME
14399   expand %{
14400     vsha2I_reg(dst, src, shift);
14401   %}
14402 %}
14403 
14404 instruct vsra4I_reg(vecX dst, vecX src, vecX shift) %{
14405   predicate(n->as_Vector()->length() == 4);
14406   match(Set dst (RShiftVI src shift));
14407   size(4);
14408   ins_cost(DEFAULT_COST); // FIXME
14409   expand %{
14410     vsha4I_reg(dst, src, shift);
14411   %}
14412 %}
14413 
14414 instruct vsra2I_immI(vecD dst, vecD src, immI shift) %{
14415   predicate(n->as_Vector()->length() == 2);
14416   match(Set dst (RShiftVI src shift));
14417   size(4);
14418   ins_cost(DEFAULT_COST); // FIXME
14419   format %{
14420     "VSHR.S32 $dst.D,$src.D,$shift\t! logical right shift packed2I"
14421   %}
14422   ins_encode %{
14423     bool quad = false;
14424     __ vshrSI($dst$$FloatRegister, $src$$FloatRegister, 32, $shift$$constant,
14425              quad);
14426   %}
14427   ins_pipe( ialu_reg_reg ); // FIXME
14428 %}
14429 
14430 instruct vsra4I_immI(vecX dst, vecX src, immI shift) %{
14431   predicate(n->as_Vector()->length() == 4);
14432   match(Set dst (RShiftVI src shift));
14433   size(4);
14434   ins_cost(DEFAULT_COST); // FIXME
14435   format %{
14436     "VSHR.S32 $dst.Q,$src.Q,$shift\t! logical right shift packed4I"
14437   %}
14438   ins_encode %{
14439     bool quad = true;
14440     __ vshrSI($dst$$FloatRegister, $src$$FloatRegister, 32, $shift$$constant,
14441              quad);
14442   %}
14443   ins_pipe( ialu_reg_reg ); // FIXME
14444 %}
14445 
14446 // Longs vector arithmetic right shift
14447 instruct vsra2L_reg(vecX dst, vecX src, vecX shift) %{
14448   predicate(n->as_Vector()->length() == 2);
14449   match(Set dst (RShiftVL src shift));
14450   size(4);
14451   ins_cost(DEFAULT_COST); // FIXME
14452   expand %{
14453     vsha2L_reg(dst, src, shift);
14454   %}
14455 %}
14456 
14457 instruct vsra2L_immI(vecX dst, vecX src, immI shift) %{
14458   predicate(n->as_Vector()->length() == 2);
14459   match(Set dst (RShiftVL src shift));
14460   size(4);
14461   ins_cost(DEFAULT_COST); // FIXME
14462   format %{
14463     "VSHR.S64 $dst.Q,$src.Q,$shift\t! logical right shift packed2L"
14464   %}
14465   ins_encode %{
14466     bool quad = true;
14467     __ vshrSI($dst$$FloatRegister, $src$$FloatRegister, 64, $shift$$constant,
14468              quad);
14469   %}
14470   ins_pipe( ialu_reg_reg ); // FIXME
14471 %}
14472 
14473 // --------------------------------- AND --------------------------------------
14474 
14475 instruct vandD(vecD dst, vecD src1, vecD src2) %{
14476   predicate(n->as_Vector()->length_in_bytes() == 8);
14477   match(Set dst (AndV src1 src2));
14478   format %{ "VAND    $dst.D,$src1.D,$src2.D\t! and vectors (8 bytes)" %}
14479   ins_encode %{
14480     bool quad = false;
14481     __ vandI($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister,
14482              quad);
14483   %}
14484   ins_pipe( ialu_reg_reg ); // FIXME
14485 %}
14486 
14487 instruct vandX(vecX dst, vecX src1, vecX src2) %{
14488   predicate(n->as_Vector()->length_in_bytes() == 16);
14489   match(Set dst (AndV src1 src2));
14490   format %{ "VAND    $dst.Q,$src1.Q,$src2.Q\t! and vectors (16 bytes)" %}
14491   ins_encode %{
14492     bool quad = true;
14493     __ vandI($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister,
14494              quad);
14495   %}
14496   ins_pipe( ialu_reg_reg ); // FIXME
14497 %}
14498 
14499 // --------------------------------- OR ---------------------------------------
14500 
14501 instruct vorD(vecD dst, vecD src1, vecD src2) %{
14502   predicate(n->as_Vector()->length_in_bytes() == 8);
14503   match(Set dst (OrV src1 src2));
14504   format %{ "VOR     $dst.D,$src1.D,$src2.D\t! and vectors (8 bytes)" %}
14505   ins_encode %{
14506     bool quad = false;
14507     __ vorI($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister,
14508             quad);
14509   %}
14510   ins_pipe( ialu_reg_reg ); // FIXME
14511 %}
14512 
14513 instruct vorX(vecX dst, vecX src1, vecX src2) %{
14514   predicate(n->as_Vector()->length_in_bytes() == 16);
14515   match(Set dst (OrV src1 src2));
14516   format %{ "VOR     $dst.Q,$src1.Q,$src2.Q\t! and vectors (16 bytes)" %}
14517   ins_encode %{
14518     bool quad = true;
14519     __ vorI($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister,
14520             quad);
14521   %}
14522   ins_pipe( ialu_reg_reg ); // FIXME
14523 %}
14524 
14525 // --------------------------------- XOR --------------------------------------
14526 
14527 instruct vxorD(vecD dst, vecD src1, vecD src2) %{
14528   predicate(n->as_Vector()->length_in_bytes() == 8);
14529   match(Set dst (XorV src1 src2));
14530   format %{ "VXOR    $dst.D,$src1.D,$src2.D\t! and vectors (8 bytes)" %}
14531   ins_encode %{
14532     bool quad = false;
14533     __ vxorI($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister,
14534              quad);
14535   %}
14536   ins_pipe( ialu_reg_reg ); // FIXME
14537 %}
14538 
14539 instruct vxorX(vecX dst, vecX src1, vecX src2) %{
14540   predicate(n->as_Vector()->length_in_bytes() == 16);
14541   match(Set dst (XorV src1 src2));
14542   format %{ "VXOR    $dst.Q,$src1.Q,$src2.Q\t! and vectors (16 bytes)" %}
14543   ins_encode %{
14544     bool quad = true;
14545     __ vxorI($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister,
14546              quad);
14547   %}
14548   ins_pipe( ialu_reg_reg ); // FIXME
14549 %}
14550 
14551 
14552 //----------PEEPHOLE RULES-----------------------------------------------------
14553 // These must follow all instruction definitions as they use the names
14554 // defined in the instructions definitions.
14555 //
14556 // peepmatch ( root_instr_name [preceding_instruction]* );
14557 //
14558 // peepconstraint %{
14559 // (instruction_number.operand_name relational_op instruction_number.operand_name
14560 //  [, ...] );
14561 // // instruction numbers are zero-based using left to right order in peepmatch
14562 //
14563 // peepreplace ( instr_name  ( [instruction_number.operand_name]* ) );
14564 // // provide an instruction_number.operand_name for each operand that appears
14565 // // in the replacement instruction's match rule
14566 //
14567 // ---------VM FLAGS---------------------------------------------------------
14568 //
14569 // All peephole optimizations can be turned off using -XX:-OptoPeephole
14570 //
14571 // Each peephole rule is given an identifying number starting with zero and
14572 // increasing by one in the order seen by the parser.  An individual peephole
14573 // can be enabled, and all others disabled, by using -XX:OptoPeepholeAt=#
14574 // on the command-line.
14575 //
14576 // ---------CURRENT LIMITATIONS----------------------------------------------
14577 //
14578 // Only match adjacent instructions in same basic block
14579 // Only equality constraints
14580 // Only constraints between operands, not (0.dest_reg == EAX_enc)
14581 // Only one replacement instruction
14582 //
14583 // ---------EXAMPLE----------------------------------------------------------
14584 //
14585 // // pertinent parts of existing instructions in architecture description
14586 // instruct movI(eRegI dst, eRegI src) %{
14587 //   match(Set dst (CopyI src));
14588 // %}
14589 //
14590 // instruct incI_eReg(eRegI dst, immI1 src, eFlagsReg cr) %{
14591 //   match(Set dst (AddI dst src));
14592 //   effect(KILL cr);
14593 // %}
14594 //
14595 // // Change (inc mov) to lea
14596 // peephole %{
14597 //   // increment preceeded by register-register move
14598 //   peepmatch ( incI_eReg movI );
14599 //   // require that the destination register of the increment
14600 //   // match the destination register of the move
14601 //   peepconstraint ( 0.dst == 1.dst );
14602 //   // construct a replacement instruction that sets
14603 //   // the destination to ( move's source register + one )
14604 //   peepreplace ( incI_eReg_immI1( 0.dst 1.src 0.src ) );
14605 // %}
14606 //
14607 
14608 // // Change load of spilled value to only a spill
14609 // instruct storeI(memory mem, eRegI src) %{
14610 //   match(Set mem (StoreI mem src));
14611 // %}
14612 //
14613 // instruct loadI(eRegI dst, memory mem) %{
14614 //   match(Set dst (LoadI mem));
14615 // %}
14616 //
14617 // peephole %{
14618 //   peepmatch ( loadI storeI );
14619 //   peepconstraint ( 1.src == 0.dst, 1.mem == 0.mem );
14620 //   peepreplace ( storeI( 1.mem 1.mem 1.src ) );
14621 // %}
14622 
14623 //----------SMARTSPILL RULES---------------------------------------------------
14624 // These must follow all instruction definitions as they use the names
14625 // defined in the instructions definitions.
14626 //
14627 // ARM will probably not have any of these rules due to RISC instruction set.
14628 
14629 //----------PIPELINE-----------------------------------------------------------
14630 // Rules which define the behavior of the target architectures pipeline.
--- EOF ---