1 //
   2 // Copyright (c) 2008, 2015, Oracle and/or its affiliates. All rights reserved.
   3 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4 //
   5 // This code is free software; you can redistribute it and/or modify it
   6 // under the terms of the GNU General Public License version 2 only, as
   7 // published by the Free Software Foundation.
   8 //
   9 // This code is distributed in the hope that it will be useful, but WITHOUT
  10 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11 // FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12 // version 2 for more details (a copy is included in the LICENSE file that
  13 // accompanied this code).
  14 //
  15 // You should have received a copy of the GNU General Public License version
  16 // 2 along with this work; if not, write to the Free Software Foundation,
  17 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18 //
  19 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20 // or visit www.oracle.com if you need additional information or have any
  21 // questions.
  22 //
  23 
  24 // ARM Architecture Description File
  25 
  26 //----------DEFINITION BLOCK---------------------------------------------------
  27 // Define name --> value mappings to inform the ADLC of an integer valued name
  28 // Current support includes integer values in the range [0, 0x7FFFFFFF]
  29 // Format:
  30 //        int_def  <name>         ( <int_value>, <expression>);
  31 // Generated Code in ad_<arch>.hpp
  32 //        #define  <name>   (<expression>)
  33 //        // value == <int_value>
  34 // Generated code in ad_<arch>.cpp adlc_verification()
  35 //        assert( <name> == <int_value>, "Expect (<expression>) to equal <int_value>");
  36 //
  37 definitions %{
  38 // The default cost (of an ALU instruction).
  39   int_def DEFAULT_COST      (    100,     100);
  40   int_def HUGE_COST         (1000000, 1000000);
  41 
  42 // Memory refs are twice as expensive as run-of-the-mill.
  43   int_def MEMORY_REF_COST   (    200, DEFAULT_COST * 2);
  44 
  45 // Branches are even more expensive.
  46   int_def BRANCH_COST       (    300, DEFAULT_COST * 3);
  47   int_def CALL_COST         (    300, DEFAULT_COST * 3);
  48 %}
  49 
  50 
  51 //----------SOURCE BLOCK-------------------------------------------------------
  52 // This is a block of C++ code which provides values, functions, and
  53 // definitions necessary in the rest of the architecture description
  54 source_hpp %{
  55 // Header information of the source block.
  56 // Method declarations/definitions which are used outside
  57 // the ad-scope can conveniently be defined here.
  58 //
  59 // To keep related declarations/definitions/uses close together,
  60 // we switch between source %{ }% and source_hpp %{ }% freely as needed.
  61 
  62 // Does destination need to be loaded in a register then passed to a
  63 // branch instruction?
  64 extern bool maybe_far_call(const CallNode *n);
  65 extern bool maybe_far_call(const MachCallNode *n);
  66 static inline bool cache_reachable() {
  67   return MacroAssembler::_cache_fully_reachable();
  68 }
  69 
  70 #ifdef AARCH64
  71 #define ldr_32 ldr_w
  72 #define str_32 str_w
  73 #else
  74 #define ldr_32 ldr
  75 #define str_32 str
  76 #define tst_32 tst
  77 #define teq_32 teq
  78 #endif
  79 #if 1
  80 extern bool PrintOptoAssembly;
  81 #endif
  82 
  83 class c2 {
  84 public:
  85   static OptoRegPair return_value(int ideal_reg);
  86 };
  87 
  88 class CallStubImpl {
  89 
  90   //--------------------------------------------------------------
  91   //---<  Used for optimization in Compile::Shorten_branches  >---
  92   //--------------------------------------------------------------
  93 
  94  public:
  95   // Size of call trampoline stub.
  96   static uint size_call_trampoline() {
  97     return 0; // no call trampolines on this platform
  98   }
  99 
 100   // number of relocations needed by a call trampoline stub
 101   static uint reloc_call_trampoline() {
 102     return 0; // no call trampolines on this platform
 103   }
 104 };
 105 
 106 class HandlerImpl {
 107 
 108  public:
 109 
 110   static int emit_exception_handler(CodeBuffer &cbuf);
 111   static int emit_deopt_handler(CodeBuffer& cbuf);
 112 
 113   static uint size_exception_handler() {
 114 #ifdef AARCH64
 115     // ldr_literal; br; (pad); <literal>
 116     return 3 * Assembler::InstructionSize + wordSize;
 117 #else
 118     return ( 3 * 4 );
 119 #endif
 120   }
 121 
 122 
 123   static uint size_deopt_handler() {
 124     return ( 9 * 4 );
 125   }
 126 
 127 };
 128 
 129 %}
 130 
 131 source %{
 132 #define __ _masm.
 133 
 134 static FloatRegister reg_to_FloatRegister_object(int register_encoding);
 135 static Register reg_to_register_object(int register_encoding);
 136 
 137 
 138 // ****************************************************************************
 139 
 140 // REQUIRED FUNCTIONALITY
 141 
 142 // Indicate if the safepoint node needs the polling page as an input.
 143 // Since ARM does not have absolute addressing, it does.
 144 bool SafePointNode::needs_polling_address_input() {
 145   return true;
 146 }
 147 
 148 // emit an interrupt that is caught by the debugger (for debugging compiler)
 149 void emit_break(CodeBuffer &cbuf) {
 150   MacroAssembler _masm(&cbuf);
 151   __ breakpoint();
 152 }
 153 
 154 #ifndef PRODUCT
 155 void MachBreakpointNode::format( PhaseRegAlloc *, outputStream *st ) const {
 156   st->print("TA");
 157 }
 158 #endif
 159 
 160 void MachBreakpointNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 161   emit_break(cbuf);
 162 }
 163 
 164 uint MachBreakpointNode::size(PhaseRegAlloc *ra_) const {
 165   return MachNode::size(ra_);
 166 }
 167 
 168 
 169 void emit_nop(CodeBuffer &cbuf) {
 170   MacroAssembler _masm(&cbuf);
 171   __ nop();
 172 }
 173 
 174 
 175 void emit_call_reloc(CodeBuffer &cbuf, const MachCallNode *n, MachOper *m, RelocationHolder const& rspec) {
 176   int ret_addr_offset0 = n->as_MachCall()->ret_addr_offset();
 177   int call_site_offset = cbuf.insts()->mark_off();
 178   MacroAssembler _masm(&cbuf);
 179   __ set_inst_mark(); // needed in emit_to_interp_stub() to locate the call
 180   address target = (address)m->method();
 181   assert(n->as_MachCall()->entry_point() == target, "sanity");
 182   assert(maybe_far_call(n) == !__ reachable_from_cache(target), "sanity");
 183   assert(cache_reachable() == __ cache_fully_reachable(), "sanity");
 184 
 185   assert(target != NULL, "need real address");
 186 
 187   int ret_addr_offset = -1;
 188   if (rspec.type() == relocInfo::runtime_call_type) {
 189     __ call(target, rspec);
 190     ret_addr_offset = __ offset();
 191   } else {
 192     // scratches Rtemp
 193     ret_addr_offset = __ patchable_call(target, rspec, true);
 194   }
 195   assert(ret_addr_offset - call_site_offset == ret_addr_offset0, "fix ret_addr_offset()");
 196 }
 197 
 198 //=============================================================================
 199 // REQUIRED FUNCTIONALITY for encoding
 200 void emit_lo(CodeBuffer &cbuf, int val) {  }
 201 void emit_hi(CodeBuffer &cbuf, int val) {  }
 202 
 203 
 204 //=============================================================================
 205 const RegMask& MachConstantBaseNode::_out_RegMask = PTR_REG_mask();
 206 
 207 int Compile::ConstantTable::calculate_table_base_offset() const {
 208 #ifdef AARCH64
 209   return 0;
 210 #else
 211   int offset = -(size() / 2);
 212   // flds, fldd: 8-bit  offset multiplied by 4: +/- 1024
 213   // ldr, ldrb : 12-bit offset:                 +/- 4096
 214   if (!Assembler::is_simm10(offset)) {
 215     offset = Assembler::min_simm10();
 216   }
 217   return offset;
 218 #endif
 219 }
 220 
 221 bool MachConstantBaseNode::requires_postalloc_expand() const { return false; }
 222 void MachConstantBaseNode::postalloc_expand(GrowableArray <Node *> *nodes, PhaseRegAlloc *ra_) {
 223   ShouldNotReachHere();
 224 }
 225 
 226 void MachConstantBaseNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const {
 227   Compile* C = ra_->C;
 228   Compile::ConstantTable& constant_table = C->constant_table();
 229   MacroAssembler _masm(&cbuf);
 230 
 231   Register r = as_Register(ra_->get_encode(this));
 232   CodeSection* consts_section = __ code()->consts();
 233   int consts_size = consts_section->align_at_start(consts_section->size());
 234   assert(constant_table.size() == consts_size, "must be: %d == %d", constant_table.size(), consts_size);
 235 
 236   // Materialize the constant table base.
 237   address baseaddr = consts_section->start() + -(constant_table.table_base_offset());
 238   RelocationHolder rspec = internal_word_Relocation::spec(baseaddr);
 239   __ mov_address(r, baseaddr, rspec);
 240 }
 241 
 242 uint MachConstantBaseNode::size(PhaseRegAlloc*) const {
 243 #ifdef AARCH64
 244   return 5 * Assembler::InstructionSize;
 245 #else
 246   return 8;
 247 #endif
 248 }
 249 
 250 #ifndef PRODUCT
 251 void MachConstantBaseNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
 252   char reg[128];
 253   ra_->dump_register(this, reg);
 254   st->print("MOV_SLOW    &constanttable,%s\t! constant table base", reg);
 255 }
 256 #endif
 257 
 258 #ifndef PRODUCT
 259 void MachPrologNode::format( PhaseRegAlloc *ra_, outputStream *st ) const {
 260   Compile* C = ra_->C;
 261 
 262   for (int i = 0; i < OptoPrologueNops; i++) {
 263     st->print_cr("NOP"); st->print("\t");
 264   }
 265 #ifdef AARCH64
 266   if (OptoPrologueNops <= 0) {
 267     st->print_cr("NOP\t! required for safe patching");
 268     st->print("\t");
 269   }
 270 #endif
 271 
 272   size_t framesize = C->frame_size_in_bytes();
 273   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
 274   int bangsize = C->bang_size_in_bytes();
 275   // Remove two words for return addr and rbp,
 276   framesize -= 2*wordSize;
 277   bangsize -= 2*wordSize;
 278 
 279   // Calls to C2R adapters often do not accept exceptional returns.
 280   // We require that their callers must bang for them.  But be careful, because
 281   // some VM calls (such as call site linkage) can use several kilobytes of
 282   // stack.  But the stack safety zone should account for that.
 283   // See bugs 4446381, 4468289, 4497237.
 284   if (C->need_stack_bang(bangsize)) {
 285     st->print_cr("! stack bang (%d bytes)", bangsize); st->print("\t");
 286   }
 287   st->print_cr("PUSH   R_FP|R_LR_LR"); st->print("\t");
 288   if (framesize != 0) {
 289     st->print   ("SUB    R_SP, R_SP, " SIZE_FORMAT,framesize);
 290   }
 291 }
 292 #endif
 293 
 294 void MachPrologNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 295   Compile* C = ra_->C;
 296   MacroAssembler _masm(&cbuf);
 297 
 298   for (int i = 0; i < OptoPrologueNops; i++) {
 299     __ nop();
 300   }
 301 #ifdef AARCH64
 302   if (OptoPrologueNops <= 0) {
 303     __ nop(); // required for safe patching by patch_verified_entry()
 304   }
 305 #endif
 306 
 307   size_t framesize = C->frame_size_in_bytes();
 308   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
 309   int bangsize = C->bang_size_in_bytes();
 310   // Remove two words for return addr and fp,
 311   framesize -= 2*wordSize;
 312   bangsize -= 2*wordSize;
 313 
 314   // Calls to C2R adapters often do not accept exceptional returns.
 315   // We require that their callers must bang for them.  But be careful, because
 316   // some VM calls (such as call site linkage) can use several kilobytes of
 317   // stack.  But the stack safety zone should account for that.
 318   // See bugs 4446381, 4468289, 4497237.
 319   if (C->need_stack_bang(bangsize)) {
 320     __ arm_stack_overflow_check(bangsize, Rtemp);
 321   }
 322 
 323   __ raw_push(FP, LR);
 324   if (framesize != 0) {
 325     __ sub_slow(SP, SP, framesize);
 326   }
 327 
 328   // offset from scratch buffer is not valid
 329   if (strcmp(cbuf.name(), "Compile::Fill_buffer") == 0) {
 330     C->set_frame_complete( __ offset() );
 331   }
 332 
 333   if (C->has_mach_constant_base_node()) {
 334     // NOTE: We set the table base offset here because users might be
 335     // emitted before MachConstantBaseNode.
 336     Compile::ConstantTable& constant_table = C->constant_table();
 337     constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
 338   }
 339 }
 340 
 341 uint MachPrologNode::size(PhaseRegAlloc *ra_) const {
 342   return MachNode::size(ra_);
 343 }
 344 
 345 int MachPrologNode::reloc() const {
 346   return 10; // a large enough number
 347 }
 348 
 349 //=============================================================================
 350 #ifndef PRODUCT
 351 void MachEpilogNode::format( PhaseRegAlloc *ra_, outputStream *st ) const {
 352   Compile* C = ra_->C;
 353 
 354   size_t framesize = C->frame_size_in_bytes();
 355   framesize -= 2*wordSize;
 356 
 357   if (framesize != 0) {
 358     st->print("ADD    R_SP, R_SP, " SIZE_FORMAT "\n\t",framesize);
 359   }
 360   st->print("POP    R_FP|R_LR_LR");
 361 
 362   if (do_polling() && ra_->C->is_method_compilation()) {
 363     st->print("\n\t");
 364 #ifdef AARCH64
 365     if (MacroAssembler::page_reachable_from_cache(os::get_polling_page())) {
 366       st->print("ADRP     Rtemp, #PollAddr\t! Load Polling address\n\t");
 367       st->print("LDR      ZR,[Rtemp + #PollAddr & 0xfff]\t!Poll for Safepointing");
 368     } else {
 369       st->print("mov_slow Rtemp, #PollAddr\t! Load Polling address\n\t");
 370       st->print("LDR      ZR,[Rtemp]\t!Poll for Safepointing");
 371     }
 372 #else
 373     st->print("MOV    Rtemp, #PollAddr\t! Load Polling address\n\t");
 374     st->print("LDR    Rtemp,[Rtemp]\t!Poll for Safepointing");
 375 #endif
 376   }
 377 }
 378 #endif
 379 
 380 void MachEpilogNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 381   MacroAssembler _masm(&cbuf);
 382   Compile* C = ra_->C;
 383 
 384   size_t framesize = C->frame_size_in_bytes();
 385   framesize -= 2*wordSize;
 386   if (framesize != 0) {
 387     __ add_slow(SP, SP, framesize);
 388   }
 389   __ raw_pop(FP, LR);
 390 
 391   // If this does safepoint polling, then do it here
 392   if (do_polling() && ra_->C->is_method_compilation()) {
 393 #ifdef AARCH64
 394     if (false && MacroAssembler::page_reachable_from_cache(os::get_polling_page())) {
 395 /* FIXME: TODO
 396       __ relocate(relocInfo::xxx);
 397       __ adrp(Rtemp, (intptr_t)os::get_polling_page());
 398       __ relocate(relocInfo::poll_return_type);
 399       int offset = os::get_polling_page() & 0xfff;
 400       __ ldr(ZR, Address(Rtemp + offset));
 401 */
 402     } else {
 403       __ mov_address(Rtemp, (address)os::get_polling_page(), symbolic_Relocation::polling_page_reference);
 404       __ relocate(relocInfo::poll_return_type);
 405       __ ldr(ZR, Address(Rtemp));
 406     }
 407 #else
 408     // mov_slow here is usually one or two instruction
 409     __ mov_address(Rtemp, (address)os::get_polling_page(), symbolic_Relocation::polling_page_reference);
 410     __ relocate(relocInfo::poll_return_type);
 411     __ ldr(Rtemp, Address(Rtemp));
 412 #endif
 413   }
 414 }
 415 
 416 uint MachEpilogNode::size(PhaseRegAlloc *ra_) const {
 417 #ifdef AARCH64
 418   // allow for added alignment nop from mov_address bind_literal
 419   return MachNode::size(ra_) + 1 * Assembler::InstructionSize;
 420 #else
 421   return MachNode::size(ra_);
 422 #endif
 423 }
 424 
 425 int MachEpilogNode::reloc() const {
 426   return 16; // a large enough number
 427 }
 428 
 429 const Pipeline * MachEpilogNode::pipeline() const {
 430   return MachNode::pipeline_class();
 431 }
 432 
 433 int MachEpilogNode::safepoint_offset() const {
 434   assert( do_polling(), "no return for this epilog node");
 435   //  return MacroAssembler::size_of_sethi(os::get_polling_page());
 436   Unimplemented();
 437   return 0;
 438 }
 439 
 440 //=============================================================================
 441 
 442 // Figure out which register class each belongs in: rc_int, rc_float, rc_stack
 443 enum RC { rc_bad, rc_int, rc_float, rc_stack };
 444 static enum RC rc_class( OptoReg::Name reg ) {
 445   if (!OptoReg::is_valid(reg)) return rc_bad;
 446   if (OptoReg::is_stack(reg)) return rc_stack;
 447   VMReg r = OptoReg::as_VMReg(reg);
 448   if (r->is_Register()) return rc_int;
 449   assert(r->is_FloatRegister(), "must be");
 450   return rc_float;
 451 }
 452 
 453 static inline bool is_iRegLd_memhd(OptoReg::Name src_first, OptoReg::Name src_second, int offset) {
 454 #ifdef AARCH64
 455   return is_memoryHD(offset);
 456 #else
 457   int rlo = Matcher::_regEncode[src_first];
 458   int rhi = Matcher::_regEncode[src_second];
 459   if (!((rlo&1)==0 && (rlo+1 == rhi))) {
 460     tty->print_cr("CAUGHT BAD LDRD/STRD");
 461   }
 462   return (rlo&1)==0 && (rlo+1 == rhi) && is_memoryHD(offset);
 463 #endif
 464 }
 465 
 466 uint MachSpillCopyNode::implementation( CodeBuffer *cbuf,
 467                                         PhaseRegAlloc *ra_,
 468                                         bool do_size,
 469                                         outputStream* st ) const {
 470   // Get registers to move
 471   OptoReg::Name src_second = ra_->get_reg_second(in(1));
 472   OptoReg::Name src_first = ra_->get_reg_first(in(1));
 473   OptoReg::Name dst_second = ra_->get_reg_second(this );
 474   OptoReg::Name dst_first = ra_->get_reg_first(this );
 475 
 476   enum RC src_second_rc = rc_class(src_second);
 477   enum RC src_first_rc = rc_class(src_first);
 478   enum RC dst_second_rc = rc_class(dst_second);
 479   enum RC dst_first_rc = rc_class(dst_first);
 480 
 481   assert( OptoReg::is_valid(src_first) && OptoReg::is_valid(dst_first), "must move at least 1 register" );
 482 
 483   // Generate spill code!
 484   int size = 0;
 485 
 486   if (src_first == dst_first && src_second == dst_second)
 487     return size;            // Self copy, no move
 488 
 489 #ifdef TODO
 490   if (bottom_type()->isa_vect() != NULL) {
 491   }
 492 #endif
 493 
 494   // Shared code does not expect instruction set capability based bailouts here.
 495   // Handle offset unreachable bailout with minimal change in shared code.
 496   // Bailout only for real instruction emit.
 497   // This requires a single comment change in shared code. ( see output.cpp "Normal" instruction case )
 498 
 499   MacroAssembler _masm(cbuf);
 500 
 501   // --------------------------------------
 502   // Check for mem-mem move.  Load into unused float registers and fall into
 503   // the float-store case.
 504   if (src_first_rc == rc_stack && dst_first_rc == rc_stack) {
 505     int offset = ra_->reg2offset(src_first);
 506     if (cbuf && !is_memoryfp(offset)) {
 507       ra_->C->record_method_not_compilable("unable to handle large constant offsets");
 508       return 0;
 509     } else {
 510       if (src_second_rc != rc_bad) {
 511         assert((src_first&1)==0 && src_first+1 == src_second, "pair of registers must be aligned/contiguous");
 512         src_first     = OptoReg::Name(R_mem_copy_lo_num);
 513         src_second    = OptoReg::Name(R_mem_copy_hi_num);
 514         src_first_rc  = rc_float;
 515         src_second_rc = rc_float;
 516         if (cbuf) {
 517           __ ldr_double(Rmemcopy, Address(SP, offset));
 518         } else if (!do_size) {
 519           st->print(LDR_DOUBLE "   R_%s,[R_SP + #%d]\t! spill",OptoReg::regname(src_first),offset);
 520         }
 521       } else {
 522         src_first     = OptoReg::Name(R_mem_copy_lo_num);
 523         src_first_rc  = rc_float;
 524         if (cbuf) {
 525           __ ldr_float(Rmemcopy, Address(SP, offset));
 526         } else if (!do_size) {
 527           st->print(LDR_FLOAT "   R_%s,[R_SP + #%d]\t! spill",OptoReg::regname(src_first),offset);
 528         }
 529       }
 530       size += 4;
 531     }
 532   }
 533 
 534   if (src_second_rc == rc_stack && dst_second_rc == rc_stack) {
 535     Unimplemented();
 536   }
 537 
 538   // --------------------------------------
 539   // Check for integer reg-reg copy
 540   if (src_first_rc == rc_int && dst_first_rc == rc_int) {
 541     // Else normal reg-reg copy
 542     assert( src_second != dst_first, "smashed second before evacuating it" );
 543     if (cbuf) {
 544       __ mov(reg_to_register_object(Matcher::_regEncode[dst_first]), reg_to_register_object(Matcher::_regEncode[src_first]));
 545 #ifndef PRODUCT
 546     } else if (!do_size) {
 547       st->print("MOV    R_%s, R_%s\t# spill",
 548                 Matcher::regName[dst_first],
 549                 Matcher::regName[src_first]);
 550 #endif
 551     }
 552 #ifdef AARCH64
 553     if (src_first+1 == src_second && dst_first+1 == dst_second) {
 554       return size + 4;
 555     }
 556 #endif
 557     size += 4;
 558   }
 559 
 560   // Check for integer store
 561   if (src_first_rc == rc_int && dst_first_rc == rc_stack) {
 562     int offset = ra_->reg2offset(dst_first);
 563     if (cbuf && !is_memoryI(offset)) {
 564       ra_->C->record_method_not_compilable("unable to handle large constant offsets");
 565       return 0;
 566     } else {
 567       if (src_second_rc != rc_bad && is_iRegLd_memhd(src_first, src_second, offset)) {
 568         assert((src_first&1)==0 && src_first+1 == src_second, "pair of registers must be aligned/contiguous");
 569         if (cbuf) {
 570           __ str_64(reg_to_register_object(Matcher::_regEncode[src_first]), Address(SP, offset));
 571 #ifndef PRODUCT
 572         } else if (!do_size) {
 573           if (size != 0) st->print("\n\t");
 574           st->print(STR_64 "   R_%s,[R_SP + #%d]\t! spill",OptoReg::regname(src_first), offset);
 575 #endif
 576         }
 577         return size + 4;
 578       } else {
 579         if (cbuf) {
 580           __ str_32(reg_to_register_object(Matcher::_regEncode[src_first]), Address(SP, offset));
 581 #ifndef PRODUCT
 582         } else if (!do_size) {
 583           if (size != 0) st->print("\n\t");
 584           st->print(STR_32 "   R_%s,[R_SP + #%d]\t! spill",OptoReg::regname(src_first), offset);
 585 #endif
 586         }
 587       }
 588     }
 589     size += 4;
 590   }
 591 
 592   // Check for integer load
 593   if (dst_first_rc == rc_int && src_first_rc == rc_stack) {
 594     int offset = ra_->reg2offset(src_first);
 595     if (cbuf && !is_memoryI(offset)) {
 596       ra_->C->record_method_not_compilable("unable to handle large constant offsets");
 597       return 0;
 598     } else {
 599       if (src_second_rc != rc_bad && is_iRegLd_memhd(dst_first, dst_second, offset)) {
 600         assert((src_first&1)==0 && src_first+1 == src_second, "pair of registers must be aligned/contiguous");
 601         if (cbuf) {
 602           __ ldr_64(reg_to_register_object(Matcher::_regEncode[dst_first]), Address(SP, offset));
 603 #ifndef PRODUCT
 604         } else if (!do_size) {
 605           if (size != 0) st->print("\n\t");
 606           st->print(LDR_64 "   R_%s,[R_SP + #%d]\t! spill",OptoReg::regname(dst_first), offset);
 607 #endif
 608         }
 609         return size + 4;
 610       } else {
 611         if (cbuf) {
 612           __ ldr_32(reg_to_register_object(Matcher::_regEncode[dst_first]), Address(SP, offset));
 613 #ifndef PRODUCT
 614         } else if (!do_size) {
 615           if (size != 0) st->print("\n\t");
 616           st->print(LDR_32 "   R_%s,[R_SP + #%d]\t! spill",OptoReg::regname(dst_first), offset);
 617 #endif
 618         }
 619       }
 620     }
 621     size += 4;
 622   }
 623 
 624   // Check for float reg-reg copy
 625   if (src_first_rc == rc_float && dst_first_rc == rc_float) {
 626     if (src_second_rc != rc_bad) {
 627       assert((src_first&1)==0 && src_first+1 == src_second && (dst_first&1)==0 && dst_first+1 == dst_second, "pairs of registers must be aligned/contiguous");
 628       if (cbuf) {
 629       __ mov_double(reg_to_FloatRegister_object(Matcher::_regEncode[dst_first]), reg_to_FloatRegister_object(Matcher::_regEncode[src_first]));
 630 #ifndef PRODUCT
 631       } else if (!do_size) {
 632         st->print(MOV_DOUBLE "    R_%s, R_%s\t# spill",
 633                   Matcher::regName[dst_first],
 634                   Matcher::regName[src_first]);
 635 #endif
 636       }
 637       return 4;
 638     }
 639     if (cbuf) {
 640       __ mov_float(reg_to_FloatRegister_object(Matcher::_regEncode[dst_first]), reg_to_FloatRegister_object(Matcher::_regEncode[src_first]));
 641 #ifndef PRODUCT
 642     } else if (!do_size) {
 643       st->print(MOV_FLOAT "    R_%s, R_%s\t# spill",
 644                 Matcher::regName[dst_first],
 645                 Matcher::regName[src_first]);
 646 #endif
 647     }
 648     size = 4;
 649   }
 650 
 651   // Check for float store
 652   if (src_first_rc == rc_float && dst_first_rc == rc_stack) {
 653     int offset = ra_->reg2offset(dst_first);
 654     if (cbuf && !is_memoryfp(offset)) {
 655       ra_->C->record_method_not_compilable("unable to handle large constant offsets");
 656       return 0;
 657     } else {
 658       // Further check for aligned-adjacent pair, so we can use a double store
 659       if (src_second_rc != rc_bad) {
 660         assert((src_first&1)==0 && src_first+1 == src_second && (dst_first&1)==0 && dst_first+1 == dst_second, "pairs of registers and stack slots must be aligned/contiguous");
 661         if (cbuf) {
 662           __ str_double(reg_to_FloatRegister_object(Matcher::_regEncode[src_first]), Address(SP, offset));
 663 #ifndef PRODUCT
 664         } else if (!do_size) {
 665           if (size != 0) st->print("\n\t");
 666           st->print(STR_DOUBLE "   R_%s,[R_SP + #%d]\t! spill",OptoReg::regname(src_first),offset);
 667 #endif
 668         }
 669         return size + 4;
 670       } else {
 671         if (cbuf) {
 672           __ str_float(reg_to_FloatRegister_object(Matcher::_regEncode[src_first]), Address(SP, offset));
 673 #ifndef PRODUCT
 674         } else if (!do_size) {
 675           if (size != 0) st->print("\n\t");
 676           st->print(STR_FLOAT "   R_%s,[R_SP + #%d]\t! spill",OptoReg::regname(src_first),offset);
 677 #endif
 678         }
 679       }
 680     }
 681     size += 4;
 682   }
 683 
 684   // Check for float load
 685   if (dst_first_rc == rc_float && src_first_rc == rc_stack) {
 686     int offset = ra_->reg2offset(src_first);
 687     if (cbuf && !is_memoryfp(offset)) {
 688       ra_->C->record_method_not_compilable("unable to handle large constant offsets");
 689       return 0;
 690     } else {
 691       // Further check for aligned-adjacent pair, so we can use a double store
 692       if (src_second_rc != rc_bad) {
 693         assert((src_first&1)==0 && src_first+1 == src_second && (dst_first&1)==0 && dst_first+1 == dst_second, "pairs of registers and stack slots must be aligned/contiguous");
 694         if (cbuf) {
 695           __ ldr_double(reg_to_FloatRegister_object(Matcher::_regEncode[dst_first]), Address(SP, offset));
 696 #ifndef PRODUCT
 697         } else if (!do_size) {
 698           if (size != 0) st->print("\n\t");
 699           st->print(LDR_DOUBLE "   R_%s,[R_SP + #%d]\t! spill",OptoReg::regname(dst_first),offset);
 700 #endif
 701         }
 702         return size + 4;
 703       } else {
 704         if (cbuf) {
 705           __ ldr_float(reg_to_FloatRegister_object(Matcher::_regEncode[dst_first]), Address(SP, offset));
 706 #ifndef PRODUCT
 707         } else if (!do_size) {
 708           if (size != 0) st->print("\n\t");
 709           st->print(LDR_FLOAT "   R_%s,[R_SP + #%d]\t! spill",OptoReg::regname(dst_first),offset);
 710 #endif
 711         }
 712       }
 713     }
 714     size += 4;
 715   }
 716 
 717   // check for int reg -> float reg move
 718   if (src_first_rc == rc_int && dst_first_rc == rc_float) {
 719     // Further check for aligned-adjacent pair, so we can use a single instruction
 720     if (src_second_rc != rc_bad) {
 721       assert((dst_first&1)==0 && dst_first+1 == dst_second, "pairs of registers must be aligned/contiguous");
 722       assert((src_first&1)==0 && src_first+1 == src_second, "pairs of registers must be aligned/contiguous");
 723       assert(src_second_rc == rc_int && dst_second_rc == rc_float, "unsupported");
 724       if (cbuf) {
 725 #ifdef AARCH64
 726         __ fmov_dx(reg_to_FloatRegister_object(Matcher::_regEncode[dst_first]), reg_to_register_object(Matcher::_regEncode[src_first]));
 727 #else
 728         __ fmdrr(reg_to_FloatRegister_object(Matcher::_regEncode[dst_first]), reg_to_register_object(Matcher::_regEncode[src_first]), reg_to_register_object(Matcher::_regEncode[src_second]));
 729 #endif
 730 #ifndef PRODUCT
 731       } else if (!do_size) {
 732         if (size != 0) st->print("\n\t");
 733 #ifdef AARCH64
 734         st->print("FMOV_DX   R_%s, R_%s\t! spill",OptoReg::regname(dst_first), OptoReg::regname(src_first));
 735 #else
 736         st->print("FMDRR   R_%s, R_%s, R_%s\t! spill",OptoReg::regname(dst_first), OptoReg::regname(src_first), OptoReg::regname(src_second));
 737 #endif
 738 #endif
 739       }
 740       return size + 4;
 741     } else {
 742       if (cbuf) {
 743         __ fmsr(reg_to_FloatRegister_object(Matcher::_regEncode[dst_first]), reg_to_register_object(Matcher::_regEncode[src_first]));
 744 #ifndef PRODUCT
 745       } else if (!do_size) {
 746         if (size != 0) st->print("\n\t");
 747         st->print(FMSR "   R_%s, R_%s\t! spill",OptoReg::regname(dst_first), OptoReg::regname(src_first));
 748 #endif
 749       }
 750       size += 4;
 751     }
 752   }
 753 
 754   // check for float reg -> int reg move
 755   if (src_first_rc == rc_float && dst_first_rc == rc_int) {
 756     // Further check for aligned-adjacent pair, so we can use a single instruction
 757     if (src_second_rc != rc_bad) {
 758       assert((src_first&1)==0 && src_first+1 == src_second, "pairs of registers must be aligned/contiguous");
 759       assert((dst_first&1)==0 && dst_first+1 == dst_second, "pairs of registers must be aligned/contiguous");
 760       assert(src_second_rc == rc_float && dst_second_rc == rc_int, "unsupported");
 761       if (cbuf) {
 762 #ifdef AARCH64
 763         __ fmov_xd(reg_to_register_object(Matcher::_regEncode[dst_first]), reg_to_FloatRegister_object(Matcher::_regEncode[src_first]));
 764 #else
 765         __ fmrrd(reg_to_register_object(Matcher::_regEncode[dst_first]), reg_to_register_object(Matcher::_regEncode[dst_second]), reg_to_FloatRegister_object(Matcher::_regEncode[src_first]));
 766 #endif
 767 #ifndef PRODUCT
 768       } else if (!do_size) {
 769         if (size != 0) st->print("\n\t");
 770 #ifdef AARCH64
 771         st->print("FMOV_XD R_%s, R_%s\t! spill",OptoReg::regname(dst_first), OptoReg::regname(src_first));
 772 #else
 773         st->print("FMRRD   R_%s, R_%s, R_%s\t! spill",OptoReg::regname(dst_first), OptoReg::regname(dst_second), OptoReg::regname(src_first));
 774 #endif
 775 #endif
 776       }
 777       return size + 4;
 778     } else {
 779       if (cbuf) {
 780         __ fmrs(reg_to_register_object(Matcher::_regEncode[dst_first]), reg_to_FloatRegister_object(Matcher::_regEncode[src_first]));
 781 #ifndef PRODUCT
 782       } else if (!do_size) {
 783         if (size != 0) st->print("\n\t");
 784         st->print(FMRS "   R_%s, R_%s\t! spill",OptoReg::regname(dst_first), OptoReg::regname(src_first));
 785 #endif
 786       }
 787       size += 4;
 788     }
 789   }
 790 
 791   // --------------------------------------------------------------------
 792   // Check for hi bits still needing moving.  Only happens for misaligned
 793   // arguments to native calls.
 794   if (src_second == dst_second)
 795     return size;               // Self copy; no move
 796   assert( src_second_rc != rc_bad && dst_second_rc != rc_bad, "src_second & dst_second cannot be Bad" );
 797 
 798 #ifndef AARCH64
 799   // Check for integer reg-reg copy.  Hi bits are stuck up in the top
 800   // 32-bits of a 64-bit register, but are needed in low bits of another
 801   // register (else it's a hi-bits-to-hi-bits copy which should have
 802   // happened already as part of a 64-bit move)
 803   if (src_second_rc == rc_int && dst_second_rc == rc_int) {
 804     if (cbuf) {
 805       __ mov(reg_to_register_object(Matcher::_regEncode[dst_second]), reg_to_register_object(Matcher::_regEncode[src_second]));
 806 #ifndef PRODUCT
 807     } else if (!do_size) {
 808       if (size != 0) st->print("\n\t");
 809       st->print("MOV    R_%s, R_%s\t# spill high",
 810                 Matcher::regName[dst_second],
 811                 Matcher::regName[src_second]);
 812 #endif
 813     }
 814     return size+4;
 815   }
 816 
 817   // Check for high word integer store
 818   if (src_second_rc == rc_int && dst_second_rc == rc_stack) {
 819     int offset = ra_->reg2offset(dst_second);
 820 
 821     if (cbuf && !is_memoryP(offset)) {
 822       ra_->C->record_method_not_compilable("unable to handle large constant offsets");
 823       return 0;
 824     } else {
 825       if (cbuf) {
 826         __ str(reg_to_register_object(Matcher::_regEncode[src_second]), Address(SP, offset));
 827 #ifndef PRODUCT
 828       } else if (!do_size) {
 829         if (size != 0) st->print("\n\t");
 830         st->print("STR   R_%s,[R_SP + #%d]\t! spill",OptoReg::regname(src_second), offset);
 831 #endif
 832       }
 833     }
 834     return size + 4;
 835   }
 836 
 837   // Check for high word integer load
 838   if (dst_second_rc == rc_int && src_second_rc == rc_stack) {
 839     int offset = ra_->reg2offset(src_second);
 840     if (cbuf && !is_memoryP(offset)) {
 841       ra_->C->record_method_not_compilable("unable to handle large constant offsets");
 842       return 0;
 843     } else {
 844       if (cbuf) {
 845         __ ldr(reg_to_register_object(Matcher::_regEncode[dst_second]), Address(SP, offset));
 846 #ifndef PRODUCT
 847       } else if (!do_size) {
 848         if (size != 0) st->print("\n\t");
 849         st->print("LDR   R_%s,[R_SP + #%d]\t! spill",OptoReg::regname(dst_second), offset);
 850 #endif
 851       }
 852     }
 853     return size + 4;
 854   }
 855 #endif
 856 
 857   Unimplemented();
 858   return 0; // Mute compiler
 859 }
 860 
 861 #ifndef PRODUCT
 862 void MachSpillCopyNode::format( PhaseRegAlloc *ra_, outputStream *st ) const {
 863   implementation( NULL, ra_, false, st );
 864 }
 865 #endif
 866 
 867 void MachSpillCopyNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 868   implementation( &cbuf, ra_, false, NULL );
 869 }
 870 
 871 uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
 872   return implementation( NULL, ra_, true, NULL );
 873 }
 874 
 875 //=============================================================================
 876 #ifndef PRODUCT
 877 void MachNopNode::format( PhaseRegAlloc *, outputStream *st ) const {
 878   st->print("NOP \t# %d bytes pad for loops and calls", 4 * _count);
 879 }
 880 #endif
 881 
 882 void MachNopNode::emit(CodeBuffer &cbuf, PhaseRegAlloc * ) const {
 883   MacroAssembler _masm(&cbuf);
 884   for(int i = 0; i < _count; i += 1) {
 885     __ nop();
 886   }
 887 }
 888 
 889 uint MachNopNode::size(PhaseRegAlloc *ra_) const {
 890   return 4 * _count;
 891 }
 892 
 893 
 894 //=============================================================================
 895 #ifndef PRODUCT
 896 void BoxLockNode::format( PhaseRegAlloc *ra_, outputStream *st ) const {
 897   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
 898   int reg = ra_->get_reg_first(this);
 899   st->print("ADD    %s,R_SP+#%d",Matcher::regName[reg], offset);
 900 }
 901 #endif
 902 
 903 void BoxLockNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 904   MacroAssembler _masm(&cbuf);
 905   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
 906   int reg = ra_->get_encode(this);
 907   Register dst = reg_to_register_object(reg);
 908 
 909   if (is_aimm(offset)) {
 910     __ add(dst, SP, offset);
 911   } else {
 912     __ mov_slow(dst, offset);
 913 #ifdef AARCH64
 914     __ add(dst, SP, dst, ex_lsl);
 915 #else
 916     __ add(dst, SP, dst);
 917 #endif
 918   }
 919 }
 920 
 921 uint BoxLockNode::size(PhaseRegAlloc *ra_) const {
 922   // BoxLockNode is not a MachNode, so we can't just call MachNode::size(ra_)
 923   assert(ra_ == ra_->C->regalloc(), "sanity");
 924   return ra_->C->scratch_emit_size(this);
 925 }
 926 
 927 //=============================================================================
 928 #ifndef PRODUCT
 929 #ifdef AARCH64
 930 #define R_RTEMP "R_R16"
 931 #else
 932 #define R_RTEMP "R_R12"
 933 #endif
 934 void MachUEPNode::format( PhaseRegAlloc *ra_, outputStream *st ) const {
 935   st->print_cr("\nUEP:");
 936   if (UseCompressedClassPointers) {
 937     st->print_cr("\tLDR_w " R_RTEMP ",[R_R0 + oopDesc::klass_offset_in_bytes]\t! Inline cache check");
 938     st->print_cr("\tdecode_klass " R_RTEMP);
 939   } else {
 940     st->print_cr("\tLDR   " R_RTEMP ",[R_R0 + oopDesc::klass_offset_in_bytes]\t! Inline cache check");
 941   }
 942   st->print_cr("\tCMP   " R_RTEMP ",R_R8" );
 943   st->print   ("\tB.NE  SharedRuntime::handle_ic_miss_stub");
 944 }
 945 #endif
 946 
 947 void MachUEPNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 948   MacroAssembler _masm(&cbuf);
 949   Register iCache  = reg_to_register_object(Matcher::inline_cache_reg_encode());
 950   assert(iCache == Ricklass, "should be");
 951   Register receiver = R0;
 952 
 953   __ load_klass(Rtemp, receiver);
 954   __ cmp(Rtemp, iCache);
 955 #ifdef AARCH64
 956   Label match;
 957   __ b(match, eq);
 958   __ jump(SharedRuntime::get_ic_miss_stub(), relocInfo::runtime_call_type, Rtemp);
 959   __ bind(match);
 960 #else
 961   __ jump(SharedRuntime::get_ic_miss_stub(), relocInfo::runtime_call_type, noreg, ne);
 962 #endif
 963 }
 964 
 965 uint MachUEPNode::size(PhaseRegAlloc *ra_) const {
 966   return MachNode::size(ra_);
 967 }
 968 
 969 
 970 //=============================================================================
 971 
 972 // Emit exception handler code.
 973 int HandlerImpl::emit_exception_handler(CodeBuffer& cbuf) {
 974   MacroAssembler _masm(&cbuf);
 975 
 976   address base = __ start_a_stub(size_exception_handler());
 977   if (base == NULL) {
 978     ciEnv::current()->record_failure("CodeCache is full");
 979     return 0;  // CodeBuffer::expand failed
 980   }
 981 
 982   int offset = __ offset();
 983 
 984   // OK to trash LR, because exception blob will kill it
 985   __ jump(OptoRuntime::exception_blob()->entry_point(), relocInfo::runtime_call_type, LR_tmp);
 986 
 987   assert(__ offset() - offset <= (int) size_exception_handler(), "overflow");
 988 
 989   __ end_a_stub();
 990 
 991   return offset;
 992 }
 993 
 994 int HandlerImpl::emit_deopt_handler(CodeBuffer& cbuf) {
 995   // Can't use any of the current frame's registers as we may have deopted
 996   // at a poll and everything can be live.
 997   MacroAssembler _masm(&cbuf);
 998 
 999   address base = __ start_a_stub(size_deopt_handler());
1000   if (base == NULL) {
1001     ciEnv::current()->record_failure("CodeCache is full");
1002     return 0;  // CodeBuffer::expand failed
1003   }
1004 
1005   int offset = __ offset();
1006   address deopt_pc = __ pc();
1007 
1008 #ifdef AARCH64
1009   // See LR saved by caller in sharedRuntime_arm.cpp
1010   // see also hse1 ws
1011   // see also LIR_Assembler::emit_deopt_handler
1012 
1013   __ raw_push(LR, LR); // preserve LR in both slots
1014   __ mov_relative_address(LR, deopt_pc);
1015   __ str(LR, Address(SP, 1 * wordSize)); // save deopt PC
1016   // OK to kill LR, because deopt blob will restore it from SP[0]
1017   __ jump(SharedRuntime::deopt_blob()->unpack(), relocInfo::runtime_call_type, LR_tmp);
1018 #else
1019   __ sub(SP, SP, wordSize); // make room for saved PC
1020   __ push(LR); // save LR that may be live when we get here
1021   __ mov_relative_address(LR, deopt_pc);
1022   __ str(LR, Address(SP, wordSize)); // save deopt PC
1023   __ pop(LR); // restore LR
1024   __ jump(SharedRuntime::deopt_blob()->unpack(), relocInfo::runtime_call_type, noreg);
1025 #endif
1026 
1027   assert(__ offset() - offset <= (int) size_deopt_handler(), "overflow");
1028 
1029   __ end_a_stub();
1030   return offset;
1031 }
1032 
1033 const bool Matcher::match_rule_supported(int opcode) {
1034   if (!has_match_rule(opcode))
1035     return false;
1036 
1037   switch (opcode) {
1038   case Op_PopCountI:
1039   case Op_PopCountL:
1040     if (!UsePopCountInstruction)
1041       return false;
1042     break;
1043   case Op_LShiftCntV:
1044   case Op_RShiftCntV:
1045   case Op_AddVB:
1046   case Op_AddVS:
1047   case Op_AddVI:
1048   case Op_AddVL:
1049   case Op_SubVB:
1050   case Op_SubVS:
1051   case Op_SubVI:
1052   case Op_SubVL:
1053   case Op_MulVS:
1054   case Op_MulVI:
1055   case Op_LShiftVB:
1056   case Op_LShiftVS:
1057   case Op_LShiftVI:
1058   case Op_LShiftVL:
1059   case Op_RShiftVB:
1060   case Op_RShiftVS:
1061   case Op_RShiftVI:
1062   case Op_RShiftVL:
1063   case Op_URShiftVB:
1064   case Op_URShiftVS:
1065   case Op_URShiftVI:
1066   case Op_URShiftVL:
1067   case Op_AndV:
1068   case Op_OrV:
1069   case Op_XorV:
1070     return VM_Version::has_simd();
1071   case Op_LoadVector:
1072   case Op_StoreVector:
1073   case Op_AddVF:
1074   case Op_SubVF:
1075   case Op_MulVF:
1076 #ifdef AARCH64
1077     return VM_Version::has_simd();
1078 #else
1079     return VM_Version::has_vfp() || VM_Version::has_simd();
1080 #endif
1081   case Op_AddVD:
1082   case Op_SubVD:
1083   case Op_MulVD:
1084   case Op_DivVF:
1085   case Op_DivVD:
1086 #ifdef AARCH64
1087     return VM_Version::has_simd();
1088 #else
1089     return VM_Version::has_vfp();
1090 #endif
1091   }
1092 
1093   return true;  // Per default match rules are supported.
1094 }
1095 
1096 const bool Matcher::match_rule_supported_vector(int opcode, int vlen) {
1097 
1098   // TODO
1099   // identify extra cases that we might want to provide match rules for
1100   // e.g. Op_ vector nodes and other intrinsics while guarding with vlen
1101   bool ret_value = match_rule_supported(opcode);
1102   // Add rules here.
1103 
1104   return ret_value;  // Per default match rules are supported.
1105 }
1106 
1107 const bool Matcher::has_predicated_vectors(void) {
1108   return false;
1109 }
1110 
1111 const int Matcher::float_pressure(int default_pressure_threshold) {
1112   return default_pressure_threshold;
1113 }
1114 
1115 int Matcher::regnum_to_fpu_offset(int regnum) {
1116   return regnum - 32; // The FP registers are in the second chunk
1117 }
1118 
1119 // Vector width in bytes
1120 const int Matcher::vector_width_in_bytes(BasicType bt) {
1121   return MaxVectorSize;
1122 }
1123 
1124 // Vector ideal reg corresponding to specified size in bytes
1125 const int Matcher::vector_ideal_reg(int size) {
1126   assert(MaxVectorSize >= size, "");
1127   switch(size) {
1128     case  8: return Op_VecD;
1129     case 16: return Op_VecX;
1130   }
1131   ShouldNotReachHere();
1132   return 0;
1133 }
1134 
1135 const int Matcher::vector_shift_count_ideal_reg(int size) {
1136   return vector_ideal_reg(size);
1137 }
1138 
1139 // Limits on vector size (number of elements) loaded into vector.
1140 const int Matcher::max_vector_size(const BasicType bt) {
1141   assert(is_java_primitive(bt), "only primitive type vectors");
1142   return vector_width_in_bytes(bt)/type2aelembytes(bt);
1143 }
1144 
1145 const int Matcher::min_vector_size(const BasicType bt) {
1146   assert(is_java_primitive(bt), "only primitive type vectors");
1147   return 8/type2aelembytes(bt);
1148 }
1149 
1150 // ARM doesn't support misaligned vectors store/load.
1151 const bool Matcher::misaligned_vectors_ok() {
1152   return false;
1153 }
1154 
1155 // ARM doesn't support AES intrinsics
1156 const bool Matcher::pass_original_key_for_aes() {
1157   return false;
1158 }
1159 
1160 const bool Matcher::convL2FSupported(void) {
1161 #ifdef AARCH64
1162   return true;
1163 #else
1164   return false;
1165 #endif
1166 }
1167 
1168 // Is this branch offset short enough that a short branch can be used?
1169 //
1170 // NOTE: If the platform does not provide any short branch variants, then
1171 //       this method should return false for offset 0.
1172 bool Matcher::is_short_branch_offset(int rule, int br_size, int offset) {
1173   // The passed offset is relative to address of the branch.
1174   // On ARM a branch displacement is calculated relative to address
1175   // of the branch + 8.
1176   //
1177   // offset -= 8;
1178   // return (Assembler::is_simm24(offset));
1179   return false;
1180 }
1181 
1182 const bool Matcher::isSimpleConstant64(jlong value) {
1183   // Will one (StoreL ConL) be cheaper than two (StoreI ConI)?.
1184 #ifdef AARCH64
1185   return (value == 0);
1186 #else
1187   return false;
1188 #endif
1189 }
1190 
1191 // No scaling for the parameter the ClearArray node.
1192 const bool Matcher::init_array_count_is_in_bytes = true;
1193 
1194 #ifdef AARCH64
1195 const int Matcher::long_cmove_cost() { return 1; }
1196 #else
1197 // Needs 2 CMOV's for longs.
1198 const int Matcher::long_cmove_cost() { return 2; }
1199 #endif
1200 
1201 #ifdef AARCH64
1202 const int Matcher::float_cmove_cost() { return 1; }
1203 #else
1204 // CMOVF/CMOVD are expensive on ARM.
1205 const int Matcher::float_cmove_cost() { return ConditionalMoveLimit; }
1206 #endif
1207 
1208 // Does the CPU require late expand (see block.cpp for description of late expand)?
1209 const bool Matcher::require_postalloc_expand = false;
1210 
1211 // Do we need to mask the count passed to shift instructions or does
1212 // the cpu only look at the lower 5/6 bits anyway?
1213 // FIXME: does this handle vector shifts as well?
1214 #ifdef AARCH64
1215 const bool Matcher::need_masked_shift_count = false;
1216 #else
1217 const bool Matcher::need_masked_shift_count = true;
1218 #endif
1219 
1220 const bool Matcher::convi2l_type_required = true;
1221 
1222 // Should the Matcher clone shifts on addressing modes, expecting them
1223 // to be subsumed into complex addressing expressions or compute them
1224 // into registers?
1225 bool Matcher::clone_address_expressions(AddPNode* m, Matcher::MStack& mstack, VectorSet& address_visited) {
1226   return clone_base_plus_offset_address(m, mstack, address_visited);
1227 }
1228 
1229 void Compile::reshape_address(AddPNode* addp) {
1230 }
1231 
1232 bool Matcher::narrow_oop_use_complex_address() {
1233   NOT_LP64(ShouldNotCallThis());
1234   assert(UseCompressedOops, "only for compressed oops code");
1235   return false;
1236 }
1237 
1238 bool Matcher::narrow_klass_use_complex_address() {
1239   NOT_LP64(ShouldNotCallThis());
1240   assert(UseCompressedClassPointers, "only for compressed klass code");
1241   return false;
1242 }
1243 
1244 // Is it better to copy float constants, or load them directly from memory?
1245 // Intel can load a float constant from a direct address, requiring no
1246 // extra registers.  Most RISCs will have to materialize an address into a
1247 // register first, so they would do better to copy the constant from stack.
1248 const bool Matcher::rematerialize_float_constants = false;
1249 
1250 // If CPU can load and store mis-aligned doubles directly then no fixup is
1251 // needed.  Else we split the double into 2 integer pieces and move it
1252 // piece-by-piece.  Only happens when passing doubles into C code as the
1253 // Java calling convention forces doubles to be aligned.
1254 #ifdef AARCH64
1255 // On stack replacement support:
1256 // We don't need Load[DL]_unaligned support, because interpreter stack
1257 // has correct alignment
1258 const bool Matcher::misaligned_doubles_ok = true;
1259 #else
1260 const bool Matcher::misaligned_doubles_ok = false;
1261 #endif
1262 
1263 // No-op on ARM.
1264 void Matcher::pd_implicit_null_fixup(MachNode *node, uint idx) {
1265 }
1266 
1267 // Advertise here if the CPU requires explicit rounding operations
1268 // to implement the UseStrictFP mode.
1269 const bool Matcher::strict_fp_requires_explicit_rounding = false;
1270 
1271 // Are floats converted to double when stored to stack during deoptimization?
1272 // ARM does not handle callee-save floats.
1273 bool Matcher::float_in_double() {
1274   return false;
1275 }
1276 
1277 // Do ints take an entire long register or just half?
1278 // Note that we if-def off of _LP64.
1279 // The relevant question is how the int is callee-saved.  In _LP64
1280 // the whole long is written but de-opt'ing will have to extract
1281 // the relevant 32 bits, in not-_LP64 only the low 32 bits is written.
1282 #ifdef _LP64
1283 const bool Matcher::int_in_long = true;
1284 #else
1285 const bool Matcher::int_in_long = false;
1286 #endif
1287 
1288 // Return whether or not this register is ever used as an argument.  This
1289 // function is used on startup to build the trampoline stubs in generateOptoStub.
1290 // Registers not mentioned will be killed by the VM call in the trampoline, and
1291 // arguments in those registers not be available to the callee.
1292 bool Matcher::can_be_java_arg( int reg ) {
1293 #ifdef AARCH64
1294   if (reg >= R_R0_num && reg < R_R8_num) return true;
1295   if (reg >= R_V0_num && reg <= R_V7b_num && ((reg & 3) < 2)) return true;
1296 #else
1297   if (reg == R_R0_num ||
1298       reg == R_R1_num ||
1299       reg == R_R2_num ||
1300       reg == R_R3_num) return true;
1301 
1302   if (reg >= R_S0_num &&
1303       reg <= R_S13_num) return true;
1304 #endif
1305   return false;
1306 }
1307 
1308 bool Matcher::is_spillable_arg( int reg ) {
1309   return can_be_java_arg(reg);
1310 }
1311 
1312 bool Matcher::use_asm_for_ldiv_by_con( jlong divisor ) {
1313   return false;
1314 }
1315 
1316 // Register for DIVI projection of divmodI
1317 RegMask Matcher::divI_proj_mask() {
1318   ShouldNotReachHere();
1319   return RegMask();
1320 }
1321 
1322 // Register for MODI projection of divmodI
1323 RegMask Matcher::modI_proj_mask() {
1324   ShouldNotReachHere();
1325   return RegMask();
1326 }
1327 
1328 // Register for DIVL projection of divmodL
1329 RegMask Matcher::divL_proj_mask() {
1330   ShouldNotReachHere();
1331   return RegMask();
1332 }
1333 
1334 // Register for MODL projection of divmodL
1335 RegMask Matcher::modL_proj_mask() {
1336   ShouldNotReachHere();
1337   return RegMask();
1338 }
1339 
1340 const RegMask Matcher::method_handle_invoke_SP_save_mask() {
1341   return FP_REGP_mask();
1342 }
1343 
1344 bool maybe_far_call(const CallNode *n) {
1345   return !MacroAssembler::_reachable_from_cache(n->as_Call()->entry_point());
1346 }
1347 
1348 bool maybe_far_call(const MachCallNode *n) {
1349   return !MacroAssembler::_reachable_from_cache(n->as_MachCall()->entry_point());
1350 }
1351 
1352 %}
1353 
1354 //----------ENCODING BLOCK-----------------------------------------------------
1355 // This block specifies the encoding classes used by the compiler to output
1356 // byte streams.  Encoding classes are parameterized macros used by
1357 // Machine Instruction Nodes in order to generate the bit encoding of the
1358 // instruction.  Operands specify their base encoding interface with the
1359 // interface keyword.  There are currently supported four interfaces,
1360 // REG_INTER, CONST_INTER, MEMORY_INTER, & COND_INTER.  REG_INTER causes an
1361 // operand to generate a function which returns its register number when
1362 // queried.   CONST_INTER causes an operand to generate a function which
1363 // returns the value of the constant when queried.  MEMORY_INTER causes an
1364 // operand to generate four functions which return the Base Register, the
1365 // Index Register, the Scale Value, and the Offset Value of the operand when
1366 // queried.  COND_INTER causes an operand to generate six functions which
1367 // return the encoding code (ie - encoding bits for the instruction)
1368 // associated with each basic boolean condition for a conditional instruction.
1369 //
1370 // Instructions specify two basic values for encoding.  Again, a function
1371 // is available to check if the constant displacement is an oop. They use the
1372 // ins_encode keyword to specify their encoding classes (which must be
1373 // a sequence of enc_class names, and their parameters, specified in
1374 // the encoding block), and they use the
1375 // opcode keyword to specify, in order, their primary, secondary, and
1376 // tertiary opcode.  Only the opcode sections which a particular instruction
1377 // needs for encoding need to be specified.
1378 encode %{
1379   enc_class call_epilog %{
1380     // nothing
1381   %}
1382 
1383   enc_class Java_To_Runtime (method meth) %{
1384     // CALL directly to the runtime
1385     emit_call_reloc(cbuf, as_MachCall(), $meth, runtime_call_Relocation::spec());
1386   %}
1387 
1388   enc_class Java_Static_Call (method meth) %{
1389     // CALL to fixup routine.  Fixup routine uses ScopeDesc info to determine
1390     // who we intended to call.
1391 
1392     if ( !_method) {
1393       emit_call_reloc(cbuf, as_MachCall(), $meth, runtime_call_Relocation::spec());
1394     } else {
1395       int method_index = resolved_method_index(cbuf);
1396       RelocationHolder rspec = _optimized_virtual ? opt_virtual_call_Relocation::spec(method_index)
1397                                                   : static_call_Relocation::spec(method_index);
1398       emit_call_reloc(cbuf, as_MachCall(), $meth, rspec);
1399 
1400       // Emit stubs for static call.
1401       address stub = CompiledStaticCall::emit_to_interp_stub(cbuf);
1402       if (stub == NULL) {
1403         ciEnv::current()->record_failure("CodeCache is full");
1404         return;
1405       }
1406     }
1407   %}
1408 
1409   enc_class save_last_PC %{
1410     // preserve mark
1411     address mark = cbuf.insts()->mark();
1412     debug_only(int off0 = cbuf.insts_size());
1413     MacroAssembler _masm(&cbuf);
1414     int ret_addr_offset = as_MachCall()->ret_addr_offset();
1415     __ adr(LR, mark + ret_addr_offset);
1416     __ str(LR, Address(Rthread, JavaThread::last_Java_pc_offset()));
1417     debug_only(int off1 = cbuf.insts_size());
1418     assert(off1 - off0 == 2 * Assembler::InstructionSize, "correct size prediction");
1419     // restore mark
1420     cbuf.insts()->set_mark(mark);
1421   %}
1422 
1423   enc_class preserve_SP %{
1424     // preserve mark
1425     address mark = cbuf.insts()->mark();
1426     debug_only(int off0 = cbuf.insts_size());
1427     MacroAssembler _masm(&cbuf);
1428     // FP is preserved across all calls, even compiled calls.
1429     // Use it to preserve SP in places where the callee might change the SP.
1430     __ mov(Rmh_SP_save, SP);
1431     debug_only(int off1 = cbuf.insts_size());
1432     assert(off1 - off0 == 4, "correct size prediction");
1433     // restore mark
1434     cbuf.insts()->set_mark(mark);
1435   %}
1436 
1437   enc_class restore_SP %{
1438     MacroAssembler _masm(&cbuf);
1439     __ mov(SP, Rmh_SP_save);
1440   %}
1441 
1442   enc_class Java_Dynamic_Call (method meth) %{
1443     MacroAssembler _masm(&cbuf);
1444     Register R8_ic_reg = reg_to_register_object(Matcher::inline_cache_reg_encode());
1445     assert(R8_ic_reg == Ricklass, "should be");
1446     __ set_inst_mark();
1447 #ifdef AARCH64
1448 // TODO: see C1 LIR_Assembler::ic_call()
1449     InlinedAddress oop_literal((address)Universe::non_oop_word());
1450     int offset = __ offset();
1451     int fixed_size = mov_oop_size * 4;
1452     if (VM_Version::prefer_moves_over_load_literal()) {
1453       uintptr_t val = (uintptr_t)Universe::non_oop_word();
1454       __ movz(R8_ic_reg, (val >>  0) & 0xffff,  0);
1455       __ movk(R8_ic_reg, (val >> 16) & 0xffff, 16);
1456       __ movk(R8_ic_reg, (val >> 32) & 0xffff, 32);
1457       __ movk(R8_ic_reg, (val >> 48) & 0xffff, 48);
1458     } else {
1459       __ ldr_literal(R8_ic_reg, oop_literal);
1460     }
1461     assert(__ offset() - offset == fixed_size, "bad mov_oop size");
1462 #else
1463     __ movw(R8_ic_reg, ((unsigned int)Universe::non_oop_word()) & 0xffff);
1464     __ movt(R8_ic_reg, ((unsigned int)Universe::non_oop_word()) >> 16);
1465 #endif
1466     address  virtual_call_oop_addr = __ inst_mark();
1467     // CALL to fixup routine.  Fixup routine uses ScopeDesc info to determine
1468     // who we intended to call.
1469     int method_index = resolved_method_index(cbuf);
1470     __ relocate(virtual_call_Relocation::spec(virtual_call_oop_addr, method_index));
1471     emit_call_reloc(cbuf, as_MachCall(), $meth, RelocationHolder::none);
1472 #ifdef AARCH64
1473     if (!VM_Version::prefer_moves_over_load_literal()) {
1474       Label skip_literal;
1475       __ b(skip_literal);
1476       int off2 = __ offset();
1477       __ bind_literal(oop_literal);
1478       if (__ offset() - off2 == wordSize) {
1479         // no padding, so insert nop for worst-case sizing
1480         __ nop();
1481       }
1482       __ bind(skip_literal);
1483     }
1484 #endif
1485   %}
1486 
1487   enc_class LdReplImmI(immI src, regD dst, iRegI tmp, int cnt, int wth) %{
1488     // FIXME: load from constant table?
1489     // Load a constant replicated "count" times with width "width"
1490     int count = $cnt$$constant;
1491     int width = $wth$$constant;
1492     assert(count*width == 4, "sanity");
1493     int val = $src$$constant;
1494     if (width < 4) {
1495       int bit_width = width * 8;
1496       val &= (((int)1) << bit_width) - 1; // mask off sign bits
1497       for (int i = 0; i < count - 1; i++) {
1498         val |= (val << bit_width);
1499       }
1500     }
1501     MacroAssembler _masm(&cbuf);
1502 
1503     if (val == -1) {
1504       __ mvn($tmp$$Register, 0);
1505     } else if (val == 0) {
1506       __ mov($tmp$$Register, 0);
1507     } else {
1508       __ movw($tmp$$Register, val & 0xffff);
1509       __ movt($tmp$$Register, (unsigned int)val >> 16);
1510     }
1511     __ fmdrr($dst$$FloatRegister, $tmp$$Register, $tmp$$Register);
1512   %}
1513 
1514   enc_class LdReplImmF(immF src, regD dst, iRegI tmp) %{
1515     // Replicate float con 2 times and pack into vector (8 bytes) in regD.
1516     float fval = $src$$constant;
1517     int val = *((int*)&fval);
1518     MacroAssembler _masm(&cbuf);
1519 
1520     if (val == -1) {
1521       __ mvn($tmp$$Register, 0);
1522     } else if (val == 0) {
1523       __ mov($tmp$$Register, 0);
1524     } else {
1525       __ movw($tmp$$Register, val & 0xffff);
1526       __ movt($tmp$$Register, (unsigned int)val >> 16);
1527     }
1528     __ fmdrr($dst$$FloatRegister, $tmp$$Register, $tmp$$Register);
1529   %}
1530 
1531   enc_class enc_String_Compare(R0RegP str1, R1RegP str2, R2RegI cnt1, R3RegI cnt2, iRegI result, iRegI tmp1, iRegI tmp2) %{
1532     Label Ldone, Lloop;
1533     MacroAssembler _masm(&cbuf);
1534 
1535     Register   str1_reg = $str1$$Register;
1536     Register   str2_reg = $str2$$Register;
1537     Register   cnt1_reg = $cnt1$$Register; // int
1538     Register   cnt2_reg = $cnt2$$Register; // int
1539     Register   tmp1_reg = $tmp1$$Register;
1540     Register   tmp2_reg = $tmp2$$Register;
1541     Register result_reg = $result$$Register;
1542 
1543     assert_different_registers(str1_reg, str2_reg, cnt1_reg, cnt2_reg, tmp1_reg, tmp2_reg);
1544 
1545     // Compute the minimum of the string lengths(str1_reg) and the
1546     // difference of the string lengths (stack)
1547 
1548     // See if the lengths are different, and calculate min in str1_reg.
1549     // Stash diff in tmp2 in case we need it for a tie-breaker.
1550     __ subs_32(tmp2_reg, cnt1_reg, cnt2_reg);
1551 #ifdef AARCH64
1552     Label Lskip;
1553     __ _lsl_w(cnt1_reg, cnt1_reg, exact_log2(sizeof(jchar))); // scale the limit
1554     __ b(Lskip, mi);
1555     __ _lsl_w(cnt1_reg, cnt2_reg, exact_log2(sizeof(jchar))); // scale the limit
1556     __ bind(Lskip);
1557 #else
1558     __ mov(cnt1_reg, AsmOperand(cnt1_reg, lsl, exact_log2(sizeof(jchar)))); // scale the limit
1559     __ mov(cnt1_reg, AsmOperand(cnt2_reg, lsl, exact_log2(sizeof(jchar))), pl); // scale the limit
1560 #endif
1561 
1562     // reallocate cnt1_reg, cnt2_reg, result_reg
1563     // Note:  limit_reg holds the string length pre-scaled by 2
1564     Register limit_reg = cnt1_reg;
1565     Register  chr2_reg = cnt2_reg;
1566     Register  chr1_reg = tmp1_reg;
1567     // str{12} are the base pointers
1568 
1569     // Is the minimum length zero?
1570     __ cmp_32(limit_reg, 0);
1571     if (result_reg != tmp2_reg) {
1572       __ mov(result_reg, tmp2_reg, eq);
1573     }
1574     __ b(Ldone, eq);
1575 
1576     // Load first characters
1577     __ ldrh(chr1_reg, Address(str1_reg, 0));
1578     __ ldrh(chr2_reg, Address(str2_reg, 0));
1579 
1580     // Compare first characters
1581     __ subs(chr1_reg, chr1_reg, chr2_reg);
1582     if (result_reg != chr1_reg) {
1583       __ mov(result_reg, chr1_reg, ne);
1584     }
1585     __ b(Ldone, ne);
1586 
1587     {
1588       // Check after comparing first character to see if strings are equivalent
1589       // Check if the strings start at same location
1590       __ cmp(str1_reg, str2_reg);
1591       // Check if the length difference is zero
1592       __ cond_cmp(tmp2_reg, 0, eq);
1593       __ mov(result_reg, 0, eq); // result is zero
1594       __ b(Ldone, eq);
1595       // Strings might not be equal
1596     }
1597 
1598     __ subs(chr1_reg, limit_reg, 1 * sizeof(jchar));
1599     if (result_reg != tmp2_reg) {
1600       __ mov(result_reg, tmp2_reg, eq);
1601     }
1602     __ b(Ldone, eq);
1603 
1604     // Shift str1_reg and str2_reg to the end of the arrays, negate limit
1605     __ add(str1_reg, str1_reg, limit_reg);
1606     __ add(str2_reg, str2_reg, limit_reg);
1607     __ neg(limit_reg, chr1_reg);  // limit = -(limit-2)
1608 
1609     // Compare the rest of the characters
1610     __ bind(Lloop);
1611     __ ldrh(chr1_reg, Address(str1_reg, limit_reg));
1612     __ ldrh(chr2_reg, Address(str2_reg, limit_reg));
1613     __ subs(chr1_reg, chr1_reg, chr2_reg);
1614     if (result_reg != chr1_reg) {
1615       __ mov(result_reg, chr1_reg, ne);
1616     }
1617     __ b(Ldone, ne);
1618 
1619     __ adds(limit_reg, limit_reg, sizeof(jchar));
1620     __ b(Lloop, ne);
1621 
1622     // If strings are equal up to min length, return the length difference.
1623     if (result_reg != tmp2_reg) {
1624       __ mov(result_reg, tmp2_reg);
1625     }
1626 
1627     // Otherwise, return the difference between the first mismatched chars.
1628     __ bind(Ldone);
1629   %}
1630 
1631   enc_class enc_String_Equals(R0RegP str1, R1RegP str2, R2RegI cnt, iRegI result, iRegI tmp1, iRegI tmp2) %{
1632     Label Lword_loop, Lpost_word, Lchar, Lchar_loop, Ldone, Lequal;
1633     MacroAssembler _masm(&cbuf);
1634 
1635     Register   str1_reg = $str1$$Register;
1636     Register   str2_reg = $str2$$Register;
1637     Register    cnt_reg = $cnt$$Register; // int
1638     Register   tmp1_reg = $tmp1$$Register;
1639     Register   tmp2_reg = $tmp2$$Register;
1640     Register result_reg = $result$$Register;
1641 
1642     assert_different_registers(str1_reg, str2_reg, cnt_reg, tmp1_reg, tmp2_reg, result_reg);
1643 
1644     __ cmp(str1_reg, str2_reg); //same char[] ?
1645     __ b(Lequal, eq);
1646 
1647     __ cbz_32(cnt_reg, Lequal); // count == 0
1648 
1649     //rename registers
1650     Register limit_reg = cnt_reg;
1651     Register  chr1_reg = tmp1_reg;
1652     Register  chr2_reg = tmp2_reg;
1653 
1654     __ logical_shift_left(limit_reg, limit_reg, exact_log2(sizeof(jchar)));
1655 
1656     //check for alignment and position the pointers to the ends
1657     __ orr(chr1_reg, str1_reg, str2_reg);
1658     __ tst(chr1_reg, 0x3);
1659 
1660     // notZero means at least one not 4-byte aligned.
1661     // We could optimize the case when both arrays are not aligned
1662     // but it is not frequent case and it requires additional checks.
1663     __ b(Lchar, ne);
1664 
1665     // Compare char[] arrays aligned to 4 bytes.
1666     __ char_arrays_equals(str1_reg, str2_reg, limit_reg, result_reg,
1667                           chr1_reg, chr2_reg, Ldone);
1668 
1669     __ b(Lequal); // equal
1670 
1671     // char by char compare
1672     __ bind(Lchar);
1673     __ mov(result_reg, 0);
1674     __ add(str1_reg, limit_reg, str1_reg);
1675     __ add(str2_reg, limit_reg, str2_reg);
1676     __ neg(limit_reg, limit_reg); //negate count
1677 
1678     // Lchar_loop
1679     __ bind(Lchar_loop);
1680     __ ldrh(chr1_reg, Address(str1_reg, limit_reg));
1681     __ ldrh(chr2_reg, Address(str2_reg, limit_reg));
1682     __ cmp(chr1_reg, chr2_reg);
1683     __ b(Ldone, ne);
1684     __ adds(limit_reg, limit_reg, sizeof(jchar));
1685     __ b(Lchar_loop, ne);
1686 
1687     __ bind(Lequal);
1688     __ mov(result_reg, 1);  //equal
1689 
1690     __ bind(Ldone);
1691   %}
1692 
1693   enc_class enc_Array_Equals(R0RegP ary1, R1RegP ary2, iRegI tmp1, iRegI tmp2, iRegI tmp3, iRegI result) %{
1694     Label Lvector, Ldone, Lloop, Lequal;
1695     MacroAssembler _masm(&cbuf);
1696 
1697     Register   ary1_reg = $ary1$$Register;
1698     Register   ary2_reg = $ary2$$Register;
1699     Register   tmp1_reg = $tmp1$$Register;
1700     Register   tmp2_reg = $tmp2$$Register;
1701     Register   tmp3_reg = $tmp3$$Register;
1702     Register result_reg = $result$$Register;
1703 
1704     assert_different_registers(ary1_reg, ary2_reg, tmp1_reg, tmp2_reg, tmp3_reg, result_reg);
1705 
1706     int length_offset  = arrayOopDesc::length_offset_in_bytes();
1707     int base_offset    = arrayOopDesc::base_offset_in_bytes(T_CHAR);
1708 
1709     // return true if the same array
1710 #ifdef AARCH64
1711     __ cmp(ary1_reg, ary2_reg);
1712     __ b(Lequal, eq);
1713 
1714     __ mov(result_reg, 0);
1715 
1716     __ cbz(ary1_reg, Ldone); // not equal
1717 
1718     __ cbz(ary2_reg, Ldone); // not equal
1719 #else
1720     __ teq(ary1_reg, ary2_reg);
1721     __ mov(result_reg, 1, eq);
1722     __ b(Ldone, eq); // equal
1723 
1724     __ tst(ary1_reg, ary1_reg);
1725     __ mov(result_reg, 0, eq);
1726     __ b(Ldone, eq);    // not equal
1727 
1728     __ tst(ary2_reg, ary2_reg);
1729     __ mov(result_reg, 0, eq);
1730     __ b(Ldone, eq);    // not equal
1731 #endif
1732 
1733     //load the lengths of arrays
1734     __ ldr_s32(tmp1_reg, Address(ary1_reg, length_offset)); // int
1735     __ ldr_s32(tmp2_reg, Address(ary2_reg, length_offset)); // int
1736 
1737     // return false if the two arrays are not equal length
1738 #ifdef AARCH64
1739     __ cmp_w(tmp1_reg, tmp2_reg);
1740     __ b(Ldone, ne);    // not equal
1741 
1742     __ cbz_w(tmp1_reg, Lequal); // zero-length arrays are equal
1743 #else
1744     __ teq_32(tmp1_reg, tmp2_reg);
1745     __ mov(result_reg, 0, ne);
1746     __ b(Ldone, ne);    // not equal
1747 
1748     __ tst(tmp1_reg, tmp1_reg);
1749     __ mov(result_reg, 1, eq);
1750     __ b(Ldone, eq);    // zero-length arrays are equal
1751 #endif
1752 
1753     // load array addresses
1754     __ add(ary1_reg, ary1_reg, base_offset);
1755     __ add(ary2_reg, ary2_reg, base_offset);
1756 
1757     // renaming registers
1758     Register chr1_reg  =  tmp3_reg;   // for characters in ary1
1759     Register chr2_reg  =  tmp2_reg;   // for characters in ary2
1760     Register limit_reg =  tmp1_reg;   // length
1761 
1762     // set byte count
1763     __ logical_shift_left_32(limit_reg, limit_reg, exact_log2(sizeof(jchar)));
1764 
1765     // Compare char[] arrays aligned to 4 bytes.
1766     __ char_arrays_equals(ary1_reg, ary2_reg, limit_reg, result_reg,
1767                           chr1_reg, chr2_reg, Ldone);
1768     __ bind(Lequal);
1769     __ mov(result_reg, 1);  //equal
1770 
1771     __ bind(Ldone);
1772     %}
1773 %}
1774 
1775 //----------FRAME--------------------------------------------------------------
1776 // Definition of frame structure and management information.
1777 //
1778 //  S T A C K   L A Y O U T    Allocators stack-slot number
1779 //                             |   (to get allocators register number
1780 //  G  Owned by    |        |  v    add VMRegImpl::stack0)
1781 //  r   CALLER     |        |
1782 //  o     |        +--------+      pad to even-align allocators stack-slot
1783 //  w     V        |  pad0  |        numbers; owned by CALLER
1784 //  t   -----------+--------+----> Matcher::_in_arg_limit, unaligned
1785 //  h     ^        |   in   |  5
1786 //        |        |  args  |  4   Holes in incoming args owned by SELF
1787 //  |     |        |        |  3
1788 //  |     |        +--------+
1789 //  V     |        | old out|      Empty on Intel, window on Sparc
1790 //        |    old |preserve|      Must be even aligned.
1791 //        |     SP-+--------+----> Matcher::_old_SP, 8 (or 16 in LP64)-byte aligned
1792 //        |        |   in   |  3   area for Intel ret address
1793 //     Owned by    |preserve|      Empty on Sparc.
1794 //       SELF      +--------+
1795 //        |        |  pad2  |  2   pad to align old SP
1796 //        |        +--------+  1
1797 //        |        | locks  |  0
1798 //        |        +--------+----> VMRegImpl::stack0, 8 (or 16 in LP64)-byte aligned
1799 //        |        |  pad1  | 11   pad to align new SP
1800 //        |        +--------+
1801 //        |        |        | 10
1802 //        |        | spills |  9   spills
1803 //        V        |        |  8   (pad0 slot for callee)
1804 //      -----------+--------+----> Matcher::_out_arg_limit, unaligned
1805 //        ^        |  out   |  7
1806 //        |        |  args  |  6   Holes in outgoing args owned by CALLEE
1807 //     Owned by    +--------+
1808 //      CALLEE     | new out|  6   Empty on Intel, window on Sparc
1809 //        |    new |preserve|      Must be even-aligned.
1810 //        |     SP-+--------+----> Matcher::_new_SP, even aligned
1811 //        |        |        |
1812 //
1813 // Note 1: Only region 8-11 is determined by the allocator.  Region 0-5 is
1814 //         known from SELF's arguments and the Java calling convention.
1815 //         Region 6-7 is determined per call site.
1816 // Note 2: If the calling convention leaves holes in the incoming argument
1817 //         area, those holes are owned by SELF.  Holes in the outgoing area
1818 //         are owned by the CALLEE.  Holes should not be nessecary in the
1819 //         incoming area, as the Java calling convention is completely under
1820 //         the control of the AD file.  Doubles can be sorted and packed to
1821 //         avoid holes.  Holes in the outgoing arguments may be nessecary for
1822 //         varargs C calling conventions.
1823 // Note 3: Region 0-3 is even aligned, with pad2 as needed.  Region 3-5 is
1824 //         even aligned with pad0 as needed.
1825 //         Region 6 is even aligned.  Region 6-7 is NOT even aligned;
1826 //         region 6-11 is even aligned; it may be padded out more so that
1827 //         the region from SP to FP meets the minimum stack alignment.
1828 
1829 frame %{
1830   // What direction does stack grow in (assumed to be same for native & Java)
1831   stack_direction(TOWARDS_LOW);
1832 
1833   // These two registers define part of the calling convention
1834   // between compiled code and the interpreter.
1835   inline_cache_reg(R_Ricklass);          // Inline Cache Register or Method* for I2C
1836   interpreter_method_oop_reg(R_Rmethod); // Method Oop Register when calling interpreter
1837 
1838   // Optional: name the operand used by cisc-spilling to access [stack_pointer + offset]
1839   cisc_spilling_operand_name(indOffset);
1840 
1841   // Number of stack slots consumed by a Monitor enter
1842   sync_stack_slots(1 * VMRegImpl::slots_per_word);
1843 
1844   // Compiled code's Frame Pointer
1845 #ifdef AARCH64
1846   frame_pointer(R_SP);
1847 #else
1848   frame_pointer(R_R13);
1849 #endif
1850 
1851   // Stack alignment requirement
1852   stack_alignment(StackAlignmentInBytes);
1853   //  LP64: Alignment size in bytes (128-bit -> 16 bytes)
1854   // !LP64: Alignment size in bytes (64-bit  ->  8 bytes)
1855 
1856   // Number of stack slots between incoming argument block and the start of
1857   // a new frame.  The PROLOG must add this many slots to the stack.  The
1858   // EPILOG must remove this many slots.
1859   // FP + LR
1860   in_preserve_stack_slots(2 * VMRegImpl::slots_per_word);
1861 
1862   // Number of outgoing stack slots killed above the out_preserve_stack_slots
1863   // for calls to C.  Supports the var-args backing area for register parms.
1864   // ADLC doesn't support parsing expressions, so I folded the math by hand.
1865   varargs_C_out_slots_killed( 0);
1866 
1867   // The after-PROLOG location of the return address.  Location of
1868   // return address specifies a type (REG or STACK) and a number
1869   // representing the register number (i.e. - use a register name) or
1870   // stack slot.
1871   // Ret Addr is on stack in slot 0 if no locks or verification or alignment.
1872   // Otherwise, it is above the locks and verification slot and alignment word
1873   return_addr(STACK - 1*VMRegImpl::slots_per_word +
1874               round_to((Compile::current()->in_preserve_stack_slots() +
1875                         Compile::current()->fixed_slots()),
1876                        stack_alignment_in_slots()));
1877 
1878   // Body of function which returns an OptoRegs array locating
1879   // arguments either in registers or in stack slots for calling
1880   // java
1881   calling_convention %{
1882     (void) SharedRuntime::java_calling_convention(sig_bt, regs, length, is_outgoing);
1883 
1884   %}
1885 
1886   // Body of function which returns an OptoRegs array locating
1887   // arguments either in registers or in stack slots for callin
1888   // C.
1889   c_calling_convention %{
1890     // This is obviously always outgoing
1891     (void) SharedRuntime::c_calling_convention(sig_bt, regs, /*regs2=*/NULL, length);
1892   %}
1893 
1894   // Location of compiled Java return values.  Same as C
1895   return_value %{
1896     return c2::return_value(ideal_reg);
1897   %}
1898 
1899 %}
1900 
1901 //----------ATTRIBUTES---------------------------------------------------------
1902 //----------Instruction Attributes---------------------------------------------
1903 ins_attrib ins_cost(DEFAULT_COST); // Required cost attribute
1904 ins_attrib ins_size(32);           // Required size attribute (in bits)
1905 ins_attrib ins_short_branch(0);    // Required flag: is this instruction a
1906                                    // non-matching short branch variant of some
1907                                                             // long branch?
1908 
1909 //----------OPERANDS-----------------------------------------------------------
1910 // Operand definitions must precede instruction definitions for correct parsing
1911 // in the ADLC because operands constitute user defined types which are used in
1912 // instruction definitions.
1913 
1914 //----------Simple Operands----------------------------------------------------
1915 // Immediate Operands
1916 // Integer Immediate: 32-bit
1917 operand immI() %{
1918   match(ConI);
1919 
1920   op_cost(0);
1921   // formats are generated automatically for constants and base registers
1922   format %{ %}
1923   interface(CONST_INTER);
1924 %}
1925 
1926 // Integer Immediate: 8-bit unsigned - for VMOV
1927 operand immU8() %{
1928   predicate(0 <= n->get_int() && (n->get_int() <= 255));
1929   match(ConI);
1930   op_cost(0);
1931 
1932   format %{ %}
1933   interface(CONST_INTER);
1934 %}
1935 
1936 // Integer Immediate: 16-bit
1937 operand immI16() %{
1938   predicate((n->get_int() >> 16) == 0 && VM_Version::supports_movw());
1939   match(ConI);
1940   op_cost(0);
1941 
1942   format %{ %}
1943   interface(CONST_INTER);
1944 %}
1945 
1946 #ifndef AARCH64
1947 // Integer Immediate: offset for half and double word loads and stores
1948 operand immIHD() %{
1949   predicate(is_memoryHD(n->get_int()));
1950   match(ConI);
1951   op_cost(0);
1952   format %{ %}
1953   interface(CONST_INTER);
1954 %}
1955 
1956 // Integer Immediate: offset for fp loads and stores
1957 operand immIFP() %{
1958   predicate(is_memoryfp(n->get_int()) && ((n->get_int() & 3) == 0));
1959   match(ConI);
1960   op_cost(0);
1961 
1962   format %{ %}
1963   interface(CONST_INTER);
1964 %}
1965 #endif
1966 
1967 // Valid scale values for addressing modes and shifts
1968 operand immU5() %{
1969   predicate(0 <= n->get_int() && (n->get_int() <= 31));
1970   match(ConI);
1971   op_cost(0);
1972 
1973   format %{ %}
1974   interface(CONST_INTER);
1975 %}
1976 
1977 // Integer Immediate: 6-bit
1978 operand immU6Big() %{
1979   predicate(n->get_int() >= 32 && n->get_int() <= 63);
1980   match(ConI);
1981   op_cost(0);
1982   format %{ %}
1983   interface(CONST_INTER);
1984 %}
1985 
1986 // Integer Immediate: 0-bit
1987 operand immI0() %{
1988   predicate(n->get_int() == 0);
1989   match(ConI);
1990   op_cost(0);
1991 
1992   format %{ %}
1993   interface(CONST_INTER);
1994 %}
1995 
1996 // Integer Immediate: the value 1
1997 operand immI_1() %{
1998   predicate(n->get_int() == 1);
1999   match(ConI);
2000   op_cost(0);
2001 
2002   format %{ %}
2003   interface(CONST_INTER);
2004 %}
2005 
2006 // Integer Immediate: the value 2
2007 operand immI_2() %{
2008   predicate(n->get_int() == 2);
2009   match(ConI);
2010   op_cost(0);
2011 
2012   format %{ %}
2013   interface(CONST_INTER);
2014 %}
2015 
2016 // Integer Immediate: the value 3
2017 operand immI_3() %{
2018   predicate(n->get_int() == 3);
2019   match(ConI);
2020   op_cost(0);
2021 
2022   format %{ %}
2023   interface(CONST_INTER);
2024 %}
2025 
2026 // Integer Immediate: the value 4
2027 operand immI_4() %{
2028   predicate(n->get_int() == 4);
2029   match(ConI);
2030   op_cost(0);
2031 
2032   format %{ %}
2033   interface(CONST_INTER);
2034 %}
2035 
2036 // Integer Immediate: the value 8
2037 operand immI_8() %{
2038   predicate(n->get_int() == 8);
2039   match(ConI);
2040   op_cost(0);
2041 
2042   format %{ %}
2043   interface(CONST_INTER);
2044 %}
2045 
2046 // Int Immediate non-negative
2047 operand immU31()
2048 %{
2049   predicate(n->get_int() >= 0);
2050   match(ConI);
2051 
2052   op_cost(0);
2053   format %{ %}
2054   interface(CONST_INTER);
2055 %}
2056 
2057 // Integer Immediate: the values 32-63
2058 operand immI_32_63() %{
2059   predicate(n->get_int() >= 32 && n->get_int() <= 63);
2060   match(ConI);
2061   op_cost(0);
2062 
2063   format %{ %}
2064   interface(CONST_INTER);
2065 %}
2066 
2067 // Immediates for special shifts (sign extend)
2068 
2069 // Integer Immediate: the value 16
2070 operand immI_16() %{
2071   predicate(n->get_int() == 16);
2072   match(ConI);
2073   op_cost(0);
2074 
2075   format %{ %}
2076   interface(CONST_INTER);
2077 %}
2078 
2079 // Integer Immediate: the value 24
2080 operand immI_24() %{
2081   predicate(n->get_int() == 24);
2082   match(ConI);
2083   op_cost(0);
2084 
2085   format %{ %}
2086   interface(CONST_INTER);
2087 %}
2088 
2089 // Integer Immediate: the value 255
2090 operand immI_255() %{
2091   predicate( n->get_int() == 255 );
2092   match(ConI);
2093   op_cost(0);
2094 
2095   format %{ %}
2096   interface(CONST_INTER);
2097 %}
2098 
2099 // Integer Immediate: the value 65535
2100 operand immI_65535() %{
2101   predicate(n->get_int() == 65535);
2102   match(ConI);
2103   op_cost(0);
2104 
2105   format %{ %}
2106   interface(CONST_INTER);
2107 %}
2108 
2109 // Integer Immediates for arithmetic instructions
2110 
2111 operand aimmI() %{
2112   predicate(is_aimm(n->get_int()));
2113   match(ConI);
2114   op_cost(0);
2115 
2116   format %{ %}
2117   interface(CONST_INTER);
2118 %}
2119 
2120 operand aimmIneg() %{
2121   predicate(is_aimm(-n->get_int()));
2122   match(ConI);
2123   op_cost(0);
2124 
2125   format %{ %}
2126   interface(CONST_INTER);
2127 %}
2128 
2129 operand aimmU31() %{
2130   predicate((0 <= n->get_int()) && is_aimm(n->get_int()));
2131   match(ConI);
2132   op_cost(0);
2133 
2134   format %{ %}
2135   interface(CONST_INTER);
2136 %}
2137 
2138 // Integer Immediates for logical instructions
2139 
2140 operand limmI() %{
2141   predicate(is_limmI(n->get_int()));
2142   match(ConI);
2143   op_cost(0);
2144 
2145   format %{ %}
2146   interface(CONST_INTER);
2147 %}
2148 
2149 operand limmIlow8() %{
2150   predicate(is_limmI_low(n->get_int(), 8));
2151   match(ConI);
2152   op_cost(0);
2153 
2154   format %{ %}
2155   interface(CONST_INTER);
2156 %}
2157 
2158 operand limmU31() %{
2159   predicate(0 <= n->get_int() && is_limmI(n->get_int()));
2160   match(ConI);
2161   op_cost(0);
2162 
2163   format %{ %}
2164   interface(CONST_INTER);
2165 %}
2166 
2167 operand limmIn() %{
2168   predicate(is_limmI(~n->get_int()));
2169   match(ConI);
2170   op_cost(0);
2171 
2172   format %{ %}
2173   interface(CONST_INTER);
2174 %}
2175 
2176 #ifdef AARCH64
2177 // Long Immediate: for logical instruction
2178 operand limmL() %{
2179   predicate(is_limmL(n->get_long()));
2180   match(ConL);
2181   op_cost(0);
2182 
2183   format %{ %}
2184   interface(CONST_INTER);
2185 %}
2186 
2187 operand limmLn() %{
2188   predicate(is_limmL(~n->get_long()));
2189   match(ConL);
2190   op_cost(0);
2191 
2192   format %{ %}
2193   interface(CONST_INTER);
2194 %}
2195 
2196 // Long Immediate: for arithmetic instruction
2197 operand aimmL() %{
2198   predicate(is_aimm(n->get_long()));
2199   match(ConL);
2200   op_cost(0);
2201 
2202   format %{ %}
2203   interface(CONST_INTER);
2204 %}
2205 
2206 operand aimmLneg() %{
2207   predicate(is_aimm(-n->get_long()));
2208   match(ConL);
2209   op_cost(0);
2210 
2211   format %{ %}
2212   interface(CONST_INTER);
2213 %}
2214 #endif // AARCH64
2215 
2216 // Long Immediate: the value FF
2217 operand immL_FF() %{
2218   predicate( n->get_long() == 0xFFL );
2219   match(ConL);
2220   op_cost(0);
2221 
2222   format %{ %}
2223   interface(CONST_INTER);
2224 %}
2225 
2226 // Long Immediate: the value FFFF
2227 operand immL_FFFF() %{
2228   predicate( n->get_long() == 0xFFFFL );
2229   match(ConL);
2230   op_cost(0);
2231 
2232   format %{ %}
2233   interface(CONST_INTER);
2234 %}
2235 
2236 // Pointer Immediate: 32 or 64-bit
2237 operand immP() %{
2238   match(ConP);
2239 
2240   op_cost(5);
2241   // formats are generated automatically for constants and base registers
2242   format %{ %}
2243   interface(CONST_INTER);
2244 %}
2245 
2246 operand immP0() %{
2247   predicate(n->get_ptr() == 0);
2248   match(ConP);
2249   op_cost(0);
2250 
2251   format %{ %}
2252   interface(CONST_INTER);
2253 %}
2254 
2255 operand immP_poll() %{
2256   predicate(n->get_ptr() != 0 && n->get_ptr() == (intptr_t)os::get_polling_page());
2257   match(ConP);
2258 
2259   // formats are generated automatically for constants and base registers
2260   format %{ %}
2261   interface(CONST_INTER);
2262 %}
2263 
2264 // Pointer Immediate
2265 operand immN()
2266 %{
2267   match(ConN);
2268 
2269   op_cost(10);
2270   format %{ %}
2271   interface(CONST_INTER);
2272 %}
2273 
2274 operand immNKlass()
2275 %{
2276   match(ConNKlass);
2277 
2278   op_cost(10);
2279   format %{ %}
2280   interface(CONST_INTER);
2281 %}
2282 
2283 // NULL Pointer Immediate
2284 operand immN0()
2285 %{
2286   predicate(n->get_narrowcon() == 0);
2287   match(ConN);
2288 
2289   op_cost(0);
2290   format %{ %}
2291   interface(CONST_INTER);
2292 %}
2293 
2294 operand immL() %{
2295   match(ConL);
2296   op_cost(40);
2297   // formats are generated automatically for constants and base registers
2298   format %{ %}
2299   interface(CONST_INTER);
2300 %}
2301 
2302 operand immL0() %{
2303   predicate(n->get_long() == 0L);
2304   match(ConL);
2305   op_cost(0);
2306   // formats are generated automatically for constants and base registers
2307   format %{ %}
2308   interface(CONST_INTER);
2309 %}
2310 
2311 // Long Immediate: 16-bit
2312 operand immL16() %{
2313   predicate(n->get_long() >= 0 && n->get_long() < (1<<16)  && VM_Version::supports_movw());
2314   match(ConL);
2315   op_cost(0);
2316 
2317   format %{ %}
2318   interface(CONST_INTER);
2319 %}
2320 
2321 // Long Immediate: low 32-bit mask
2322 operand immL_32bits() %{
2323   predicate(n->get_long() == 0xFFFFFFFFL);
2324   match(ConL);
2325   op_cost(0);
2326 
2327   format %{ %}
2328   interface(CONST_INTER);
2329 %}
2330 
2331 // Double Immediate
2332 operand immD() %{
2333   match(ConD);
2334 
2335   op_cost(40);
2336   format %{ %}
2337   interface(CONST_INTER);
2338 %}
2339 
2340 // Double Immediate: +0.0d.
2341 operand immD0() %{
2342   predicate(jlong_cast(n->getd()) == 0);
2343 
2344   match(ConD);
2345   op_cost(0);
2346   format %{ %}
2347   interface(CONST_INTER);
2348 %}
2349 
2350 operand imm8D() %{
2351   predicate(Assembler::double_num(n->getd()).can_be_imm8());
2352   match(ConD);
2353 
2354   op_cost(0);
2355   format %{ %}
2356   interface(CONST_INTER);
2357 %}
2358 
2359 // Float Immediate
2360 operand immF() %{
2361   match(ConF);
2362 
2363   op_cost(20);
2364   format %{ %}
2365   interface(CONST_INTER);
2366 %}
2367 
2368 // Float Immediate: +0.0f
2369 operand immF0() %{
2370   predicate(jint_cast(n->getf()) == 0);
2371   match(ConF);
2372 
2373   op_cost(0);
2374   format %{ %}
2375   interface(CONST_INTER);
2376 %}
2377 
2378 // Float Immediate: encoded as 8 bits
2379 operand imm8F() %{
2380   predicate(Assembler::float_num(n->getf()).can_be_imm8());
2381   match(ConF);
2382 
2383   op_cost(0);
2384   format %{ %}
2385   interface(CONST_INTER);
2386 %}
2387 
2388 // Integer Register Operands
2389 // Integer Register
2390 operand iRegI() %{
2391   constraint(ALLOC_IN_RC(int_reg));
2392   match(RegI);
2393   match(R0RegI);
2394   match(R1RegI);
2395   match(R2RegI);
2396   match(R3RegI);
2397 #ifdef AARCH64
2398   match(ZRRegI);
2399 #else
2400   match(R12RegI);
2401 #endif
2402 
2403   format %{ %}
2404   interface(REG_INTER);
2405 %}
2406 
2407 // Pointer Register
2408 operand iRegP() %{
2409   constraint(ALLOC_IN_RC(ptr_reg));
2410   match(RegP);
2411   match(R0RegP);
2412   match(R1RegP);
2413   match(R2RegP);
2414   match(RExceptionRegP);
2415   match(R8RegP);
2416   match(R9RegP);
2417   match(RthreadRegP); // FIXME: move to sp_ptr_RegP?
2418   match(R12RegP);
2419   match(LRRegP);
2420 
2421   match(sp_ptr_RegP);
2422   match(store_ptr_RegP);
2423 
2424   format %{ %}
2425   interface(REG_INTER);
2426 %}
2427 
2428 // GPRs + Rthread + SP
2429 operand sp_ptr_RegP() %{
2430   constraint(ALLOC_IN_RC(sp_ptr_reg));
2431   match(RegP);
2432   match(iRegP);
2433   match(SPRegP); // FIXME: check cost
2434 
2435   format %{ %}
2436   interface(REG_INTER);
2437 %}
2438 
2439 #ifdef AARCH64
2440 // Like sp_ptr_reg, but exclude regs (Aarch64 SP) that can't be
2441 // stored directly.  Includes ZR, so can't be used as a destination.
2442 operand store_ptr_RegP() %{
2443   constraint(ALLOC_IN_RC(store_ptr_reg));
2444   match(RegP);
2445   match(iRegP);
2446   match(ZRRegP);
2447 
2448   format %{ %}
2449   interface(REG_INTER);
2450 %}
2451 
2452 operand store_RegI() %{
2453   constraint(ALLOC_IN_RC(store_reg));
2454   match(RegI);
2455   match(iRegI);
2456   match(ZRRegI);
2457 
2458   format %{ %}
2459   interface(REG_INTER);
2460 %}
2461 
2462 operand store_RegL() %{
2463   constraint(ALLOC_IN_RC(store_ptr_reg));
2464   match(RegL);
2465   match(iRegL);
2466   match(ZRRegL);
2467 
2468   format %{ %}
2469   interface(REG_INTER);
2470 %}
2471 
2472 operand store_RegN() %{
2473   constraint(ALLOC_IN_RC(store_reg));
2474   match(RegN);
2475   match(iRegN);
2476   match(ZRRegN);
2477 
2478   format %{ %}
2479   interface(REG_INTER);
2480 %}
2481 #endif
2482 
2483 operand R0RegP() %{
2484   constraint(ALLOC_IN_RC(R0_regP));
2485   match(iRegP);
2486 
2487   format %{ %}
2488   interface(REG_INTER);
2489 %}
2490 
2491 operand R1RegP() %{
2492   constraint(ALLOC_IN_RC(R1_regP));
2493   match(iRegP);
2494 
2495   format %{ %}
2496   interface(REG_INTER);
2497 %}
2498 
2499 operand R2RegP() %{
2500   constraint(ALLOC_IN_RC(R2_regP));
2501   match(iRegP);
2502 
2503   format %{ %}
2504   interface(REG_INTER);
2505 %}
2506 
2507 operand RExceptionRegP() %{
2508   constraint(ALLOC_IN_RC(Rexception_regP));
2509   match(iRegP);
2510 
2511   format %{ %}
2512   interface(REG_INTER);
2513 %}
2514 
2515 operand RthreadRegP() %{
2516   constraint(ALLOC_IN_RC(Rthread_regP));
2517   match(iRegP);
2518 
2519   format %{ %}
2520   interface(REG_INTER);
2521 %}
2522 
2523 operand IPRegP() %{
2524   constraint(ALLOC_IN_RC(IP_regP));
2525   match(iRegP);
2526 
2527   format %{ %}
2528   interface(REG_INTER);
2529 %}
2530 
2531 operand LRRegP() %{
2532   constraint(ALLOC_IN_RC(LR_regP));
2533   match(iRegP);
2534 
2535   format %{ %}
2536   interface(REG_INTER);
2537 %}
2538 
2539 operand R0RegI() %{
2540   constraint(ALLOC_IN_RC(R0_regI));
2541   match(iRegI);
2542 
2543   format %{ %}
2544   interface(REG_INTER);
2545 %}
2546 
2547 operand R1RegI() %{
2548   constraint(ALLOC_IN_RC(R1_regI));
2549   match(iRegI);
2550 
2551   format %{ %}
2552   interface(REG_INTER);
2553 %}
2554 
2555 operand R2RegI() %{
2556   constraint(ALLOC_IN_RC(R2_regI));
2557   match(iRegI);
2558 
2559   format %{ %}
2560   interface(REG_INTER);
2561 %}
2562 
2563 operand R3RegI() %{
2564   constraint(ALLOC_IN_RC(R3_regI));
2565   match(iRegI);
2566 
2567   format %{ %}
2568   interface(REG_INTER);
2569 %}
2570 
2571 #ifndef AARCH64
2572 operand R12RegI() %{
2573   constraint(ALLOC_IN_RC(R12_regI));
2574   match(iRegI);
2575 
2576   format %{ %}
2577   interface(REG_INTER);
2578 %}
2579 #endif
2580 
2581 // Long Register
2582 operand iRegL() %{
2583   constraint(ALLOC_IN_RC(long_reg));
2584   match(RegL);
2585 #ifdef AARCH64
2586   match(iRegLd);
2587 #else
2588   match(R0R1RegL);
2589   match(R2R3RegL);
2590 #endif
2591 //match(iRegLex);
2592 
2593   format %{ %}
2594   interface(REG_INTER);
2595 %}
2596 
2597 operand iRegLd() %{
2598   constraint(ALLOC_IN_RC(long_reg_align));
2599   match(iRegL); // FIXME: allows unaligned R11/R12?
2600 
2601   format %{ %}
2602   interface(REG_INTER);
2603 %}
2604 
2605 #ifndef AARCH64
2606 // first long arg, or return value
2607 operand R0R1RegL() %{
2608   constraint(ALLOC_IN_RC(R0R1_regL));
2609   match(iRegL);
2610 
2611   format %{ %}
2612   interface(REG_INTER);
2613 %}
2614 
2615 operand R2R3RegL() %{
2616   constraint(ALLOC_IN_RC(R2R3_regL));
2617   match(iRegL);
2618 
2619   format %{ %}
2620   interface(REG_INTER);
2621 %}
2622 #endif
2623 
2624 // Condition Code Flag Register
2625 operand flagsReg() %{
2626   constraint(ALLOC_IN_RC(int_flags));
2627   match(RegFlags);
2628 
2629   format %{ "apsr" %}
2630   interface(REG_INTER);
2631 %}
2632 
2633 // Result of compare to 0 (TST)
2634 operand flagsReg_EQNELTGE() %{
2635   constraint(ALLOC_IN_RC(int_flags));
2636   match(RegFlags);
2637 
2638   format %{ "apsr_EQNELTGE" %}
2639   interface(REG_INTER);
2640 %}
2641 
2642 // Condition Code Register, unsigned comparisons.
2643 operand flagsRegU() %{
2644   constraint(ALLOC_IN_RC(int_flags));
2645   match(RegFlags);
2646 #ifdef TODO
2647   match(RegFlagsP);
2648 #endif
2649 
2650   format %{ "apsr_U" %}
2651   interface(REG_INTER);
2652 %}
2653 
2654 // Condition Code Register, pointer comparisons.
2655 operand flagsRegP() %{
2656   constraint(ALLOC_IN_RC(int_flags));
2657   match(RegFlags);
2658 
2659   format %{ "apsr_P" %}
2660   interface(REG_INTER);
2661 %}
2662 
2663 // Condition Code Register, long comparisons.
2664 #ifndef AARCH64
2665 operand flagsRegL_LTGE() %{
2666   constraint(ALLOC_IN_RC(int_flags));
2667   match(RegFlags);
2668 
2669   format %{ "apsr_L_LTGE" %}
2670   interface(REG_INTER);
2671 %}
2672 
2673 operand flagsRegL_EQNE() %{
2674   constraint(ALLOC_IN_RC(int_flags));
2675   match(RegFlags);
2676 
2677   format %{ "apsr_L_EQNE" %}
2678   interface(REG_INTER);
2679 %}
2680 
2681 operand flagsRegL_LEGT() %{
2682   constraint(ALLOC_IN_RC(int_flags));
2683   match(RegFlags);
2684 
2685   format %{ "apsr_L_LEGT" %}
2686   interface(REG_INTER);
2687 %}
2688 #endif
2689 
2690 // Condition Code Register, floating comparisons, unordered same as "less".
2691 operand flagsRegF() %{
2692   constraint(ALLOC_IN_RC(float_flags));
2693   match(RegFlags);
2694 
2695   format %{ "fpscr_F" %}
2696   interface(REG_INTER);
2697 %}
2698 
2699 // Vectors
2700 operand vecD() %{
2701   constraint(ALLOC_IN_RC(actual_dflt_reg));
2702   match(VecD);
2703 
2704   format %{ %}
2705   interface(REG_INTER);
2706 %}
2707 
2708 operand vecX() %{
2709   constraint(ALLOC_IN_RC(vectorx_reg));
2710   match(VecX);
2711 
2712   format %{ %}
2713   interface(REG_INTER);
2714 %}
2715 
2716 operand regD() %{
2717   constraint(ALLOC_IN_RC(actual_dflt_reg));
2718   match(RegD);
2719   match(regD_low);
2720 
2721   format %{ %}
2722   interface(REG_INTER);
2723 %}
2724 
2725 operand regF() %{
2726   constraint(ALLOC_IN_RC(sflt_reg));
2727   match(RegF);
2728 
2729   format %{ %}
2730   interface(REG_INTER);
2731 %}
2732 
2733 operand regD_low() %{
2734   constraint(ALLOC_IN_RC(dflt_low_reg));
2735   match(RegD);
2736 
2737   format %{ %}
2738   interface(REG_INTER);
2739 %}
2740 
2741 // Special Registers
2742 
2743 // Method Register
2744 operand inline_cache_regP(iRegP reg) %{
2745   constraint(ALLOC_IN_RC(Ricklass_regP));
2746   match(reg);
2747   format %{ %}
2748   interface(REG_INTER);
2749 %}
2750 
2751 operand interpreter_method_oop_regP(iRegP reg) %{
2752   constraint(ALLOC_IN_RC(Rmethod_regP));
2753   match(reg);
2754   format %{ %}
2755   interface(REG_INTER);
2756 %}
2757 
2758 
2759 //----------Complex Operands---------------------------------------------------
2760 // Indirect Memory Reference
2761 operand indirect(sp_ptr_RegP reg) %{
2762   constraint(ALLOC_IN_RC(sp_ptr_reg));
2763   match(reg);
2764 
2765   op_cost(100);
2766   format %{ "[$reg]" %}
2767   interface(MEMORY_INTER) %{
2768     base($reg);
2769 #ifdef AARCH64
2770     index(0xff); // 0xff => no index
2771 #else
2772     index(0xf); // PC => no index
2773 #endif
2774     scale(0x0);
2775     disp(0x0);
2776   %}
2777 %}
2778 
2779 #ifdef AARCH64
2780 // Indirect with scaled*1 uimm12 offset
2781 operand indOffsetU12ScaleB(sp_ptr_RegP reg, immUL12 offset) %{
2782   constraint(ALLOC_IN_RC(sp_ptr_reg));
2783   match(AddP reg offset);
2784 
2785   op_cost(100);
2786   format %{ "[$reg + $offset]" %}
2787   interface(MEMORY_INTER) %{
2788     base($reg);
2789 #ifdef AARCH64
2790     index(0xff); // 0xff => no index
2791 #else
2792     index(0xf); // PC => no index
2793 #endif
2794     scale(0x0);
2795     disp($offset);
2796   %}
2797 %}
2798 
2799 // Indirect with scaled*2 uimm12 offset
2800 operand indOffsetU12ScaleS(sp_ptr_RegP reg, immUL12x2 offset) %{
2801   constraint(ALLOC_IN_RC(sp_ptr_reg));
2802   match(AddP reg offset);
2803 
2804   op_cost(100);
2805   format %{ "[$reg + $offset]" %}
2806   interface(MEMORY_INTER) %{
2807     base($reg);
2808 #ifdef AARCH64
2809     index(0xff); // 0xff => no index
2810 #else
2811     index(0xf); // PC => no index
2812 #endif
2813     scale(0x0);
2814     disp($offset);
2815   %}
2816 %}
2817 
2818 // Indirect with scaled*4 uimm12 offset
2819 operand indOffsetU12ScaleI(sp_ptr_RegP reg, immUL12x4 offset) %{
2820   constraint(ALLOC_IN_RC(sp_ptr_reg));
2821   match(AddP reg offset);
2822 
2823   op_cost(100);
2824   format %{ "[$reg + $offset]" %}
2825   interface(MEMORY_INTER) %{
2826     base($reg);
2827 #ifdef AARCH64
2828     index(0xff); // 0xff => no index
2829 #else
2830     index(0xf); // PC => no index
2831 #endif
2832     scale(0x0);
2833     disp($offset);
2834   %}
2835 %}
2836 
2837 // Indirect with scaled*8 uimm12 offset
2838 operand indOffsetU12ScaleL(sp_ptr_RegP reg, immUL12x8 offset) %{
2839   constraint(ALLOC_IN_RC(sp_ptr_reg));
2840   match(AddP reg offset);
2841 
2842   op_cost(100);
2843   format %{ "[$reg + $offset]" %}
2844   interface(MEMORY_INTER) %{
2845     base($reg);
2846 #ifdef AARCH64
2847     index(0xff); // 0xff => no index
2848 #else
2849     index(0xf); // PC => no index
2850 #endif
2851     scale(0x0);
2852     disp($offset);
2853   %}
2854 %}
2855 
2856 // Indirect with scaled*16 uimm12 offset
2857 operand indOffsetU12ScaleQ(sp_ptr_RegP reg, immUL12x16 offset) %{
2858   constraint(ALLOC_IN_RC(sp_ptr_reg));
2859   match(AddP reg offset);
2860 
2861   op_cost(100);
2862   format %{ "[$reg + $offset]" %}
2863   interface(MEMORY_INTER) %{
2864     base($reg);
2865 #ifdef AARCH64
2866     index(0xff); // 0xff => no index
2867 #else
2868     index(0xf); // PC => no index
2869 #endif
2870     scale(0x0);
2871     disp($offset);
2872   %}
2873 %}
2874 
2875 #else // ! AARCH64
2876 
2877 // Indirect with Offset in ]-4096, 4096[
2878 operand indOffset12(sp_ptr_RegP reg, immI12 offset) %{
2879   constraint(ALLOC_IN_RC(sp_ptr_reg));
2880   match(AddP reg offset);
2881 
2882   op_cost(100);
2883   format %{ "[$reg + $offset]" %}
2884   interface(MEMORY_INTER) %{
2885     base($reg);
2886 #ifdef AARCH64
2887     index(0xff); // 0xff => no index
2888 #else
2889     index(0xf); // PC => no index
2890 #endif
2891     scale(0x0);
2892     disp($offset);
2893   %}
2894 %}
2895 
2896 // Indirect with offset for float load/store
2897 operand indOffsetFP(sp_ptr_RegP reg, immIFP offset) %{
2898   constraint(ALLOC_IN_RC(sp_ptr_reg));
2899   match(AddP reg offset);
2900 
2901   op_cost(100);
2902   format %{ "[$reg + $offset]" %}
2903   interface(MEMORY_INTER) %{
2904     base($reg);
2905 #ifdef AARCH64
2906     index(0xff); // 0xff => no index
2907 #else
2908     index(0xf); // PC => no index
2909 #endif
2910     scale(0x0);
2911     disp($offset);
2912   %}
2913 %}
2914 
2915 // Indirect with Offset for half and double words
2916 operand indOffsetHD(sp_ptr_RegP reg, immIHD offset) %{
2917   constraint(ALLOC_IN_RC(sp_ptr_reg));
2918   match(AddP reg offset);
2919 
2920   op_cost(100);
2921   format %{ "[$reg + $offset]" %}
2922   interface(MEMORY_INTER) %{
2923     base($reg);
2924 #ifdef AARCH64
2925     index(0xff); // 0xff => no index
2926 #else
2927     index(0xf); // PC => no index
2928 #endif
2929     scale(0x0);
2930     disp($offset);
2931   %}
2932 %}
2933 
2934 // Indirect with Offset and Offset+4 in ]-1024, 1024[
2935 operand indOffsetFPx2(sp_ptr_RegP reg, immX10x2 offset) %{
2936   constraint(ALLOC_IN_RC(sp_ptr_reg));
2937   match(AddP reg offset);
2938 
2939   op_cost(100);
2940   format %{ "[$reg + $offset]" %}
2941   interface(MEMORY_INTER) %{
2942     base($reg);
2943 #ifdef AARCH64
2944     index(0xff); // 0xff => no index
2945 #else
2946     index(0xf); // PC => no index
2947 #endif
2948     scale(0x0);
2949     disp($offset);
2950   %}
2951 %}
2952 
2953 // Indirect with Offset and Offset+4 in ]-4096, 4096[
2954 operand indOffset12x2(sp_ptr_RegP reg, immI12x2 offset) %{
2955   constraint(ALLOC_IN_RC(sp_ptr_reg));
2956   match(AddP reg offset);
2957 
2958   op_cost(100);
2959   format %{ "[$reg + $offset]" %}
2960   interface(MEMORY_INTER) %{
2961     base($reg);
2962 #ifdef AARCH64
2963     index(0xff); // 0xff => no index
2964 #else
2965     index(0xf); // PC => no index
2966 #endif
2967     scale(0x0);
2968     disp($offset);
2969   %}
2970 %}
2971 #endif // !AARCH64
2972 
2973 // Indirect with Register Index
2974 operand indIndex(iRegP addr, iRegX index) %{
2975   constraint(ALLOC_IN_RC(ptr_reg));
2976   match(AddP addr index);
2977 
2978   op_cost(100);
2979   format %{ "[$addr + $index]" %}
2980   interface(MEMORY_INTER) %{
2981     base($addr);
2982     index($index);
2983     scale(0x0);
2984     disp(0x0);
2985   %}
2986 %}
2987 
2988 #ifdef AARCH64
2989 // Indirect Memory Times Scale Plus Index Register
2990 operand indIndexScaleS(iRegP addr, iRegX index, immI_1 scale) %{
2991   constraint(ALLOC_IN_RC(ptr_reg));
2992   match(AddP addr (LShiftX index scale));
2993 
2994   op_cost(100);
2995   format %{"[$addr + $index << $scale]" %}
2996   interface(MEMORY_INTER) %{
2997     base($addr);
2998     index($index);
2999     scale($scale);
3000     disp(0x0);
3001   %}
3002 %}
3003 
3004 // Indirect Memory Times Scale Plus 32-bit Index Register
3005 operand indIndexIScaleS(iRegP addr, iRegI index, immI_1 scale) %{
3006   constraint(ALLOC_IN_RC(ptr_reg));
3007   match(AddP addr (LShiftX (ConvI2L index) scale));
3008 
3009   op_cost(100);
3010   format %{"[$addr + $index.w << $scale]" %}
3011   interface(MEMORY_INTER) %{
3012     base($addr);
3013     index($index);
3014     scale($scale);
3015     disp(0x7fffffff); // sxtw
3016   %}
3017 %}
3018 
3019 // Indirect Memory Times Scale Plus Index Register
3020 operand indIndexScaleI(iRegP addr, iRegX index, immI_2 scale) %{
3021   constraint(ALLOC_IN_RC(ptr_reg));
3022   match(AddP addr (LShiftX index scale));
3023 
3024   op_cost(100);
3025   format %{"[$addr + $index << $scale]" %}
3026   interface(MEMORY_INTER) %{
3027     base($addr);
3028     index($index);
3029     scale($scale);
3030     disp(0x0);
3031   %}
3032 %}
3033 
3034 // Indirect Memory Times Scale Plus 32-bit Index Register
3035 operand indIndexIScaleI(iRegP addr, iRegI index, immI_2 scale) %{
3036   constraint(ALLOC_IN_RC(ptr_reg));
3037   match(AddP addr (LShiftX (ConvI2L index) scale));
3038 
3039   op_cost(100);
3040   format %{"[$addr + $index.w << $scale]" %}
3041   interface(MEMORY_INTER) %{
3042     base($addr);
3043     index($index);
3044     scale($scale);
3045     disp(0x7fffffff); // sxtw
3046   %}
3047 %}
3048 
3049 // Indirect Memory Times Scale Plus Index Register
3050 operand indIndexScaleL(iRegP addr, iRegX index, immI_3 scale) %{
3051   constraint(ALLOC_IN_RC(ptr_reg));
3052   match(AddP addr (LShiftX index scale));
3053 
3054   op_cost(100);
3055   format %{"[$addr + $index << $scale]" %}
3056   interface(MEMORY_INTER) %{
3057     base($addr);
3058     index($index);
3059     scale($scale);
3060     disp(0x0);
3061   %}
3062 %}
3063 
3064 // Indirect Memory Times Scale Plus 32-bit Index Register
3065 operand indIndexIScaleL(iRegP addr, iRegI index, immI_3 scale) %{
3066   constraint(ALLOC_IN_RC(ptr_reg));
3067   match(AddP addr (LShiftX (ConvI2L index) scale));
3068 
3069   op_cost(100);
3070   format %{"[$addr + $index.w << $scale]" %}
3071   interface(MEMORY_INTER) %{
3072     base($addr);
3073     index($index);
3074     scale($scale);
3075     disp(0x7fffffff); // sxtw
3076   %}
3077 %}
3078 
3079 // Indirect Memory Times Scale Plus Index Register
3080 operand indIndexScaleQ(iRegP addr, iRegX index, immI_4 scale) %{
3081   constraint(ALLOC_IN_RC(ptr_reg));
3082   match(AddP addr (LShiftX index scale));
3083 
3084   op_cost(100);
3085   format %{"[$addr + $index << $scale]" %}
3086   interface(MEMORY_INTER) %{
3087     base($addr);
3088     index($index);
3089     scale($scale);
3090     disp(0x0);
3091   %}
3092 %}
3093 
3094 // Indirect Memory Times Scale Plus 32-bit Index Register
3095 operand indIndexIScaleQ(iRegP addr, iRegI index, immI_4 scale) %{
3096   constraint(ALLOC_IN_RC(ptr_reg));
3097   match(AddP addr (LShiftX (ConvI2L index) scale));
3098 
3099   op_cost(100);
3100   format %{"[$addr + $index.w << $scale]" %}
3101   interface(MEMORY_INTER) %{
3102     base($addr);
3103     index($index);
3104     scale($scale);
3105     disp(0x7fffffff); // sxtw
3106   %}
3107 %}
3108 #else
3109 // Indirect Memory Times Scale Plus Index Register
3110 operand indIndexScale(iRegP addr, iRegX index, immU5 scale) %{
3111   constraint(ALLOC_IN_RC(ptr_reg));
3112   match(AddP addr (LShiftX index scale));
3113 
3114   op_cost(100);
3115   format %{"[$addr + $index << $scale]" %}
3116   interface(MEMORY_INTER) %{
3117     base($addr);
3118     index($index);
3119     scale($scale);
3120     disp(0x0);
3121   %}
3122 %}
3123 #endif
3124 
3125 // Operands for expressing Control Flow
3126 // NOTE:  Label is a predefined operand which should not be redefined in
3127 //        the AD file.  It is generically handled within the ADLC.
3128 
3129 //----------Conditional Branch Operands----------------------------------------
3130 // Comparison Op  - This is the operation of the comparison, and is limited to
3131 //                  the following set of codes:
3132 //                  L (<), LE (<=), G (>), GE (>=), E (==), NE (!=)
3133 //
3134 // Other attributes of the comparison, such as unsignedness, are specified
3135 // by the comparison instruction that sets a condition code flags register.
3136 // That result is represented by a flags operand whose subtype is appropriate
3137 // to the unsignedness (etc.) of the comparison.
3138 //
3139 // Later, the instruction which matches both the Comparison Op (a Bool) and
3140 // the flags (produced by the Cmp) specifies the coding of the comparison op
3141 // by matching a specific subtype of Bool operand below, such as cmpOpU.
3142 
3143 operand cmpOp() %{
3144   match(Bool);
3145 
3146   format %{ "" %}
3147   interface(COND_INTER) %{
3148     equal(0x0);
3149     not_equal(0x1);
3150     less(0xb);
3151     greater_equal(0xa);
3152     less_equal(0xd);
3153     greater(0xc);
3154     overflow(0x0); // unsupported/unimplemented
3155     no_overflow(0x0); // unsupported/unimplemented
3156   %}
3157 %}
3158 
3159 // integer comparison with 0, signed
3160 operand cmpOp0() %{
3161   match(Bool);
3162 
3163   format %{ "" %}
3164   interface(COND_INTER) %{
3165     equal(0x0);
3166     not_equal(0x1);
3167     less(0x4);
3168     greater_equal(0x5);
3169     less_equal(0xd); // unsupported
3170     greater(0xc); // unsupported
3171     overflow(0x0); // unsupported/unimplemented
3172     no_overflow(0x0); // unsupported/unimplemented
3173   %}
3174 %}
3175 
3176 // Comparison Op, unsigned
3177 operand cmpOpU() %{
3178   match(Bool);
3179 
3180   format %{ "u" %}
3181   interface(COND_INTER) %{
3182     equal(0x0);
3183     not_equal(0x1);
3184     less(0x3);
3185     greater_equal(0x2);
3186     less_equal(0x9);
3187     greater(0x8);
3188     overflow(0x0); // unsupported/unimplemented
3189     no_overflow(0x0); // unsupported/unimplemented
3190   %}
3191 %}
3192 
3193 // Comparison Op, pointer (same as unsigned)
3194 operand cmpOpP() %{
3195   match(Bool);
3196 
3197   format %{ "p" %}
3198   interface(COND_INTER) %{
3199     equal(0x0);
3200     not_equal(0x1);
3201     less(0x3);
3202     greater_equal(0x2);
3203     less_equal(0x9);
3204     greater(0x8);
3205     overflow(0x0); // unsupported/unimplemented
3206     no_overflow(0x0); // unsupported/unimplemented
3207   %}
3208 %}
3209 
3210 operand cmpOpL() %{
3211   match(Bool);
3212 
3213   format %{ "L" %}
3214   interface(COND_INTER) %{
3215     equal(0x0);
3216     not_equal(0x1);
3217     less(0xb);
3218     greater_equal(0xa);
3219     less_equal(0xd);
3220     greater(0xc);
3221     overflow(0x0); // unsupported/unimplemented
3222     no_overflow(0x0); // unsupported/unimplemented
3223   %}
3224 %}
3225 
3226 operand cmpOpL_commute() %{
3227   match(Bool);
3228 
3229   format %{ "L" %}
3230   interface(COND_INTER) %{
3231     equal(0x0);
3232     not_equal(0x1);
3233     less(0xc);
3234     greater_equal(0xd);
3235     less_equal(0xa);
3236     greater(0xb);
3237     overflow(0x0); // unsupported/unimplemented
3238     no_overflow(0x0); // unsupported/unimplemented
3239   %}
3240 %}
3241 
3242 //----------OPERAND CLASSES----------------------------------------------------
3243 // Operand Classes are groups of operands that are used to simplify
3244 // instruction definitions by not requiring the AD writer to specify separate
3245 // instructions for every form of operand when the instruction accepts
3246 // multiple operand types with the same basic encoding and format.  The classic
3247 // case of this is memory operands.
3248 #ifdef AARCH64
3249 opclass memoryB(indirect, indIndex, indOffsetU12ScaleB);
3250 opclass memoryS(indirect, indIndex, indIndexScaleS, indIndexIScaleS, indOffsetU12ScaleS);
3251 opclass memoryI(indirect, indIndex, indIndexScaleI, indIndexIScaleI, indOffsetU12ScaleI);
3252 opclass memoryL(indirect, indIndex, indIndexScaleL, indIndexIScaleL, indOffsetU12ScaleL);
3253 opclass memoryP(indirect, indIndex, indIndexScaleL, indIndexIScaleL, indOffsetU12ScaleL);
3254 opclass memoryQ(indirect, indIndex, indIndexScaleQ, indIndexIScaleQ, indOffsetU12ScaleQ);
3255 opclass memoryF(indirect, indIndex, indIndexScaleI, indIndexIScaleI, indOffsetU12ScaleI);
3256 opclass memoryD(indirect, indIndex, indIndexScaleL, indIndexIScaleL, indOffsetU12ScaleL);
3257 
3258 opclass memoryScaledS(indIndexScaleS, indIndexIScaleS);
3259 opclass memoryScaledI(indIndexScaleI, indIndexIScaleI);
3260 opclass memoryScaledL(indIndexScaleL, indIndexIScaleL);
3261 opclass memoryScaledP(indIndexScaleL, indIndexIScaleL);
3262 opclass memoryScaledQ(indIndexScaleQ, indIndexIScaleQ);
3263 opclass memoryScaledF(indIndexScaleI, indIndexIScaleI);
3264 opclass memoryScaledD(indIndexScaleL, indIndexIScaleL);
3265 // when ldrex/strex is used:
3266 opclass memoryex ( indirect );
3267 opclass indIndexMemory( indIndex );
3268 opclass memoryvld ( indirect /* , write back mode not implemented */ );
3269 
3270 #else
3271 
3272 opclass memoryI ( indirect, indOffset12, indIndex, indIndexScale );
3273 opclass memoryP ( indirect, indOffset12, indIndex, indIndexScale );
3274 opclass memoryF ( indirect, indOffsetFP );
3275 opclass memoryF2 ( indirect, indOffsetFPx2 );
3276 opclass memoryD ( indirect, indOffsetFP );
3277 opclass memoryfp( indirect, indOffsetFP );
3278 opclass memoryB ( indirect, indIndex, indOffsetHD );
3279 opclass memoryS ( indirect, indIndex, indOffsetHD );
3280 opclass memoryL ( indirect, indIndex, indOffsetHD );
3281 
3282 opclass memoryScaledI(indIndexScale);
3283 opclass memoryScaledP(indIndexScale);
3284 
3285 // when ldrex/strex is used:
3286 opclass memoryex ( indirect );
3287 opclass indIndexMemory( indIndex );
3288 opclass memorylong ( indirect, indOffset12x2 );
3289 opclass memoryvld ( indirect /* , write back mode not implemented */ );
3290 #endif
3291 
3292 //----------PIPELINE-----------------------------------------------------------
3293 pipeline %{
3294 
3295 //----------ATTRIBUTES---------------------------------------------------------
3296 attributes %{
3297   fixed_size_instructions;           // Fixed size instructions
3298   max_instructions_per_bundle = 4;   // Up to 4 instructions per bundle
3299   instruction_unit_size = 4;         // An instruction is 4 bytes long
3300   instruction_fetch_unit_size = 16;  // The processor fetches one line
3301   instruction_fetch_units = 1;       // of 16 bytes
3302 
3303   // List of nop instructions
3304   nops( Nop_A0, Nop_A1, Nop_MS, Nop_FA, Nop_BR );
3305 %}
3306 
3307 //----------RESOURCES----------------------------------------------------------
3308 // Resources are the functional units available to the machine
3309 resources(A0, A1, MS, BR, FA, FM, IDIV, FDIV, IALU = A0 | A1);
3310 
3311 //----------PIPELINE DESCRIPTION-----------------------------------------------
3312 // Pipeline Description specifies the stages in the machine's pipeline
3313 
3314 pipe_desc(A, P, F, B, I, J, S, R, E, C, M, W, X, T, D);
3315 
3316 //----------PIPELINE CLASSES---------------------------------------------------
3317 // Pipeline Classes describe the stages in which input and output are
3318 // referenced by the hardware pipeline.
3319 
3320 // Integer ALU reg-reg operation
3321 pipe_class ialu_reg_reg(iRegI dst, iRegI src1, iRegI src2) %{
3322     single_instruction;
3323     dst   : E(write);
3324     src1  : R(read);
3325     src2  : R(read);
3326     IALU  : R;
3327 %}
3328 
3329 // Integer ALU reg-reg long operation
3330 pipe_class ialu_reg_reg_2(iRegL dst, iRegL src1, iRegL src2) %{
3331     instruction_count(2);
3332     dst   : E(write);
3333     src1  : R(read);
3334     src2  : R(read);
3335     IALU  : R;
3336     IALU  : R;
3337 %}
3338 
3339 // Integer ALU reg-reg long dependent operation
3340 pipe_class ialu_reg_reg_2_dep(iRegL dst, iRegL src1, iRegL src2, flagsReg cr) %{
3341     instruction_count(1); multiple_bundles;
3342     dst   : E(write);
3343     src1  : R(read);
3344     src2  : R(read);
3345     cr    : E(write);
3346     IALU  : R(2);
3347 %}
3348 
3349 // Integer ALU reg-imm operaion
3350 pipe_class ialu_reg_imm(iRegI dst, iRegI src1) %{
3351     single_instruction;
3352     dst   : E(write);
3353     src1  : R(read);
3354     IALU  : R;
3355 %}
3356 
3357 // Integer ALU reg-reg operation with condition code
3358 pipe_class ialu_cc_reg_reg(iRegI dst, iRegI src1, iRegI src2, flagsReg cr) %{
3359     single_instruction;
3360     dst   : E(write);
3361     cr    : E(write);
3362     src1  : R(read);
3363     src2  : R(read);
3364     IALU  : R;
3365 %}
3366 
3367 // Integer ALU zero-reg operation
3368 pipe_class ialu_zero_reg(iRegI dst, immI0 zero, iRegI src2) %{
3369     single_instruction;
3370     dst   : E(write);
3371     src2  : R(read);
3372     IALU  : R;
3373 %}
3374 
3375 // Integer ALU zero-reg operation with condition code only
3376 pipe_class ialu_cconly_zero_reg(flagsReg cr, iRegI src) %{
3377     single_instruction;
3378     cr    : E(write);
3379     src   : R(read);
3380     IALU  : R;
3381 %}
3382 
3383 // Integer ALU reg-reg operation with condition code only
3384 pipe_class ialu_cconly_reg_reg(flagsReg cr, iRegI src1, iRegI src2) %{
3385     single_instruction;
3386     cr    : E(write);
3387     src1  : R(read);
3388     src2  : R(read);
3389     IALU  : R;
3390 %}
3391 
3392 // Integer ALU reg-imm operation with condition code only
3393 pipe_class ialu_cconly_reg_imm(flagsReg cr, iRegI src1) %{
3394     single_instruction;
3395     cr    : E(write);
3396     src1  : R(read);
3397     IALU  : R;
3398 %}
3399 
3400 // Integer ALU reg-reg-zero operation with condition code only
3401 pipe_class ialu_cconly_reg_reg_zero(flagsReg cr, iRegI src1, iRegI src2, immI0 zero) %{
3402     single_instruction;
3403     cr    : E(write);
3404     src1  : R(read);
3405     src2  : R(read);
3406     IALU  : R;
3407 %}
3408 
3409 // Integer ALU reg-imm-zero operation with condition code only
3410 pipe_class ialu_cconly_reg_imm_zero(flagsReg cr, iRegI src1, immI0 zero) %{
3411     single_instruction;
3412     cr    : E(write);
3413     src1  : R(read);
3414     IALU  : R;
3415 %}
3416 
3417 // Integer ALU reg-reg operation with condition code, src1 modified
3418 pipe_class ialu_cc_rwreg_reg(flagsReg cr, iRegI src1, iRegI src2) %{
3419     single_instruction;
3420     cr    : E(write);
3421     src1  : E(write);
3422     src1  : R(read);
3423     src2  : R(read);
3424     IALU  : R;
3425 %}
3426 
3427 pipe_class cmpL_reg(iRegI dst, iRegL src1, iRegL src2, flagsReg cr ) %{
3428     multiple_bundles;
3429     dst   : E(write)+4;
3430     cr    : E(write);
3431     src1  : R(read);
3432     src2  : R(read);
3433     IALU  : R(3);
3434     BR    : R(2);
3435 %}
3436 
3437 // Integer ALU operation
3438 pipe_class ialu_none(iRegI dst) %{
3439     single_instruction;
3440     dst   : E(write);
3441     IALU  : R;
3442 %}
3443 
3444 // Integer ALU reg operation
3445 pipe_class ialu_reg(iRegI dst, iRegI src) %{
3446     single_instruction; may_have_no_code;
3447     dst   : E(write);
3448     src   : R(read);
3449     IALU  : R;
3450 %}
3451 
3452 // Integer ALU reg conditional operation
3453 // This instruction has a 1 cycle stall, and cannot execute
3454 // in the same cycle as the instruction setting the condition
3455 // code. We kludge this by pretending to read the condition code
3456 // 1 cycle earlier, and by marking the functional units as busy
3457 // for 2 cycles with the result available 1 cycle later than
3458 // is really the case.
3459 pipe_class ialu_reg_flags( iRegI op2_out, iRegI op2_in, iRegI op1, flagsReg cr ) %{
3460     single_instruction;
3461     op2_out : C(write);
3462     op1     : R(read);
3463     cr      : R(read);       // This is really E, with a 1 cycle stall
3464     BR      : R(2);
3465     MS      : R(2);
3466 %}
3467 
3468 // Integer ALU reg operation
3469 pipe_class ialu_move_reg_L_to_I(iRegI dst, iRegL src) %{
3470     single_instruction; may_have_no_code;
3471     dst   : E(write);
3472     src   : R(read);
3473     IALU  : R;
3474 %}
3475 pipe_class ialu_move_reg_I_to_L(iRegL dst, iRegI src) %{
3476     single_instruction; may_have_no_code;
3477     dst   : E(write);
3478     src   : R(read);
3479     IALU  : R;
3480 %}
3481 
3482 // Two integer ALU reg operations
3483 pipe_class ialu_reg_2(iRegL dst, iRegL src) %{
3484     instruction_count(2);
3485     dst   : E(write);
3486     src   : R(read);
3487     A0    : R;
3488     A1    : R;
3489 %}
3490 
3491 // Two integer ALU reg operations
3492 pipe_class ialu_move_reg_L_to_L(iRegL dst, iRegL src) %{
3493     instruction_count(2); may_have_no_code;
3494     dst   : E(write);
3495     src   : R(read);
3496     A0    : R;
3497     A1    : R;
3498 %}
3499 
3500 // Integer ALU imm operation
3501 pipe_class ialu_imm(iRegI dst) %{
3502     single_instruction;
3503     dst   : E(write);
3504     IALU  : R;
3505 %}
3506 
3507 pipe_class ialu_imm_n(iRegI dst) %{
3508     single_instruction;
3509     dst   : E(write);
3510     IALU  : R;
3511 %}
3512 
3513 // Integer ALU reg-reg with carry operation
3514 pipe_class ialu_reg_reg_cy(iRegI dst, iRegI src1, iRegI src2, iRegI cy) %{
3515     single_instruction;
3516     dst   : E(write);
3517     src1  : R(read);
3518     src2  : R(read);
3519     IALU  : R;
3520 %}
3521 
3522 // Integer ALU cc operation
3523 pipe_class ialu_cc(iRegI dst, flagsReg cc) %{
3524     single_instruction;
3525     dst   : E(write);
3526     cc    : R(read);
3527     IALU  : R;
3528 %}
3529 
3530 // Integer ALU cc / second IALU operation
3531 pipe_class ialu_reg_ialu( iRegI dst, iRegI src ) %{
3532     instruction_count(1); multiple_bundles;
3533     dst   : E(write)+1;
3534     src   : R(read);
3535     IALU  : R;
3536 %}
3537 
3538 // Integer ALU cc / second IALU operation
3539 pipe_class ialu_reg_reg_ialu( iRegI dst, iRegI p, iRegI q ) %{
3540     instruction_count(1); multiple_bundles;
3541     dst   : E(write)+1;
3542     p     : R(read);
3543     q     : R(read);
3544     IALU  : R;
3545 %}
3546 
3547 // Integer ALU hi-lo-reg operation
3548 pipe_class ialu_hi_lo_reg(iRegI dst, immI src) %{
3549     instruction_count(1); multiple_bundles;
3550     dst   : E(write)+1;
3551     IALU  : R(2);
3552 %}
3553 
3554 // Long Constant
3555 pipe_class loadConL( iRegL dst, immL src ) %{
3556     instruction_count(2); multiple_bundles;
3557     dst   : E(write)+1;
3558     IALU  : R(2);
3559     IALU  : R(2);
3560 %}
3561 
3562 // Pointer Constant
3563 pipe_class loadConP( iRegP dst, immP src ) %{
3564     instruction_count(0); multiple_bundles;
3565     fixed_latency(6);
3566 %}
3567 
3568 // Polling Address
3569 pipe_class loadConP_poll( iRegP dst, immP_poll src ) %{
3570     dst   : E(write);
3571     IALU  : R;
3572 %}
3573 
3574 // Long Constant small
3575 pipe_class loadConLlo( iRegL dst, immL src ) %{
3576     instruction_count(2);
3577     dst   : E(write);
3578     IALU  : R;
3579     IALU  : R;
3580 %}
3581 
3582 // [PHH] This is wrong for 64-bit.  See LdImmF/D.
3583 pipe_class loadConFD(regF dst, immF src, iRegP tmp) %{
3584     instruction_count(1); multiple_bundles;
3585     src   : R(read);
3586     dst   : M(write)+1;
3587     IALU  : R;
3588     MS    : E;
3589 %}
3590 
3591 // Integer ALU nop operation
3592 pipe_class ialu_nop() %{
3593     single_instruction;
3594     IALU  : R;
3595 %}
3596 
3597 // Integer ALU nop operation
3598 pipe_class ialu_nop_A0() %{
3599     single_instruction;
3600     A0    : R;
3601 %}
3602 
3603 // Integer ALU nop operation
3604 pipe_class ialu_nop_A1() %{
3605     single_instruction;
3606     A1    : R;
3607 %}
3608 
3609 // Integer Multiply reg-reg operation
3610 pipe_class imul_reg_reg(iRegI dst, iRegI src1, iRegI src2) %{
3611     single_instruction;
3612     dst   : E(write);
3613     src1  : R(read);
3614     src2  : R(read);
3615     MS    : R(5);
3616 %}
3617 
3618 pipe_class mulL_reg_reg(iRegL dst, iRegL src1, iRegL src2) %{
3619     single_instruction;
3620     dst   : E(write)+4;
3621     src1  : R(read);
3622     src2  : R(read);
3623     MS    : R(6);
3624 %}
3625 
3626 // Integer Divide reg-reg
3627 pipe_class sdiv_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI temp, flagsReg cr) %{
3628     instruction_count(1); multiple_bundles;
3629     dst   : E(write);
3630     temp  : E(write);
3631     src1  : R(read);
3632     src2  : R(read);
3633     temp  : R(read);
3634     MS    : R(38);
3635 %}
3636 
3637 // Long Divide
3638 pipe_class divL_reg_reg(iRegL dst, iRegL src1, iRegL src2) %{
3639     dst  : E(write)+71;
3640     src1 : R(read);
3641     src2 : R(read)+1;
3642     MS   : R(70);
3643 %}
3644 
3645 // Floating Point Add Float
3646 pipe_class faddF_reg_reg(regF dst, regF src1, regF src2) %{
3647     single_instruction;
3648     dst   : X(write);
3649     src1  : E(read);
3650     src2  : E(read);
3651     FA    : R;
3652 %}
3653 
3654 // Floating Point Add Double
3655 pipe_class faddD_reg_reg(regD dst, regD src1, regD src2) %{
3656     single_instruction;
3657     dst   : X(write);
3658     src1  : E(read);
3659     src2  : E(read);
3660     FA    : R;
3661 %}
3662 
3663 // Floating Point Conditional Move based on integer flags
3664 pipe_class int_conditional_float_move (cmpOp cmp, flagsReg cr, regF dst, regF src) %{
3665     single_instruction;
3666     dst   : X(write);
3667     src   : E(read);
3668     cr    : R(read);
3669     FA    : R(2);
3670     BR    : R(2);
3671 %}
3672 
3673 // Floating Point Conditional Move based on integer flags
3674 pipe_class int_conditional_double_move (cmpOp cmp, flagsReg cr, regD dst, regD src) %{
3675     single_instruction;
3676     dst   : X(write);
3677     src   : E(read);
3678     cr    : R(read);
3679     FA    : R(2);
3680     BR    : R(2);
3681 %}
3682 
3683 // Floating Point Multiply Float
3684 pipe_class fmulF_reg_reg(regF dst, regF src1, regF src2) %{
3685     single_instruction;
3686     dst   : X(write);
3687     src1  : E(read);
3688     src2  : E(read);
3689     FM    : R;
3690 %}
3691 
3692 // Floating Point Multiply Double
3693 pipe_class fmulD_reg_reg(regD dst, regD src1, regD src2) %{
3694     single_instruction;
3695     dst   : X(write);
3696     src1  : E(read);
3697     src2  : E(read);
3698     FM    : R;
3699 %}
3700 
3701 // Floating Point Divide Float
3702 pipe_class fdivF_reg_reg(regF dst, regF src1, regF src2) %{
3703     single_instruction;
3704     dst   : X(write);
3705     src1  : E(read);
3706     src2  : E(read);
3707     FM    : R;
3708     FDIV  : C(14);
3709 %}
3710 
3711 // Floating Point Divide Double
3712 pipe_class fdivD_reg_reg(regD dst, regD src1, regD src2) %{
3713     single_instruction;
3714     dst   : X(write);
3715     src1  : E(read);
3716     src2  : E(read);
3717     FM    : R;
3718     FDIV  : C(17);
3719 %}
3720 
3721 // Floating Point Move/Negate/Abs Float
3722 pipe_class faddF_reg(regF dst, regF src) %{
3723     single_instruction;
3724     dst   : W(write);
3725     src   : E(read);
3726     FA    : R(1);
3727 %}
3728 
3729 // Floating Point Move/Negate/Abs Double
3730 pipe_class faddD_reg(regD dst, regD src) %{
3731     single_instruction;
3732     dst   : W(write);
3733     src   : E(read);
3734     FA    : R;
3735 %}
3736 
3737 // Floating Point Convert F->D
3738 pipe_class fcvtF2D(regD dst, regF src) %{
3739     single_instruction;
3740     dst   : X(write);
3741     src   : E(read);
3742     FA    : R;
3743 %}
3744 
3745 // Floating Point Convert I->D
3746 pipe_class fcvtI2D(regD dst, regF src) %{
3747     single_instruction;
3748     dst   : X(write);
3749     src   : E(read);
3750     FA    : R;
3751 %}
3752 
3753 // Floating Point Convert LHi->D
3754 pipe_class fcvtLHi2D(regD dst, regD src) %{
3755     single_instruction;
3756     dst   : X(write);
3757     src   : E(read);
3758     FA    : R;
3759 %}
3760 
3761 // Floating Point Convert L->D
3762 pipe_class fcvtL2D(regD dst, iRegL src) %{
3763     single_instruction;
3764     dst   : X(write);
3765     src   : E(read);
3766     FA    : R;
3767 %}
3768 
3769 // Floating Point Convert L->F
3770 pipe_class fcvtL2F(regF dst, iRegL src) %{
3771     single_instruction;
3772     dst   : X(write);
3773     src   : E(read);
3774     FA    : R;
3775 %}
3776 
3777 // Floating Point Convert D->F
3778 pipe_class fcvtD2F(regD dst, regF src) %{
3779     single_instruction;
3780     dst   : X(write);
3781     src   : E(read);
3782     FA    : R;
3783 %}
3784 
3785 // Floating Point Convert I->L
3786 pipe_class fcvtI2L(regD dst, regF src) %{
3787     single_instruction;
3788     dst   : X(write);
3789     src   : E(read);
3790     FA    : R;
3791 %}
3792 
3793 // Floating Point Convert D->F
3794 pipe_class fcvtD2I(iRegI dst, regD src, flagsReg cr) %{
3795     instruction_count(1); multiple_bundles;
3796     dst   : X(write)+6;
3797     src   : E(read);
3798     FA    : R;
3799 %}
3800 
3801 // Floating Point Convert D->L
3802 pipe_class fcvtD2L(regD dst, regD src, flagsReg cr) %{
3803     instruction_count(1); multiple_bundles;
3804     dst   : X(write)+6;
3805     src   : E(read);
3806     FA    : R;
3807 %}
3808 
3809 // Floating Point Convert F->I
3810 pipe_class fcvtF2I(regF dst, regF src, flagsReg cr) %{
3811     instruction_count(1); multiple_bundles;
3812     dst   : X(write)+6;
3813     src   : E(read);
3814     FA    : R;
3815 %}
3816 
3817 // Floating Point Convert F->L
3818 pipe_class fcvtF2L(regD dst, regF src, flagsReg cr) %{
3819     instruction_count(1); multiple_bundles;
3820     dst   : X(write)+6;
3821     src   : E(read);
3822     FA    : R;
3823 %}
3824 
3825 // Floating Point Convert I->F
3826 pipe_class fcvtI2F(regF dst, regF src) %{
3827     single_instruction;
3828     dst   : X(write);
3829     src   : E(read);
3830     FA    : R;
3831 %}
3832 
3833 // Floating Point Compare
3834 pipe_class faddF_fcc_reg_reg_zero(flagsRegF cr, regF src1, regF src2, immI0 zero) %{
3835     single_instruction;
3836     cr    : X(write);
3837     src1  : E(read);
3838     src2  : E(read);
3839     FA    : R;
3840 %}
3841 
3842 // Floating Point Compare
3843 pipe_class faddD_fcc_reg_reg_zero(flagsRegF cr, regD src1, regD src2, immI0 zero) %{
3844     single_instruction;
3845     cr    : X(write);
3846     src1  : E(read);
3847     src2  : E(read);
3848     FA    : R;
3849 %}
3850 
3851 // Floating Add Nop
3852 pipe_class fadd_nop() %{
3853     single_instruction;
3854     FA  : R;
3855 %}
3856 
3857 // Integer Store to Memory
3858 pipe_class istore_mem_reg(memoryI mem, iRegI src) %{
3859     single_instruction;
3860     mem   : R(read);
3861     src   : C(read);
3862     MS    : R;
3863 %}
3864 
3865 // Integer Store to Memory
3866 pipe_class istore_mem_spORreg(memoryI mem, sp_ptr_RegP src) %{
3867     single_instruction;
3868     mem   : R(read);
3869     src   : C(read);
3870     MS    : R;
3871 %}
3872 
3873 // Float Store
3874 pipe_class fstoreF_mem_reg(memoryF mem, RegF src) %{
3875     single_instruction;
3876     mem : R(read);
3877     src : C(read);
3878     MS  : R;
3879 %}
3880 
3881 // Float Store
3882 pipe_class fstoreF_mem_zero(memoryF mem, immF0 src) %{
3883     single_instruction;
3884     mem : R(read);
3885     MS  : R;
3886 %}
3887 
3888 // Double Store
3889 pipe_class fstoreD_mem_reg(memoryD mem, RegD src) %{
3890     instruction_count(1);
3891     mem : R(read);
3892     src : C(read);
3893     MS  : R;
3894 %}
3895 
3896 // Double Store
3897 pipe_class fstoreD_mem_zero(memoryD mem, immD0 src) %{
3898     single_instruction;
3899     mem : R(read);
3900     MS  : R;
3901 %}
3902 
3903 // Integer Load (when sign bit propagation not needed)
3904 pipe_class iload_mem(iRegI dst, memoryI mem) %{
3905     single_instruction;
3906     mem : R(read);
3907     dst : C(write);
3908     MS  : R;
3909 %}
3910 
3911 // Integer Load (when sign bit propagation or masking is needed)
3912 pipe_class iload_mask_mem(iRegI dst, memoryI mem) %{
3913     single_instruction;
3914     mem : R(read);
3915     dst : M(write);
3916     MS  : R;
3917 %}
3918 
3919 // Float Load
3920 pipe_class floadF_mem(regF dst, memoryF mem) %{
3921     single_instruction;
3922     mem : R(read);
3923     dst : M(write);
3924     MS  : R;
3925 %}
3926 
3927 // Float Load
3928 pipe_class floadD_mem(regD dst, memoryD mem) %{
3929     instruction_count(1); multiple_bundles; // Again, unaligned argument is only multiple case
3930     mem : R(read);
3931     dst : M(write);
3932     MS  : R;
3933 %}
3934 
3935 // Memory Nop
3936 pipe_class mem_nop() %{
3937     single_instruction;
3938     MS  : R;
3939 %}
3940 
3941 pipe_class sethi(iRegP dst, immI src) %{
3942     single_instruction;
3943     dst  : E(write);
3944     IALU : R;
3945 %}
3946 
3947 pipe_class loadPollP(iRegP poll) %{
3948     single_instruction;
3949     poll : R(read);
3950     MS   : R;
3951 %}
3952 
3953 pipe_class br(Universe br, label labl) %{
3954     single_instruction_with_delay_slot;
3955     BR  : R;
3956 %}
3957 
3958 pipe_class br_cc(Universe br, cmpOp cmp, flagsReg cr, label labl) %{
3959     single_instruction_with_delay_slot;
3960     cr    : E(read);
3961     BR    : R;
3962 %}
3963 
3964 pipe_class br_reg(Universe br, cmpOp cmp, iRegI op1, label labl) %{
3965     single_instruction_with_delay_slot;
3966     op1 : E(read);
3967     BR  : R;
3968     MS  : R;
3969 %}
3970 
3971 pipe_class br_nop() %{
3972     single_instruction;
3973     BR  : R;
3974 %}
3975 
3976 pipe_class simple_call(method meth) %{
3977     instruction_count(2); multiple_bundles; force_serialization;
3978     fixed_latency(100);
3979     BR  : R(1);
3980     MS  : R(1);
3981     A0  : R(1);
3982 %}
3983 
3984 pipe_class compiled_call(method meth) %{
3985     instruction_count(1); multiple_bundles; force_serialization;
3986     fixed_latency(100);
3987     MS  : R(1);
3988 %}
3989 
3990 pipe_class call(method meth) %{
3991     instruction_count(0); multiple_bundles; force_serialization;
3992     fixed_latency(100);
3993 %}
3994 
3995 pipe_class tail_call(Universe ignore, label labl) %{
3996     single_instruction; has_delay_slot;
3997     fixed_latency(100);
3998     BR  : R(1);
3999     MS  : R(1);
4000 %}
4001 
4002 pipe_class ret(Universe ignore) %{
4003     single_instruction; has_delay_slot;
4004     BR  : R(1);
4005     MS  : R(1);
4006 %}
4007 
4008 // The real do-nothing guy
4009 pipe_class empty( ) %{
4010     instruction_count(0);
4011 %}
4012 
4013 pipe_class long_memory_op() %{
4014     instruction_count(0); multiple_bundles; force_serialization;
4015     fixed_latency(25);
4016     MS  : R(1);
4017 %}
4018 
4019 // Check-cast
4020 pipe_class partial_subtype_check_pipe(Universe ignore, iRegP array, iRegP match ) %{
4021     array : R(read);
4022     match  : R(read);
4023     IALU   : R(2);
4024     BR     : R(2);
4025     MS     : R;
4026 %}
4027 
4028 // Convert FPU flags into +1,0,-1
4029 pipe_class floating_cmp( iRegI dst, regF src1, regF src2 ) %{
4030     src1  : E(read);
4031     src2  : E(read);
4032     dst   : E(write);
4033     FA    : R;
4034     MS    : R(2);
4035     BR    : R(2);
4036 %}
4037 
4038 // Compare for p < q, and conditionally add y
4039 pipe_class cadd_cmpltmask( iRegI p, iRegI q, iRegI y ) %{
4040     p     : E(read);
4041     q     : E(read);
4042     y     : E(read);
4043     IALU  : R(3)
4044 %}
4045 
4046 // Perform a compare, then move conditionally in a branch delay slot.
4047 pipe_class min_max( iRegI src2, iRegI srcdst ) %{
4048     src2   : E(read);
4049     srcdst : E(read);
4050     IALU   : R;
4051     BR     : R;
4052 %}
4053 
4054 // Define the class for the Nop node
4055 define %{
4056    MachNop = ialu_nop;
4057 %}
4058 
4059 %}
4060 
4061 //----------INSTRUCTIONS-------------------------------------------------------
4062 
4063 //------------Special Nop instructions for bundling - no match rules-----------
4064 // Nop using the A0 functional unit
4065 instruct Nop_A0() %{
4066   ins_pipe(ialu_nop_A0);
4067 %}
4068 
4069 // Nop using the A1 functional unit
4070 instruct Nop_A1( ) %{
4071   ins_pipe(ialu_nop_A1);
4072 %}
4073 
4074 // Nop using the memory functional unit
4075 instruct Nop_MS( ) %{
4076   ins_pipe(mem_nop);
4077 %}
4078 
4079 // Nop using the floating add functional unit
4080 instruct Nop_FA( ) %{
4081   ins_pipe(fadd_nop);
4082 %}
4083 
4084 // Nop using the branch functional unit
4085 instruct Nop_BR( ) %{
4086   ins_pipe(br_nop);
4087 %}
4088 
4089 //----------Load/Store/Move Instructions---------------------------------------
4090 //----------Load Instructions--------------------------------------------------
4091 // Load Byte (8bit signed)
4092 instruct loadB(iRegI dst, memoryB mem) %{
4093   match(Set dst (LoadB mem));
4094   ins_cost(MEMORY_REF_COST);
4095 
4096   size(4);
4097   format %{ "LDRSB   $dst,$mem\t! byte -> int" %}
4098   ins_encode %{
4099     // High 32 bits are harmlessly set on Aarch64
4100     __ ldrsb($dst$$Register, $mem$$Address);
4101   %}
4102   ins_pipe(iload_mask_mem);
4103 %}
4104 
4105 // Load Byte (8bit signed) into a Long Register
4106 instruct loadB2L(iRegL dst, memoryB mem) %{
4107   match(Set dst (ConvI2L (LoadB mem)));
4108   ins_cost(MEMORY_REF_COST);
4109 
4110 #ifdef AARCH64
4111   size(4);
4112   format %{ "LDRSB $dst,$mem\t! byte -> long"  %}
4113   ins_encode %{
4114     __ ldrsb($dst$$Register, $mem$$Address);
4115   %}
4116 #else
4117   size(8);
4118   format %{ "LDRSB $dst.lo,$mem\t! byte -> long\n\t"
4119             "ASR   $dst.hi,$dst.lo,31" %}
4120   ins_encode %{
4121     __ ldrsb($dst$$Register, $mem$$Address);
4122     __ mov($dst$$Register->successor(), AsmOperand($dst$$Register, asr, 31));
4123   %}
4124 #endif
4125   ins_pipe(iload_mask_mem);
4126 %}
4127 
4128 // Load Unsigned Byte (8bit UNsigned) into an int reg
4129 instruct loadUB(iRegI dst, memoryB mem) %{
4130   match(Set dst (LoadUB mem));
4131   ins_cost(MEMORY_REF_COST);
4132 
4133   size(4);
4134   format %{ "LDRB   $dst,$mem\t! ubyte -> int" %}
4135   ins_encode %{
4136     __ ldrb($dst$$Register, $mem$$Address);
4137   %}
4138   ins_pipe(iload_mem);
4139 %}
4140 
4141 // Load Unsigned Byte (8bit UNsigned) into a Long Register
4142 instruct loadUB2L(iRegL dst, memoryB mem) %{
4143   match(Set dst (ConvI2L (LoadUB mem)));
4144   ins_cost(MEMORY_REF_COST);
4145 
4146 #ifdef AARCH64
4147   size(4);
4148   format %{ "LDRB  $dst,$mem\t! ubyte -> long"  %}
4149   ins_encode %{
4150     __ ldrb($dst$$Register, $mem$$Address);
4151   %}
4152 #else
4153   size(8);
4154   format %{ "LDRB  $dst.lo,$mem\t! ubyte -> long\n\t"
4155             "MOV   $dst.hi,0" %}
4156   ins_encode %{
4157     __ ldrb($dst$$Register, $mem$$Address);
4158     __ mov($dst$$Register->successor(), 0);
4159   %}
4160 #endif
4161   ins_pipe(iload_mem);
4162 %}
4163 
4164 // Load Unsigned Byte (8 bit UNsigned) with immediate mask into Long Register
4165 instruct loadUB2L_limmI(iRegL dst, memoryB mem, limmIlow8 mask) %{
4166   match(Set dst (ConvI2L (AndI (LoadUB mem) mask)));
4167 
4168 #ifdef AARCH64
4169   ins_cost(MEMORY_REF_COST + DEFAULT_COST);
4170   size(8);
4171   format %{ "LDRB  $dst,$mem\t! ubyte -> long\n\t"
4172             "AND  $dst,$dst,$mask" %}
4173   ins_encode %{
4174     __ ldrb($dst$$Register, $mem$$Address);
4175     __ andr($dst$$Register, $dst$$Register, limmI_low($mask$$constant, 8));
4176   %}
4177 #else
4178   ins_cost(MEMORY_REF_COST + 2*DEFAULT_COST);
4179   size(12);
4180   format %{ "LDRB  $dst.lo,$mem\t! ubyte -> long\n\t"
4181             "MOV   $dst.hi,0\n\t"
4182             "AND  $dst.lo,$dst.lo,$mask" %}
4183   ins_encode %{
4184     __ ldrb($dst$$Register, $mem$$Address);
4185     __ mov($dst$$Register->successor(), 0);
4186     __ andr($dst$$Register, $dst$$Register, limmI_low($mask$$constant, 8));
4187   %}
4188 #endif
4189   ins_pipe(iload_mem);
4190 %}
4191 
4192 // Load Short (16bit signed)
4193 #ifdef AARCH64
4194 // XXX This variant shouldn't be necessary if 6217251 is implemented
4195 instruct loadSoff(iRegI dst, memoryScaledS mem, aimmX off, iRegP tmp) %{
4196   match(Set dst (LoadS (AddP mem off)));
4197   ins_cost(MEMORY_REF_COST + DEFAULT_COST); // assume shift/sign-extend is free
4198   effect(TEMP tmp);
4199   size(4 * 2);
4200 
4201   format %{ "LDRSH   $dst,$mem+$off\t! short temp=$tmp" %}
4202   ins_encode %{
4203     Register base = reg_to_register_object($mem$$base);
4204     __ add($tmp$$Register, base, $off$$constant);
4205     Address nmem = Address::make_raw($tmp$$reg, $mem$$index, $mem$$scale, $mem$$disp, relocInfo::none);
4206     __ ldrsh($dst$$Register, nmem);
4207   %}
4208   ins_pipe(iload_mask_mem);
4209 %}
4210 #endif
4211 
4212 instruct loadS(iRegI dst, memoryS mem) %{
4213   match(Set dst (LoadS mem));
4214   ins_cost(MEMORY_REF_COST);
4215 
4216   size(4);
4217   format %{ "LDRSH   $dst,$mem\t! short" %}
4218   ins_encode %{
4219     __ ldrsh($dst$$Register, $mem$$Address);
4220   %}
4221   ins_pipe(iload_mask_mem);
4222 %}
4223 
4224 // Load Short (16 bit signed) to Byte (8 bit signed)
4225 instruct loadS2B(iRegI dst, memoryS mem, immI_24 twentyfour) %{
4226   match(Set dst (RShiftI (LShiftI (LoadS mem) twentyfour) twentyfour));
4227   ins_cost(MEMORY_REF_COST);
4228 
4229   size(4);
4230 
4231   format %{ "LDRSB   $dst,$mem\t! short -> byte" %}
4232   ins_encode %{
4233     // High 32 bits are harmlessly set on Aarch64
4234     __ ldrsb($dst$$Register, $mem$$Address);
4235   %}
4236   ins_pipe(iload_mask_mem);
4237 %}
4238 
4239 // Load Short (16bit signed) into a Long Register
4240 instruct loadS2L(iRegL dst, memoryS mem) %{
4241   match(Set dst (ConvI2L (LoadS mem)));
4242   ins_cost(MEMORY_REF_COST);
4243 
4244 #ifdef AARCH64
4245   size(4);
4246   format %{ "LDRSH $dst,$mem\t! short -> long"  %}
4247   ins_encode %{
4248     __ ldrsh($dst$$Register, $mem$$Address);
4249   %}
4250 #else
4251   size(8);
4252   format %{ "LDRSH $dst.lo,$mem\t! short -> long\n\t"
4253             "ASR   $dst.hi,$dst.lo,31" %}
4254   ins_encode %{
4255     __ ldrsh($dst$$Register, $mem$$Address);
4256     __ mov($dst$$Register->successor(), AsmOperand($dst$$Register, asr, 31));
4257   %}
4258 #endif
4259   ins_pipe(iload_mask_mem);
4260 %}
4261 
4262 // Load Unsigned Short/Char (16bit UNsigned)
4263 
4264 #ifdef AARCH64
4265 // XXX This variant shouldn't be necessary if 6217251 is implemented
4266 instruct loadUSoff(iRegI dst, memoryScaledS mem, aimmX off, iRegP tmp) %{
4267   match(Set dst (LoadUS (AddP mem off)));
4268   ins_cost(MEMORY_REF_COST + DEFAULT_COST); // assume shift/sign-extend is free
4269   effect(TEMP tmp);
4270   size(4 * 2);
4271 
4272   format %{ "LDRH   $dst,$mem+$off\t! ushort/char temp=$tmp" %}
4273   ins_encode %{
4274     Register base = reg_to_register_object($mem$$base);
4275     __ add($tmp$$Register, base, $off$$constant);
4276     Address nmem = Address::make_raw($tmp$$reg, $mem$$index, $mem$$scale, $mem$$disp, relocInfo::none);
4277     __ ldrh($dst$$Register, nmem);
4278   %}
4279   ins_pipe(iload_mem);
4280 %}
4281 #endif
4282 
4283 instruct loadUS(iRegI dst, memoryS mem) %{
4284   match(Set dst (LoadUS mem));
4285   ins_cost(MEMORY_REF_COST);
4286 
4287   size(4);
4288   format %{ "LDRH   $dst,$mem\t! ushort/char" %}
4289   ins_encode %{
4290     __ ldrh($dst$$Register, $mem$$Address);
4291   %}
4292   ins_pipe(iload_mem);
4293 %}
4294 
4295 // Load Unsigned Short/Char (16 bit UNsigned) to Byte (8 bit signed)
4296 instruct loadUS2B(iRegI dst, memoryB mem, immI_24 twentyfour) %{
4297   match(Set dst (RShiftI (LShiftI (LoadUS mem) twentyfour) twentyfour));
4298   ins_cost(MEMORY_REF_COST);
4299 
4300   size(4);
4301   format %{ "LDRSB   $dst,$mem\t! ushort -> byte" %}
4302   ins_encode %{
4303     __ ldrsb($dst$$Register, $mem$$Address);
4304   %}
4305   ins_pipe(iload_mask_mem);
4306 %}
4307 
4308 // Load Unsigned Short/Char (16bit UNsigned) into a Long Register
4309 instruct loadUS2L(iRegL dst, memoryS mem) %{
4310   match(Set dst (ConvI2L (LoadUS mem)));
4311   ins_cost(MEMORY_REF_COST);
4312 
4313 #ifdef AARCH64
4314   size(4);
4315   format %{ "LDRH  $dst,$mem\t! short -> long"  %}
4316   ins_encode %{
4317     __ ldrh($dst$$Register, $mem$$Address);
4318   %}
4319 #else
4320   size(8);
4321   format %{ "LDRH  $dst.lo,$mem\t! short -> long\n\t"
4322             "MOV   $dst.hi, 0" %}
4323   ins_encode %{
4324     __ ldrh($dst$$Register, $mem$$Address);
4325     __ mov($dst$$Register->successor(), 0);
4326   %}
4327 #endif
4328   ins_pipe(iload_mem);
4329 %}
4330 
4331 // Load Unsigned Short/Char (16bit UNsigned) with mask 0xFF into a Long Register
4332 instruct loadUS2L_immI_255(iRegL dst, memoryB mem, immI_255 mask) %{
4333   match(Set dst (ConvI2L (AndI (LoadUS mem) mask)));
4334   ins_cost(MEMORY_REF_COST);
4335 
4336 #ifdef AARCH64
4337   size(4);
4338   format %{ "LDRB  $dst,$mem"  %}
4339   ins_encode %{
4340     __ ldrb($dst$$Register, $mem$$Address);
4341   %}
4342 #else
4343   size(8);
4344   format %{ "LDRB  $dst.lo,$mem\t! \n\t"
4345             "MOV   $dst.hi, 0" %}
4346   ins_encode %{
4347     __ ldrb($dst$$Register, $mem$$Address);
4348     __ mov($dst$$Register->successor(), 0);
4349   %}
4350 #endif
4351   ins_pipe(iload_mem);
4352 %}
4353 
4354 // Load Unsigned Short/Char (16bit UNsigned) with a immediate mask into a Long Register
4355 instruct loadUS2L_limmI(iRegL dst, memoryS mem, limmI mask) %{
4356   match(Set dst (ConvI2L (AndI (LoadUS mem) mask)));
4357 #ifdef AARCH64
4358   ins_cost(MEMORY_REF_COST + 1*DEFAULT_COST);
4359 
4360   size(8);
4361   format %{ "LDRH   $dst,$mem\t! ushort/char & mask -> long\n\t"
4362             "AND    $dst,$dst,$mask" %}
4363   ins_encode %{
4364     __ ldrh($dst$$Register, $mem$$Address);
4365     __ andr($dst$$Register, $dst$$Register, (uintx)$mask$$constant);
4366   %}
4367 #else
4368   ins_cost(MEMORY_REF_COST + 2*DEFAULT_COST);
4369 
4370   size(12);
4371   format %{ "LDRH   $dst,$mem\t! ushort/char & mask -> long\n\t"
4372             "MOV    $dst.hi, 0\n\t"
4373             "AND    $dst,$dst,$mask" %}
4374   ins_encode %{
4375     __ ldrh($dst$$Register, $mem$$Address);
4376     __ mov($dst$$Register->successor(), 0);
4377     __ andr($dst$$Register, $dst$$Register, $mask$$constant);
4378   %}
4379 #endif
4380   ins_pipe(iload_mem);
4381 %}
4382 
4383 // Load Integer
4384 
4385 #ifdef AARCH64
4386 // XXX This variant shouldn't be necessary if 6217251 is implemented
4387 instruct loadIoff(iRegI dst, memoryScaledI mem, aimmX off, iRegP tmp) %{
4388   match(Set dst (LoadI (AddP mem off)));
4389   ins_cost(MEMORY_REF_COST + DEFAULT_COST); // assume shift/sign-extend is free
4390   effect(TEMP tmp);
4391   size(4 * 2);
4392 
4393   format %{ "ldr_s32 $dst,$mem+$off\t! int temp=$tmp" %}
4394   ins_encode %{
4395     Register base = reg_to_register_object($mem$$base);
4396     __ add($tmp$$Register, base, $off$$constant);
4397     Address nmem = Address::make_raw($tmp$$reg, $mem$$index, $mem$$scale, $mem$$disp, relocInfo::none);
4398     __ ldr_s32($dst$$Register, nmem);
4399   %}
4400   ins_pipe(iload_mem);
4401 %}
4402 #endif
4403 
4404 instruct loadI(iRegI dst, memoryI mem) %{
4405   match(Set dst (LoadI mem));
4406   ins_cost(MEMORY_REF_COST);
4407 
4408   size(4);
4409   format %{ "ldr_s32 $dst,$mem\t! int" %}
4410   ins_encode %{
4411     __ ldr_s32($dst$$Register, $mem$$Address);
4412   %}
4413   ins_pipe(iload_mem);
4414 %}
4415 
4416 // Load Integer to Byte (8 bit signed)
4417 instruct loadI2B(iRegI dst, memoryS mem, immI_24 twentyfour) %{
4418   match(Set dst (RShiftI (LShiftI (LoadI mem) twentyfour) twentyfour));
4419   ins_cost(MEMORY_REF_COST);
4420 
4421   size(4);
4422 
4423   format %{ "LDRSB   $dst,$mem\t! int -> byte" %}
4424   ins_encode %{
4425     __ ldrsb($dst$$Register, $mem$$Address);
4426   %}
4427   ins_pipe(iload_mask_mem);
4428 %}
4429 
4430 // Load Integer to Unsigned Byte (8 bit UNsigned)
4431 instruct loadI2UB(iRegI dst, memoryB mem, immI_255 mask) %{
4432   match(Set dst (AndI (LoadI mem) mask));
4433   ins_cost(MEMORY_REF_COST);
4434 
4435   size(4);
4436 
4437   format %{ "LDRB   $dst,$mem\t! int -> ubyte" %}
4438   ins_encode %{
4439     __ ldrb($dst$$Register, $mem$$Address);
4440   %}
4441   ins_pipe(iload_mask_mem);
4442 %}
4443 
4444 // Load Integer to Short (16 bit signed)
4445 instruct loadI2S(iRegI dst, memoryS mem, immI_16 sixteen) %{
4446   match(Set dst (RShiftI (LShiftI (LoadI mem) sixteen) sixteen));
4447   ins_cost(MEMORY_REF_COST);
4448 
4449   size(4);
4450   format %{ "LDRSH   $dst,$mem\t! int -> short" %}
4451   ins_encode %{
4452     __ ldrsh($dst$$Register, $mem$$Address);
4453   %}
4454   ins_pipe(iload_mask_mem);
4455 %}
4456 
4457 // Load Integer to Unsigned Short (16 bit UNsigned)
4458 instruct loadI2US(iRegI dst, memoryS mem, immI_65535 mask) %{
4459   match(Set dst (AndI (LoadI mem) mask));
4460   ins_cost(MEMORY_REF_COST);
4461 
4462   size(4);
4463   format %{ "LDRH   $dst,$mem\t! int -> ushort/char" %}
4464   ins_encode %{
4465     __ ldrh($dst$$Register, $mem$$Address);
4466   %}
4467   ins_pipe(iload_mask_mem);
4468 %}
4469 
4470 // Load Integer into a Long Register
4471 instruct loadI2L(iRegL dst, memoryI mem) %{
4472   match(Set dst (ConvI2L (LoadI mem)));
4473 #ifdef AARCH64
4474   ins_cost(MEMORY_REF_COST);
4475 
4476   size(4);
4477   format %{ "LDRSW $dst.lo,$mem\t! int -> long"  %}
4478   ins_encode %{
4479     __ ldr_s32($dst$$Register, $mem$$Address);
4480   %}
4481 #else
4482   ins_cost(MEMORY_REF_COST);
4483 
4484   size(8);
4485   format %{ "LDR   $dst.lo,$mem\t! int -> long\n\t"
4486             "ASR   $dst.hi,$dst.lo,31\t! int->long" %}
4487   ins_encode %{
4488     __ ldr($dst$$Register, $mem$$Address);
4489     __ mov($dst$$Register->successor(), AsmOperand($dst$$Register, asr, 31));
4490   %}
4491 #endif
4492   ins_pipe(iload_mask_mem);
4493 %}
4494 
4495 // Load Integer with mask 0xFF into a Long Register
4496 instruct loadI2L_immI_255(iRegL dst, memoryB mem, immI_255 mask) %{
4497   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
4498 #ifdef AARCH64
4499   ins_cost(MEMORY_REF_COST);
4500 
4501   size(4);
4502   format %{ "LDRB   $dst.lo,$mem\t! int & 0xFF -> long"  %}
4503   ins_encode %{
4504     __ ldrb($dst$$Register, $mem$$Address);
4505   %}
4506 #else
4507   ins_cost(MEMORY_REF_COST);
4508 
4509   size(8);
4510   format %{ "LDRB   $dst.lo,$mem\t! int & 0xFF -> long\n\t"
4511             "MOV    $dst.hi, 0" %}
4512   ins_encode %{
4513     __ ldrb($dst$$Register, $mem$$Address);
4514     __ mov($dst$$Register->successor(), 0);
4515   %}
4516 #endif
4517   ins_pipe(iload_mem);
4518 %}
4519 
4520 // Load Integer with mask 0xFFFF into a Long Register
4521 instruct loadI2L_immI_65535(iRegL dst, memoryS mem, immI_65535 mask) %{
4522   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
4523   ins_cost(MEMORY_REF_COST);
4524 
4525 #ifdef AARCH64
4526   size(4);
4527   format %{ "LDRH   $dst,$mem\t! int & 0xFFFF -> long" %}
4528   ins_encode %{
4529     __ ldrh($dst$$Register, $mem$$Address);
4530   %}
4531 #else
4532   size(8);
4533   format %{ "LDRH   $dst,$mem\t! int & 0xFFFF -> long\n\t"
4534             "MOV    $dst.hi, 0" %}
4535   ins_encode %{
4536     __ ldrh($dst$$Register, $mem$$Address);
4537     __ mov($dst$$Register->successor(), 0);
4538   %}
4539 #endif
4540   ins_pipe(iload_mask_mem);
4541 %}
4542 
4543 #ifdef AARCH64
4544 // Load Integer with an immediate mask into a Long Register
4545 instruct loadI2L_limmI(iRegL dst, memoryI mem, limmI mask) %{
4546   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
4547   ins_cost(MEMORY_REF_COST + 1*DEFAULT_COST);
4548 
4549   size(8);
4550   format %{ "LDRSW $dst,$mem\t! int -> long\n\t"
4551             "AND   $dst,$dst,$mask" %}
4552 
4553   ins_encode %{
4554     __ ldr_s32($dst$$Register, $mem$$Address);
4555     __ andr($dst$$Register, $dst$$Register, (uintx)$mask$$constant);
4556   %}
4557   ins_pipe(iload_mem);
4558 %}
4559 #else
4560 // Load Integer with a 31-bit immediate mask into a Long Register
4561 instruct loadI2L_limmU31(iRegL dst, memoryI mem, limmU31 mask) %{
4562   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
4563   ins_cost(MEMORY_REF_COST + 2*DEFAULT_COST);
4564 
4565   size(12);
4566   format %{ "LDR   $dst.lo,$mem\t! int -> long\n\t"
4567             "MOV    $dst.hi, 0\n\t"
4568             "AND   $dst,$dst,$mask" %}
4569 
4570   ins_encode %{
4571     __ ldr($dst$$Register, $mem$$Address);
4572     __ mov($dst$$Register->successor(), 0);
4573     __ andr($dst$$Register, $dst$$Register, $mask$$constant);
4574   %}
4575   ins_pipe(iload_mem);
4576 %}
4577 #endif
4578 
4579 #ifdef AARCH64
4580 // Load Integer with mask into a Long Register
4581 // FIXME: use signedRegI mask, remove tmp?
4582 instruct loadI2L_immI(iRegL dst, memoryI mem, immI mask, iRegI tmp) %{
4583   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
4584   effect(TEMP dst, TEMP tmp);
4585 
4586   ins_cost(MEMORY_REF_COST + 3*DEFAULT_COST);
4587   format %{ "LDRSW    $mem,$dst\t! int & 31-bit mask -> long\n\t"
4588             "MOV_SLOW $tmp,$mask\n\t"
4589             "AND      $dst,$tmp,$dst" %}
4590   ins_encode %{
4591     __ ldrsw($dst$$Register, $mem$$Address);
4592     __ mov_slow($tmp$$Register, $mask$$constant);
4593     __ andr($dst$$Register, $dst$$Register, $tmp$$Register);
4594   %}
4595   ins_pipe(iload_mem);
4596 %}
4597 #else
4598 // Load Integer with a 31-bit mask into a Long Register
4599 // FIXME: use iRegI mask, remove tmp?
4600 instruct loadI2L_immU31(iRegL dst, memoryI mem, immU31 mask, iRegI tmp) %{
4601   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
4602   effect(TEMP dst, TEMP tmp);
4603 
4604   ins_cost(MEMORY_REF_COST + 4*DEFAULT_COST);
4605   size(20);
4606   format %{ "LDR      $mem,$dst\t! int & 31-bit mask -> long\n\t"
4607             "MOV      $dst.hi, 0\n\t"
4608             "MOV_SLOW $tmp,$mask\n\t"
4609             "AND      $dst,$tmp,$dst" %}
4610   ins_encode %{
4611     __ ldr($dst$$Register, $mem$$Address);
4612     __ mov($dst$$Register->successor(), 0);
4613     __ mov_slow($tmp$$Register, $mask$$constant);
4614     __ andr($dst$$Register, $dst$$Register, $tmp$$Register);
4615   %}
4616   ins_pipe(iload_mem);
4617 %}
4618 #endif
4619 
4620 // Load Unsigned Integer into a Long Register
4621 instruct loadUI2L(iRegL dst, memoryI mem, immL_32bits mask) %{
4622   match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
4623   ins_cost(MEMORY_REF_COST);
4624 
4625 #ifdef AARCH64
4626 //size(4);
4627   format %{ "LDR_w $dst,$mem\t! uint -> long" %}
4628   ins_encode %{
4629     __ ldr_w($dst$$Register, $mem$$Address);
4630   %}
4631 #else
4632   size(8);
4633   format %{ "LDR   $dst.lo,$mem\t! uint -> long\n\t"
4634             "MOV   $dst.hi,0" %}
4635   ins_encode %{
4636     __ ldr($dst$$Register, $mem$$Address);
4637     __ mov($dst$$Register->successor(), 0);
4638   %}
4639 #endif
4640   ins_pipe(iload_mem);
4641 %}
4642 
4643 // Load Long
4644 
4645 #ifdef AARCH64
4646 // XXX This variant shouldn't be necessary if 6217251 is implemented
4647 instruct loadLoff(iRegLd dst, memoryScaledL mem, aimmX off, iRegP tmp) %{
4648   match(Set dst (LoadL (AddP mem off)));
4649   ins_cost(MEMORY_REF_COST + DEFAULT_COST); // assume shift/sign-extend is free
4650   effect(TEMP tmp);
4651   size(4 * 2);
4652 
4653   format %{ "LDR    $dst,$mem+$off\t! long temp=$tmp" %}
4654   ins_encode %{
4655     Register base = reg_to_register_object($mem$$base);
4656     __ add($tmp$$Register, base, $off$$constant);
4657     Address nmem = Address::make_raw($tmp$$reg, $mem$$index, $mem$$scale, $mem$$disp, relocInfo::none);
4658     __ ldr($dst$$Register, nmem);
4659   %}
4660   ins_pipe(iload_mem);
4661 %}
4662 #endif
4663 
4664 instruct loadL(iRegLd dst, memoryL mem ) %{
4665 #ifdef AARCH64
4666   // already atomic for Aarch64
4667 #else
4668   predicate(!((LoadLNode*)n)->require_atomic_access());
4669 #endif
4670   match(Set dst (LoadL mem));
4671   effect(TEMP dst);
4672   ins_cost(MEMORY_REF_COST);
4673 
4674   size(4);
4675   format %{ "ldr_64  $dst,$mem\t! long" %}
4676   ins_encode %{
4677     __ ldr_64($dst$$Register, $mem$$Address);
4678   %}
4679   ins_pipe(iload_mem);
4680 %}
4681 
4682 #ifndef AARCH64
4683 instruct loadL_2instr(iRegL dst, memorylong mem ) %{
4684   predicate(!((LoadLNode*)n)->require_atomic_access());
4685   match(Set dst (LoadL mem));
4686   ins_cost(MEMORY_REF_COST + DEFAULT_COST);
4687 
4688   size(8);
4689   format %{ "LDR    $dst.lo,$mem \t! long order of instrs reversed if $dst.lo == base($mem)\n\t"
4690             "LDR    $dst.hi,$mem+4 or $mem" %}
4691   ins_encode %{
4692     Address Amemlo = Address::make_raw($mem$$base, $mem$$index, $mem$$scale, $mem$$disp, relocInfo::none);
4693     Address Amemhi = Address::make_raw($mem$$base, $mem$$index, $mem$$scale, $mem$$disp + 4, relocInfo::none);
4694 
4695     if ($dst$$Register == reg_to_register_object($mem$$base)) {
4696       __ ldr($dst$$Register->successor(), Amemhi);
4697       __ ldr($dst$$Register, Amemlo);
4698     } else {
4699       __ ldr($dst$$Register, Amemlo);
4700       __ ldr($dst$$Register->successor(), Amemhi);
4701     }
4702   %}
4703   ins_pipe(iload_mem);
4704 %}
4705 
4706 instruct loadL_volatile(iRegL dst, indirect mem ) %{
4707   predicate(((LoadLNode*)n)->require_atomic_access());
4708   match(Set dst (LoadL mem));
4709   ins_cost(MEMORY_REF_COST);
4710 
4711   size(4);
4712   format %{ "LDMIA    $dst,$mem\t! long" %}
4713   ins_encode %{
4714     // FIXME: why is ldmia considered atomic?  Should be ldrexd
4715     RegisterSet set($dst$$Register);
4716     set = set | reg_to_register_object($dst$$reg + 1);
4717     __ ldmia(reg_to_register_object($mem$$base), set);
4718   %}
4719   ins_pipe(iload_mem);
4720 %}
4721 
4722 instruct loadL_volatile_fp(iRegL dst, memoryD mem ) %{
4723   predicate(((LoadLNode*)n)->require_atomic_access());
4724   match(Set dst (LoadL mem));
4725   ins_cost(MEMORY_REF_COST);
4726 
4727   size(8);
4728   format %{ "FLDD      S14, $mem"
4729             "FMRRD    $dst, S14\t! long \n't" %}
4730   ins_encode %{
4731     __ fldd(S14, $mem$$Address);
4732     __ fmrrd($dst$$Register, $dst$$Register->successor(), S14);
4733   %}
4734   ins_pipe(iload_mem);
4735 %}
4736 
4737 instruct loadL_unaligned(iRegL dst, memorylong mem ) %{
4738   match(Set dst (LoadL_unaligned mem));
4739   ins_cost(MEMORY_REF_COST);
4740 
4741   size(8);
4742   format %{ "LDR    $dst.lo,$mem\t! long order of instrs reversed if $dst.lo == base($mem)\n\t"
4743             "LDR    $dst.hi,$mem+4" %}
4744   ins_encode %{
4745     Address Amemlo = Address::make_raw($mem$$base, $mem$$index, $mem$$scale, $mem$$disp, relocInfo::none);
4746     Address Amemhi = Address::make_raw($mem$$base, $mem$$index, $mem$$scale, $mem$$disp + 4, relocInfo::none);
4747 
4748     if ($dst$$Register == reg_to_register_object($mem$$base)) {
4749       __ ldr($dst$$Register->successor(), Amemhi);
4750       __ ldr($dst$$Register, Amemlo);
4751     } else {
4752       __ ldr($dst$$Register, Amemlo);
4753       __ ldr($dst$$Register->successor(), Amemhi);
4754     }
4755   %}
4756   ins_pipe(iload_mem);
4757 %}
4758 #endif // !AARCH64
4759 
4760 // Load Range
4761 instruct loadRange(iRegI dst, memoryI mem) %{
4762   match(Set dst (LoadRange mem));
4763   ins_cost(MEMORY_REF_COST);
4764 
4765   size(4);
4766   format %{ "LDR_u32 $dst,$mem\t! range" %}
4767   ins_encode %{
4768     __ ldr_u32($dst$$Register, $mem$$Address);
4769   %}
4770   ins_pipe(iload_mem);
4771 %}
4772 
4773 // Load Pointer
4774 
4775 #ifdef AARCH64
4776 // XXX This variant shouldn't be necessary if 6217251 is implemented
4777 instruct loadPoff(iRegP dst, memoryScaledP mem, aimmX off, iRegP tmp) %{
4778   match(Set dst (LoadP (AddP mem off)));
4779   ins_cost(MEMORY_REF_COST + DEFAULT_COST); // assume shift/sign-extend is free
4780   effect(TEMP tmp);
4781   size(4 * 2);
4782 
4783   format %{ "LDR    $dst,$mem+$off\t! ptr temp=$tmp" %}
4784   ins_encode %{
4785     Register base = reg_to_register_object($mem$$base);
4786     __ add($tmp$$Register, base, $off$$constant);
4787     Address nmem = Address::make_raw($tmp$$reg, $mem$$index, $mem$$scale, $mem$$disp, relocInfo::none);
4788     __ ldr($dst$$Register, nmem);
4789   %}
4790   ins_pipe(iload_mem);
4791 %}
4792 #endif
4793 
4794 instruct loadP(iRegP dst, memoryP mem) %{
4795   match(Set dst (LoadP mem));
4796   ins_cost(MEMORY_REF_COST);
4797   size(4);
4798 
4799   format %{ "LDR   $dst,$mem\t! ptr" %}
4800   ins_encode %{
4801     __ ldr($dst$$Register, $mem$$Address);
4802   %}
4803   ins_pipe(iload_mem);
4804 %}
4805 
4806 #ifdef XXX
4807 // FIXME XXXX
4808 //instruct loadSP(iRegP dst, memoryP mem) %{
4809 instruct loadSP(SPRegP dst, memoryP mem, iRegP tmp) %{
4810   match(Set dst (LoadP mem));
4811   effect(TEMP tmp);
4812   ins_cost(MEMORY_REF_COST+1);
4813   size(8);
4814 
4815   format %{ "LDR   $tmp,$mem\t! ptr\n\t"
4816             "MOV   $dst,$tmp\t! ptr" %}
4817   ins_encode %{
4818     __ ldr($tmp$$Register, $mem$$Address);
4819     __ mov($dst$$Register, $tmp$$Register);
4820   %}
4821   ins_pipe(iload_mem);
4822 %}
4823 #endif
4824 
4825 #ifdef _LP64
4826 // Load Compressed Pointer
4827 
4828 // XXX This variant shouldn't be necessary if 6217251 is implemented
4829 instruct loadNoff(iRegN dst, memoryScaledI mem, aimmX off, iRegP tmp) %{
4830   match(Set dst (LoadN (AddP mem off)));
4831   ins_cost(MEMORY_REF_COST + DEFAULT_COST); // assume shift/sign-extend is free
4832   effect(TEMP tmp);
4833   size(4 * 2);
4834 
4835   format %{ "ldr_u32 $dst,$mem+$off\t! compressed ptr temp=$tmp" %}
4836   ins_encode %{
4837     Register base = reg_to_register_object($mem$$base);
4838     __ add($tmp$$Register, base, $off$$constant);
4839     Address nmem = Address::make_raw($tmp$$reg, $mem$$index, $mem$$scale, $mem$$disp, relocInfo::none);
4840     __ ldr_u32($dst$$Register, nmem);
4841   %}
4842   ins_pipe(iload_mem);
4843 %}
4844 
4845 instruct loadN(iRegN dst, memoryI mem) %{
4846   match(Set dst (LoadN mem));
4847   ins_cost(MEMORY_REF_COST);
4848   size(4);
4849 
4850   format %{ "ldr_u32 $dst,$mem\t! compressed ptr" %}
4851   ins_encode %{
4852     __ ldr_u32($dst$$Register, $mem$$Address);
4853   %}
4854   ins_pipe(iload_mem);
4855 %}
4856 #endif
4857 
4858 // Load Klass Pointer
4859 instruct loadKlass(iRegP dst, memoryI mem) %{
4860   match(Set dst (LoadKlass mem));
4861   ins_cost(MEMORY_REF_COST);
4862   size(4);
4863 
4864   format %{ "LDR   $dst,$mem\t! klass ptr" %}
4865   ins_encode %{
4866     __ ldr($dst$$Register, $mem$$Address);
4867   %}
4868   ins_pipe(iload_mem);
4869 %}
4870 
4871 #ifdef _LP64
4872 // Load narrow Klass Pointer
4873 instruct loadNKlass(iRegN dst, memoryI mem) %{
4874   match(Set dst (LoadNKlass mem));
4875   ins_cost(MEMORY_REF_COST);
4876   size(4);
4877 
4878   format %{ "ldr_u32 $dst,$mem\t! compressed klass ptr" %}
4879   ins_encode %{
4880     __ ldr_u32($dst$$Register, $mem$$Address);
4881   %}
4882   ins_pipe(iload_mem);
4883 %}
4884 #endif
4885 
4886 #ifdef AARCH64
4887 // XXX This variant shouldn't be necessary if 6217251 is implemented
4888 instruct loadDoff(regD dst, memoryScaledD mem, aimmX off, iRegP tmp) %{
4889   match(Set dst (LoadD (AddP mem off)));
4890   ins_cost(MEMORY_REF_COST + DEFAULT_COST); // assume shift/sign-extend is free
4891   effect(TEMP tmp);
4892   size(4 * 2);
4893 
4894   format %{ "ldr    $dst,$mem+$off\t! double temp=$tmp" %}
4895   ins_encode %{
4896     Register base = reg_to_register_object($mem$$base);
4897     __ add($tmp$$Register, base, $off$$constant);
4898     Address nmem = Address::make_raw($tmp$$reg, $mem$$index, $mem$$scale, $mem$$disp, relocInfo::none);
4899     __ ldr_d($dst$$FloatRegister, nmem);
4900   %}
4901   ins_pipe(floadD_mem);
4902 %}
4903 #endif
4904 
4905 instruct loadD(regD dst, memoryD mem) %{
4906   match(Set dst (LoadD mem));
4907   ins_cost(MEMORY_REF_COST);
4908 
4909   size(4);
4910   // FIXME: needs to be atomic, but  ARMv7 A.R.M. guarantees
4911   // only LDREXD and STREXD are 64-bit single-copy atomic
4912   format %{ "FLDD   $dst,$mem" %}
4913   ins_encode %{
4914     __ ldr_double($dst$$FloatRegister, $mem$$Address);
4915   %}
4916   ins_pipe(floadD_mem);
4917 %}
4918 
4919 #ifndef AARCH64
4920 // Load Double - UNaligned
4921 instruct loadD_unaligned(regD_low dst, memoryF2 mem ) %{
4922   match(Set dst (LoadD_unaligned mem));
4923   ins_cost(MEMORY_REF_COST*2+DEFAULT_COST);
4924   size(8);
4925   format %{ "FLDS    $dst.lo,$mem\t! misaligned double\n"
4926           "\tFLDS    $dst.hi,$mem+4\t!" %}
4927   ins_encode %{
4928     Address Amemlo = Address::make_raw($mem$$base, $mem$$index, $mem$$scale, $mem$$disp, relocInfo::none);
4929     Address Amemhi = Address::make_raw($mem$$base, $mem$$index, $mem$$scale, $mem$$disp + 4, relocInfo::none);
4930       __ flds($dst$$FloatRegister, Amemlo);
4931       __ flds($dst$$FloatRegister->successor(), Amemhi);
4932   %}
4933   ins_pipe(iload_mem);
4934 %}
4935 #endif
4936 
4937 #ifdef AARCH64
4938 // XXX This variant shouldn't be necessary if 6217251 is implemented
4939 instruct loadFoff(regF dst, memoryScaledF mem, aimmX off, iRegP tmp) %{
4940   match(Set dst (LoadF (AddP mem off)));
4941   ins_cost(MEMORY_REF_COST + DEFAULT_COST); // assume shift/sign-extend is free
4942   effect(TEMP tmp);
4943   size(4 * 2);
4944 
4945   format %{ "ldr    $dst,$mem+$off\t! float temp=$tmp" %}
4946   ins_encode %{
4947     Register base = reg_to_register_object($mem$$base);
4948     __ add($tmp$$Register, base, $off$$constant);
4949     Address nmem = Address::make_raw($tmp$$reg, $mem$$index, $mem$$scale, $mem$$disp, relocInfo::none);
4950     __ ldr_s($dst$$FloatRegister, nmem);
4951   %}
4952   ins_pipe(floadF_mem);
4953 %}
4954 #endif
4955 
4956 instruct loadF(regF dst, memoryF mem) %{
4957   match(Set dst (LoadF mem));
4958 
4959   ins_cost(MEMORY_REF_COST);
4960   size(4);
4961   format %{ "FLDS    $dst,$mem" %}
4962   ins_encode %{
4963     __ ldr_float($dst$$FloatRegister, $mem$$Address);
4964   %}
4965   ins_pipe(floadF_mem);
4966 %}
4967 
4968 #ifdef AARCH64
4969 instruct load_limmI(iRegI dst, limmI src) %{
4970   match(Set dst src);
4971   ins_cost(DEFAULT_COST + 1); // + 1 because MOV is preferred
4972   format %{ "ORR_w  $dst, ZR, $src\t! int"  %}
4973   ins_encode %{
4974     __ orr_w($dst$$Register, ZR, (uintx)$src$$constant);
4975   %}
4976   ins_pipe(ialu_imm);
4977 %}
4978 #endif
4979 
4980 // // Load Constant
4981 instruct loadConI( iRegI dst, immI src ) %{
4982   match(Set dst src);
4983   ins_cost(DEFAULT_COST * 3/2);
4984   format %{ "MOV_SLOW    $dst, $src" %}
4985   ins_encode %{
4986     __ mov_slow($dst$$Register, $src$$constant);
4987   %}
4988   ins_pipe(ialu_hi_lo_reg);
4989 %}
4990 
4991 instruct loadConIMov( iRegI dst, immIMov src ) %{
4992   match(Set dst src);
4993   size(4);
4994   format %{ "MOV    $dst, $src" %}
4995   ins_encode %{
4996     __ mov($dst$$Register, $src$$constant);
4997   %}
4998   ins_pipe(ialu_imm);
4999 %}
5000 
5001 #ifndef AARCH64
5002 instruct loadConIMovn( iRegI dst, immIRotn src ) %{
5003   match(Set dst src);
5004   size(4);
5005   format %{ "MVN    $dst, ~$src" %}
5006   ins_encode %{
5007     __ mvn($dst$$Register, ~$src$$constant);
5008   %}
5009   ins_pipe(ialu_imm_n);
5010 %}
5011 #endif
5012 
5013 instruct loadConI16( iRegI dst, immI16 src ) %{
5014   match(Set dst src);
5015   size(4);
5016 #ifdef AARCH64
5017   format %{ "MOVZ_w  $dst, $src" %}
5018 #else
5019   format %{ "MOVW    $dst, $src" %}
5020 #endif
5021   ins_encode %{
5022 #ifdef AARCH64
5023     __ mov_w($dst$$Register, $src$$constant);
5024 #else
5025     __ movw($dst$$Register, $src$$constant);
5026 #endif
5027   %}
5028   ins_pipe(ialu_imm_n);
5029 %}
5030 
5031 instruct loadConP(iRegP dst, immP src) %{
5032   match(Set dst src);
5033   ins_cost(DEFAULT_COST * 3/2);
5034   format %{ "MOV_SLOW    $dst,$src\t!ptr" %}
5035   ins_encode %{
5036     relocInfo::relocType constant_reloc = _opnds[1]->constant_reloc();
5037     intptr_t val = $src$$constant;
5038     if (constant_reloc == relocInfo::oop_type) {
5039       __ mov_oop($dst$$Register, (jobject)val);
5040     } else if (constant_reloc == relocInfo::metadata_type) {
5041       __ mov_metadata($dst$$Register, (Metadata*)val);
5042     } else {
5043       __ mov_slow($dst$$Register, val);
5044     }
5045   %}
5046   ins_pipe(loadConP);
5047 %}
5048 
5049 
5050 instruct loadConP_poll(iRegP dst, immP_poll src) %{
5051   match(Set dst src);
5052   ins_cost(DEFAULT_COST);
5053   format %{ "MOV_SLOW    $dst,$src\t!ptr" %}
5054   ins_encode %{
5055       __ mov_slow($dst$$Register, $src$$constant);
5056   %}
5057   ins_pipe(loadConP_poll);
5058 %}
5059 
5060 #ifdef AARCH64
5061 instruct loadConP0(iRegP dst, immP0 src) %{
5062   match(Set dst src);
5063   ins_cost(DEFAULT_COST);
5064   format %{ "MOV    $dst,ZR\t!ptr" %}
5065   ins_encode %{
5066     __ mov($dst$$Register, ZR);
5067   %}
5068   ins_pipe(ialu_none);
5069 %}
5070 
5071 instruct loadConN(iRegN dst, immN src) %{
5072   match(Set dst src);
5073   ins_cost(DEFAULT_COST * 3/2);
5074   format %{ "SET    $dst,$src\t! compressed ptr" %}
5075   ins_encode %{
5076     Register dst = $dst$$Register;
5077     // FIXME: use $constanttablebase?
5078     __ set_narrow_oop(dst, (jobject)$src$$constant);
5079   %}
5080   ins_pipe(ialu_hi_lo_reg);
5081 %}
5082 
5083 instruct loadConN0(iRegN dst, immN0 src) %{
5084   match(Set dst src);
5085   ins_cost(DEFAULT_COST);
5086   format %{ "MOV    $dst,ZR\t! compressed ptr" %}
5087   ins_encode %{
5088     __ mov($dst$$Register, ZR);
5089   %}
5090   ins_pipe(ialu_none);
5091 %}
5092 
5093 instruct loadConNKlass(iRegN dst, immNKlass src) %{
5094   match(Set dst src);
5095   ins_cost(DEFAULT_COST * 3/2);
5096   format %{ "SET    $dst,$src\t! compressed klass ptr" %}
5097   ins_encode %{
5098     Register dst = $dst$$Register;
5099     // FIXME: use $constanttablebase?
5100     __ set_narrow_klass(dst, (Klass*)$src$$constant);
5101   %}
5102   ins_pipe(ialu_hi_lo_reg);
5103 %}
5104 
5105 instruct load_limmL(iRegL dst, limmL src) %{
5106   match(Set dst src);
5107   ins_cost(DEFAULT_COST);
5108   format %{ "ORR    $dst, ZR, $src\t! long"  %}
5109   ins_encode %{
5110     __ orr($dst$$Register, ZR, (uintx)$src$$constant);
5111   %}
5112   ins_pipe(loadConL);
5113 %}
5114 instruct load_immLMov(iRegL dst, immLMov src) %{
5115   match(Set dst src);
5116   ins_cost(DEFAULT_COST);
5117   format %{ "MOV    $dst, $src\t! long"  %}
5118   ins_encode %{
5119     __ mov($dst$$Register, $src$$constant);
5120   %}
5121   ins_pipe(loadConL);
5122 %}
5123 instruct loadConL(iRegL dst, immL src) %{
5124   match(Set dst src);
5125   ins_cost(DEFAULT_COST * 4); // worst case
5126   format %{ "mov_slow   $dst, $src\t! long"  %}
5127   ins_encode %{
5128     // FIXME: use $constanttablebase?
5129     __ mov_slow($dst$$Register, $src$$constant);
5130   %}
5131   ins_pipe(loadConL);
5132 %}
5133 #else
5134 instruct loadConL(iRegL dst, immL src) %{
5135   match(Set dst src);
5136   ins_cost(DEFAULT_COST * 4);
5137   format %{ "MOV_SLOW   $dst.lo, $src & 0x0FFFFFFFFL \t! long\n\t"
5138             "MOV_SLOW   $dst.hi, $src >> 32" %}
5139   ins_encode %{
5140     __ mov_slow(reg_to_register_object($dst$$reg), $src$$constant & 0x0FFFFFFFFL);
5141     __ mov_slow(reg_to_register_object($dst$$reg + 1), ((julong)($src$$constant)) >> 32);
5142   %}
5143   ins_pipe(loadConL);
5144 %}
5145 
5146 instruct loadConL16( iRegL dst, immL16 src ) %{
5147   match(Set dst src);
5148   ins_cost(DEFAULT_COST * 2);
5149 
5150   size(8);
5151   format %{ "MOVW    $dst.lo, $src \n\t"
5152             "MOVW    $dst.hi, 0 \n\t" %}
5153   ins_encode %{
5154     __ movw($dst$$Register, $src$$constant);
5155     __ movw($dst$$Register->successor(), 0);
5156   %}
5157   ins_pipe(ialu_imm);
5158 %}
5159 #endif
5160 
5161 instruct loadConF_imm8(regF dst, imm8F src) %{
5162   match(Set dst src);
5163   ins_cost(DEFAULT_COST);
5164   size(4);
5165 
5166   format %{ "FCONSTS      $dst, $src"%}
5167 
5168   ins_encode %{
5169     __ fconsts($dst$$FloatRegister, Assembler::float_num($src$$constant).imm8());
5170   %}
5171   ins_pipe(loadConFD); // FIXME
5172 %}
5173 
5174 #ifdef AARCH64
5175 instruct loadIConF(iRegI dst, immF src) %{
5176   match(Set dst src);
5177   ins_cost(DEFAULT_COST * 2);
5178 
5179   format %{ "MOV_SLOW  $dst, $src\t! loadIConF"  %}
5180 
5181   ins_encode %{
5182     // FIXME revisit once 6961697 is in
5183     union {
5184       jfloat f;
5185       int i;
5186     } v;
5187     v.f = $src$$constant;
5188     __ mov_slow($dst$$Register, v.i);
5189   %}
5190   ins_pipe(ialu_imm);
5191 %}
5192 #endif
5193 
5194 instruct loadConF(regF dst, immF src, iRegI tmp) %{
5195   match(Set dst src);
5196   ins_cost(DEFAULT_COST * 2);
5197   effect(TEMP tmp);
5198   size(3*4);
5199 
5200   format %{ "MOV_SLOW  $tmp, $src\n\t"
5201             "FMSR      $dst, $tmp"%}
5202 
5203   ins_encode %{
5204     // FIXME revisit once 6961697 is in
5205     union {
5206       jfloat f;
5207       int i;
5208     } v;
5209     v.f = $src$$constant;
5210     __ mov_slow($tmp$$Register, v.i);
5211     __ fmsr($dst$$FloatRegister, $tmp$$Register);
5212   %}
5213   ins_pipe(loadConFD); // FIXME
5214 %}
5215 
5216 instruct loadConD_imm8(regD dst, imm8D src) %{
5217   match(Set dst src);
5218   ins_cost(DEFAULT_COST);
5219   size(4);
5220 
5221   format %{ "FCONSTD      $dst, $src"%}
5222 
5223   ins_encode %{
5224     __ fconstd($dst$$FloatRegister, Assembler::double_num($src$$constant).imm8());
5225   %}
5226   ins_pipe(loadConFD); // FIXME
5227 %}
5228 
5229 instruct loadConD(regD dst, immD src, iRegP tmp) %{
5230   match(Set dst src);
5231   effect(TEMP tmp);
5232   ins_cost(MEMORY_REF_COST);
5233   format %{ "FLDD  $dst, [$constanttablebase + $constantoffset]\t! load from constant table: double=$src" %}
5234 
5235   ins_encode %{
5236     Register r = $constanttablebase;
5237     int offset  = $constantoffset($src);
5238     if (!is_memoryD(offset)) {                // can't use a predicate
5239                                               // in load constant instructs
5240       __ add_slow($tmp$$Register, r, offset);
5241       r = $tmp$$Register;
5242       offset = 0;
5243     }
5244     __ ldr_double($dst$$FloatRegister, Address(r, offset));
5245   %}
5246   ins_pipe(loadConFD);
5247 %}
5248 
5249 // Prefetch instructions.
5250 // Must be safe to execute with invalid address (cannot fault).
5251 
5252 instruct prefetchAlloc_mp( memoryP mem ) %{
5253   predicate(os::is_MP());
5254   match( PrefetchAllocation mem );
5255   ins_cost(MEMORY_REF_COST);
5256   size(4);
5257 
5258   format %{ "PLDW $mem\t! Prefetch allocation" %}
5259   ins_encode %{
5260 #ifdef AARCH64
5261     __ prfm(pstl1keep, $mem$$Address);
5262 #else
5263     __ pldw($mem$$Address);
5264 #endif
5265   %}
5266   ins_pipe(iload_mem);
5267 %}
5268 
5269 instruct prefetchAlloc_sp( memoryP mem ) %{
5270   predicate(!os::is_MP());
5271   match( PrefetchAllocation mem );
5272   ins_cost(MEMORY_REF_COST);
5273   size(4);
5274 
5275   format %{ "PLD $mem\t! Prefetch allocation" %}
5276   ins_encode %{
5277 #ifdef AARCH64
5278     __ prfm(pstl1keep, $mem$$Address);
5279 #else
5280     __ pld($mem$$Address);
5281 #endif
5282   %}
5283   ins_pipe(iload_mem);
5284 %}
5285 
5286 //----------Store Instructions-------------------------------------------------
5287 // Store Byte
5288 instruct storeB(memoryB mem, store_RegI src) %{
5289   match(Set mem (StoreB mem src));
5290   ins_cost(MEMORY_REF_COST);
5291 
5292   size(4);
5293   format %{ "STRB    $src,$mem\t! byte" %}
5294   ins_encode %{
5295     __ strb($src$$Register, $mem$$Address);
5296   %}
5297   ins_pipe(istore_mem_reg);
5298 %}
5299 
5300 instruct storeCM(memoryB mem, store_RegI src) %{
5301   match(Set mem (StoreCM mem src));
5302   ins_cost(MEMORY_REF_COST);
5303 
5304   size(4);
5305   format %{ "STRB    $src,$mem\t! CMS card-mark byte" %}
5306   ins_encode %{
5307     __ strb($src$$Register, $mem$$Address);
5308   %}
5309   ins_pipe(istore_mem_reg);
5310 %}
5311 
5312 // Store Char/Short
5313 
5314 #ifdef AARCH64
5315 // XXX This variant shouldn't be necessary if 6217251 is implemented
5316 instruct storeCoff(store_RegI src, memoryScaledS mem, aimmX off, iRegP tmp) %{
5317   match(Set mem (StoreC (AddP mem off) src));
5318   ins_cost(MEMORY_REF_COST + DEFAULT_COST); // assume shift/sign-extend is free
5319   effect(TEMP tmp);
5320   size(4 * 2);
5321 
5322   format %{ "STRH    $src,$mem+$off\t! short temp=$tmp" %}
5323   ins_encode %{
5324     Register base = reg_to_register_object($mem$$base);
5325     __ add($tmp$$Register, base, $off$$constant);
5326     Address nmem = Address::make_raw($tmp$$reg, $mem$$index, $mem$$scale, $mem$$disp, relocInfo::none);
5327     __ strh($src$$Register, nmem);
5328   %}
5329   ins_pipe(istore_mem_reg);
5330 %}
5331 #endif
5332 
5333 instruct storeC(memoryS mem, store_RegI src) %{
5334   match(Set mem (StoreC mem src));
5335   ins_cost(MEMORY_REF_COST);
5336 
5337   size(4);
5338   format %{ "STRH    $src,$mem\t! short" %}
5339   ins_encode %{
5340     __ strh($src$$Register, $mem$$Address);
5341   %}
5342   ins_pipe(istore_mem_reg);
5343 %}
5344 
5345 // Store Integer
5346 
5347 #ifdef AARCH64
5348 // XXX This variant shouldn't be necessary if 6217251 is implemented
5349 instruct storeIoff(store_RegI src, memoryScaledI mem, aimmX off, iRegP tmp) %{
5350   match(Set mem (StoreI (AddP mem off) src));
5351   ins_cost(MEMORY_REF_COST + DEFAULT_COST); // assume shift/sign-extend is free
5352   effect(TEMP tmp);
5353   size(4 * 2);
5354 
5355   format %{ "str_32 $src,$mem+$off\t! int temp=$tmp" %}
5356   ins_encode %{
5357     Register base = reg_to_register_object($mem$$base);
5358     __ add($tmp$$Register, base, $off$$constant);
5359     Address nmem = Address::make_raw($tmp$$reg, $mem$$index, $mem$$scale, $mem$$disp, relocInfo::none);
5360     __ str_32($src$$Register, nmem);
5361   %}
5362   ins_pipe(istore_mem_reg);
5363 %}
5364 #endif
5365 
5366 instruct storeI(memoryI mem, store_RegI src) %{
5367   match(Set mem (StoreI mem src));
5368   ins_cost(MEMORY_REF_COST);
5369 
5370   size(4);
5371   format %{ "str_32 $src,$mem" %}
5372   ins_encode %{
5373     __ str_32($src$$Register, $mem$$Address);
5374   %}
5375   ins_pipe(istore_mem_reg);
5376 %}
5377 
5378 // Store Long
5379 
5380 #ifdef AARCH64
5381 // XXX This variant shouldn't be necessary if 6217251 is implemented
5382 instruct storeLoff(store_RegLd src, memoryScaledL mem, aimmX off, iRegP tmp) %{
5383   match(Set mem (StoreL (AddP mem off) src));
5384   ins_cost(MEMORY_REF_COST + DEFAULT_COST); // assume shift/sign-extend is free
5385   effect(TEMP tmp);
5386   size(4 * 2);
5387 
5388   format %{ "str_64 $src,$mem+$off\t! long temp=$tmp" %}
5389   ins_encode %{
5390     Register base = reg_to_register_object($mem$$base);
5391     __ add($tmp$$Register, base, $off$$constant);
5392     Address nmem = Address::make_raw($tmp$$reg, $mem$$index, $mem$$scale, $mem$$disp, relocInfo::none);
5393     __ str_64($src$$Register, nmem);
5394   %}
5395   ins_pipe(istore_mem_reg);
5396 %}
5397 #endif
5398 
5399 instruct storeL(memoryL mem, store_RegLd src) %{
5400 #ifdef AARCH64
5401   // already atomic for Aarch64
5402 #else
5403   predicate(!((StoreLNode*)n)->require_atomic_access());
5404 #endif
5405   match(Set mem (StoreL mem src));
5406   ins_cost(MEMORY_REF_COST);
5407 
5408   size(4);
5409   format %{ "str_64  $src,$mem\t! long\n\t" %}
5410 
5411   ins_encode %{
5412     __ str_64($src$$Register, $mem$$Address);
5413   %}
5414   ins_pipe(istore_mem_reg);
5415 %}
5416 
5417 #ifndef AARCH64
5418 instruct storeL_2instr(memorylong mem, iRegL src) %{
5419   predicate(!((StoreLNode*)n)->require_atomic_access());
5420   match(Set mem (StoreL mem src));
5421   ins_cost(MEMORY_REF_COST + DEFAULT_COST);
5422 
5423   size(8);
5424   format %{ "STR    $src.lo,$mem\t! long\n\t"
5425             "STR    $src.hi,$mem+4" %}
5426 
5427   ins_encode %{
5428     Address Amemlo = Address::make_raw($mem$$base, $mem$$index, $mem$$scale, $mem$$disp, relocInfo::none);
5429     Address Amemhi = Address::make_raw($mem$$base, $mem$$index, $mem$$scale, $mem$$disp + 4, relocInfo::none);
5430     __ str($src$$Register, Amemlo);
5431     __ str($src$$Register->successor(), Amemhi);
5432   %}
5433   ins_pipe(istore_mem_reg);
5434 %}
5435 
5436 instruct storeL_volatile(indirect mem, iRegL src) %{
5437   predicate(((StoreLNode*)n)->require_atomic_access());
5438   match(Set mem (StoreL mem src));
5439   ins_cost(MEMORY_REF_COST);
5440   size(4);
5441   format %{ "STMIA    $src,$mem\t! long" %}
5442   ins_encode %{
5443     // FIXME: why is stmia considered atomic?  Should be strexd
5444     RegisterSet set($src$$Register);
5445     set = set | reg_to_register_object($src$$reg + 1);
5446     __ stmia(reg_to_register_object($mem$$base), set);
5447   %}
5448   ins_pipe(istore_mem_reg);
5449 %}
5450 #endif // !AARCH64
5451 
5452 #ifndef AARCH64
5453 instruct storeL_volatile_fp(memoryD mem, iRegL src) %{
5454   predicate(((StoreLNode*)n)->require_atomic_access());
5455   match(Set mem (StoreL mem src));
5456   ins_cost(MEMORY_REF_COST);
5457   size(8);
5458   format %{ "FMDRR    S14, $src\t! long \n\t"
5459             "FSTD     S14, $mem" %}
5460   ins_encode %{
5461     __ fmdrr(S14, $src$$Register, $src$$Register->successor());
5462     __ fstd(S14, $mem$$Address);
5463   %}
5464   ins_pipe(istore_mem_reg);
5465 %}
5466 #endif
5467 
5468 #ifdef XXX
5469 // Move SP Pointer
5470 //instruct movSP(sp_ptr_RegP dst, SPRegP src) %{
5471 //instruct movSP(iRegP dst, SPRegP src) %{
5472 instruct movSP(store_ptr_RegP dst, SPRegP src) %{
5473   match(Set dst src);
5474 //predicate(!_kids[1]->_leaf->is_Proj() || _kids[1]->_leaf->as_Proj()->_con == TypeFunc::FramePtr);
5475   ins_cost(MEMORY_REF_COST);
5476   size(4);
5477 
5478   format %{ "MOV    $dst,$src\t! SP ptr\n\t" %}
5479   ins_encode %{
5480     assert(false, "XXX1 got here");
5481     __ mov($dst$$Register, SP);
5482     __ mov($dst$$Register, $src$$Register);
5483   %}
5484   ins_pipe(ialu_reg);
5485 %}
5486 #endif
5487 
5488 #ifdef AARCH64
5489 // FIXME
5490 // Store SP Pointer
5491 instruct storeSP(memoryP mem, SPRegP src, iRegP tmp) %{
5492   match(Set mem (StoreP mem src));
5493   predicate(_kids[1]->_leaf->is_Proj() && _kids[1]->_leaf->as_Proj()->_con == TypeFunc::FramePtr);
5494   // Multiple StoreP rules, different only in register mask.
5495   // Matcher makes the last always valid.  The others will
5496   // only be valid if they cost less than the last valid
5497   // rule.  So cost(rule1) < cost(rule2) < cost(last)
5498   // Unlike immediates, register constraints are not checked
5499   // at match time.
5500   ins_cost(MEMORY_REF_COST+DEFAULT_COST+4);
5501   effect(TEMP tmp);
5502   size(8);
5503 
5504   format %{ "MOV    $tmp,$src\t! SP ptr\n\t"
5505             "STR    $tmp,$mem\t! SP ptr" %}
5506   ins_encode %{
5507     assert($src$$Register == SP, "SP expected");
5508     __ mov($tmp$$Register, $src$$Register);
5509     __ str($tmp$$Register, $mem$$Address);
5510   %}
5511   ins_pipe(istore_mem_spORreg); // FIXME
5512 %}
5513 #endif // AARCH64
5514 
5515 // Store Pointer
5516 
5517 #ifdef AARCH64
5518 // XXX This variant shouldn't be necessary if 6217251 is implemented
5519 instruct storePoff(store_ptr_RegP src, memoryScaledP mem, aimmX off, iRegP tmp) %{
5520   predicate(!_kids[1]->_leaf->is_Proj() || _kids[1]->_leaf->as_Proj()->_con != TypeFunc::FramePtr);
5521   match(Set mem (StoreP (AddP mem off) src));
5522   ins_cost(MEMORY_REF_COST + DEFAULT_COST); // assume shift/sign-extend is free
5523   effect(TEMP tmp);
5524   size(4 * 2);
5525 
5526   format %{ "STR    $src,$mem+$off\t! ptr temp=$tmp" %}
5527   ins_encode %{
5528     Register base = reg_to_register_object($mem$$base);
5529     __ add($tmp$$Register, base, $off$$constant);
5530     Address nmem = Address::make_raw($tmp$$reg, $mem$$index, $mem$$scale, $mem$$disp, relocInfo::none);
5531     __ str($src$$Register, nmem);
5532   %}
5533   ins_pipe(istore_mem_reg);
5534 %}
5535 #endif
5536 
5537 instruct storeP(memoryP mem, store_ptr_RegP src) %{
5538   match(Set mem (StoreP mem src));
5539 #ifdef AARCH64
5540   predicate(!_kids[1]->_leaf->is_Proj() || _kids[1]->_leaf->as_Proj()->_con != TypeFunc::FramePtr);
5541 #endif
5542   ins_cost(MEMORY_REF_COST);
5543   size(4);
5544 
5545   format %{ "STR    $src,$mem\t! ptr" %}
5546   ins_encode %{
5547     __ str($src$$Register, $mem$$Address);
5548   %}
5549   ins_pipe(istore_mem_spORreg);
5550 %}
5551 
5552 #ifdef AARCH64
5553 // Store NULL Pointer
5554 instruct storeP0(memoryP mem, immP0 src) %{
5555   match(Set mem (StoreP mem src));
5556   ins_cost(MEMORY_REF_COST);
5557   size(4);
5558 
5559   format %{ "STR    ZR,$mem\t! ptr" %}
5560   ins_encode %{
5561     __ str(ZR, $mem$$Address);
5562   %}
5563   ins_pipe(istore_mem_spORreg);
5564 %}
5565 #endif // AARCH64
5566 
5567 #ifdef _LP64
5568 // Store Compressed Pointer
5569 
5570 #ifdef AARCH64
5571 // XXX This variant shouldn't be necessary if 6217251 is implemented
5572 instruct storeNoff(store_RegN src, memoryScaledI mem, aimmX off, iRegP tmp) %{
5573   match(Set mem (StoreN (AddP mem off) src));
5574   ins_cost(MEMORY_REF_COST + DEFAULT_COST); // assume shift/sign-extend is free
5575   effect(TEMP tmp);
5576   size(4 * 2);
5577 
5578   format %{ "str_32 $src,$mem+$off\t! compressed ptr temp=$tmp" %}
5579   ins_encode %{
5580     Register base = reg_to_register_object($mem$$base);
5581     __ add($tmp$$Register, base, $off$$constant);
5582     Address nmem = Address::make_raw($tmp$$reg, $mem$$index, $mem$$scale, $mem$$disp, relocInfo::none);
5583     __ str_32($src$$Register, nmem);
5584   %}
5585   ins_pipe(istore_mem_reg);
5586 %}
5587 #endif
5588 
5589 instruct storeN(memoryI mem, store_RegN src) %{
5590   match(Set mem (StoreN mem src));
5591   ins_cost(MEMORY_REF_COST);
5592   size(4);
5593 
5594   format %{ "str_32 $src,$mem\t! compressed ptr" %}
5595   ins_encode %{
5596     __ str_32($src$$Register, $mem$$Address);
5597   %}
5598   ins_pipe(istore_mem_reg);
5599 %}
5600 
5601 #ifdef AARCH64
5602 // Store NULL Pointer
5603 instruct storeN0(memoryI mem, immN0 src) %{
5604   match(Set mem (StoreN mem src));
5605   ins_cost(MEMORY_REF_COST);
5606   size(4);
5607 
5608   format %{ "str_32 ZR,$mem\t! compressed ptr" %}
5609   ins_encode %{
5610     __ str_32(ZR, $mem$$Address);
5611   %}
5612   ins_pipe(istore_mem_reg);
5613 %}
5614 #endif
5615 
5616 // Store Compressed Klass Pointer
5617 instruct storeNKlass(memoryI mem, store_RegN src) %{
5618   match(Set mem (StoreNKlass mem src));
5619   ins_cost(MEMORY_REF_COST);
5620   size(4);
5621 
5622   format %{ "str_32 $src,$mem\t! compressed klass ptr" %}
5623   ins_encode %{
5624     __ str_32($src$$Register, $mem$$Address);
5625   %}
5626   ins_pipe(istore_mem_reg);
5627 %}
5628 #endif
5629 
5630 // Store Double
5631 
5632 #ifdef AARCH64
5633 // XXX This variant shouldn't be necessary if 6217251 is implemented
5634 instruct storeDoff(regD src, memoryScaledD mem, aimmX off, iRegP tmp) %{
5635   match(Set mem (StoreD (AddP mem off) src));
5636   ins_cost(MEMORY_REF_COST + DEFAULT_COST); // assume shift/sign-extend is free
5637   effect(TEMP tmp);
5638   size(4 * 2);
5639 
5640   format %{ "STR    $src,$mem+$off\t! double temp=$tmp" %}
5641   ins_encode %{
5642     Register base = reg_to_register_object($mem$$base);
5643     __ add($tmp$$Register, base, $off$$constant);
5644     Address nmem = Address::make_raw($tmp$$reg, $mem$$index, $mem$$scale, $mem$$disp, relocInfo::none);
5645     __ str_d($src$$FloatRegister, nmem);
5646   %}
5647   ins_pipe(fstoreD_mem_reg);
5648 %}
5649 #endif
5650 
5651 instruct storeD(memoryD mem, regD src) %{
5652   match(Set mem (StoreD mem src));
5653   ins_cost(MEMORY_REF_COST);
5654 
5655   size(4);
5656   // FIXME: needs to be atomic, but  ARMv7 A.R.M. guarantees
5657   // only LDREXD and STREXD are 64-bit single-copy atomic
5658   format %{ "FSTD   $src,$mem" %}
5659   ins_encode %{
5660     __ str_double($src$$FloatRegister, $mem$$Address);
5661   %}
5662   ins_pipe(fstoreD_mem_reg);
5663 %}
5664 
5665 #ifdef AARCH64
5666 instruct movI2F(regF dst, iRegI src) %{
5667   match(Set dst src);
5668   size(4);
5669 
5670   format %{ "FMOV_sw $dst,$src\t! movI2F" %}
5671   ins_encode %{
5672     __ fmov_sw($dst$$FloatRegister, $src$$Register);
5673   %}
5674   ins_pipe(ialu_reg); // FIXME
5675 %}
5676 
5677 instruct movF2I(iRegI dst, regF src) %{
5678   match(Set dst src);
5679   size(4);
5680 
5681   format %{ "FMOV_ws $dst,$src\t! movF2I" %}
5682   ins_encode %{
5683     __ fmov_ws($dst$$Register, $src$$FloatRegister);
5684   %}
5685   ins_pipe(ialu_reg); // FIXME
5686 %}
5687 #endif
5688 
5689 // Store Float
5690 
5691 #ifdef AARCH64
5692 // XXX This variant shouldn't be necessary if 6217251 is implemented
5693 instruct storeFoff(regF src, memoryScaledF mem, aimmX off, iRegP tmp) %{
5694   match(Set mem (StoreF (AddP mem off) src));
5695   ins_cost(MEMORY_REF_COST + DEFAULT_COST); // assume shift/sign-extend is free
5696   effect(TEMP tmp);
5697   size(4 * 2);
5698 
5699   format %{ "str_s  $src,$mem+$off\t! float temp=$tmp" %}
5700   ins_encode %{
5701     Register base = reg_to_register_object($mem$$base);
5702     __ add($tmp$$Register, base, $off$$constant);
5703     Address nmem = Address::make_raw($tmp$$reg, $mem$$index, $mem$$scale, $mem$$disp, relocInfo::none);
5704     __ str_s($src$$FloatRegister, nmem);
5705   %}
5706   ins_pipe(fstoreF_mem_reg);
5707 %}
5708 #endif
5709 
5710 instruct storeF( memoryF mem, regF src) %{
5711   match(Set mem (StoreF mem src));
5712   ins_cost(MEMORY_REF_COST);
5713 
5714   size(4);
5715   format %{ "FSTS    $src,$mem" %}
5716   ins_encode %{
5717     __ str_float($src$$FloatRegister, $mem$$Address);
5718   %}
5719   ins_pipe(fstoreF_mem_reg);
5720 %}
5721 
5722 #ifdef AARCH64
5723 // Convert oop pointer into compressed form
5724 instruct encodeHeapOop(iRegN dst, iRegP src, flagsReg ccr) %{
5725   predicate(n->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
5726   match(Set dst (EncodeP src));
5727   effect(KILL ccr);
5728   format %{ "encode_heap_oop $dst, $src" %}
5729   ins_encode %{
5730     __ encode_heap_oop($dst$$Register, $src$$Register);
5731   %}
5732   ins_pipe(ialu_reg);
5733 %}
5734 
5735 instruct encodeHeapOop_not_null(iRegN dst, iRegP src) %{
5736   predicate(n->bottom_type()->make_ptr()->ptr() == TypePtr::NotNull);
5737   match(Set dst (EncodeP src));
5738   format %{ "encode_heap_oop_not_null $dst, $src" %}
5739   ins_encode %{
5740     __ encode_heap_oop_not_null($dst$$Register, $src$$Register);
5741   %}
5742   ins_pipe(ialu_reg);
5743 %}
5744 
5745 instruct decodeHeapOop(iRegP dst, iRegN src, flagsReg ccr) %{
5746   predicate(n->bottom_type()->is_oopptr()->ptr() != TypePtr::NotNull &&
5747             n->bottom_type()->is_oopptr()->ptr() != TypePtr::Constant);
5748   match(Set dst (DecodeN src));
5749   effect(KILL ccr);
5750   format %{ "decode_heap_oop $dst, $src" %}
5751   ins_encode %{
5752     __ decode_heap_oop($dst$$Register, $src$$Register);
5753   %}
5754   ins_pipe(ialu_reg);
5755 %}
5756 
5757 instruct decodeHeapOop_not_null(iRegP dst, iRegN src) %{
5758   predicate(n->bottom_type()->is_oopptr()->ptr() == TypePtr::NotNull ||
5759             n->bottom_type()->is_oopptr()->ptr() == TypePtr::Constant);
5760   match(Set dst (DecodeN src));
5761   format %{ "decode_heap_oop_not_null $dst, $src" %}
5762   ins_encode %{
5763     __ decode_heap_oop_not_null($dst$$Register, $src$$Register);
5764   %}
5765   ins_pipe(ialu_reg);
5766 %}
5767 
5768 instruct encodeKlass_not_null(iRegN dst, iRegP src) %{
5769   match(Set dst (EncodePKlass src));
5770   format %{ "encode_klass_not_null $dst, $src" %}
5771   ins_encode %{
5772     __ encode_klass_not_null($dst$$Register, $src$$Register);
5773   %}
5774   ins_pipe(ialu_reg);
5775 %}
5776 
5777 instruct decodeKlass_not_null(iRegP dst, iRegN src) %{
5778   match(Set dst (DecodeNKlass src));
5779   format %{ "decode_klass_not_null $dst, $src" %}
5780   ins_encode %{
5781     __ decode_klass_not_null($dst$$Register, $src$$Register);
5782   %}
5783   ins_pipe(ialu_reg);
5784 %}
5785 #endif // AARCH64
5786 
5787 //----------MemBar Instructions-----------------------------------------------
5788 // Memory barrier flavors
5789 
5790 // TODO: take advantage of Aarch64 load-acquire, store-release, etc
5791 // pattern-match out unnecessary membars
5792 instruct membar_storestore() %{
5793   match(MemBarStoreStore);
5794   ins_cost(4*MEMORY_REF_COST);
5795 
5796   size(4);
5797   format %{ "MEMBAR-storestore" %}
5798   ins_encode %{
5799     __ membar(MacroAssembler::Membar_mask_bits(MacroAssembler::StoreStore), noreg);
5800   %}
5801   ins_pipe(long_memory_op);
5802 %}
5803 
5804 instruct membar_acquire() %{
5805   match(MemBarAcquire);
5806   match(LoadFence);
5807   ins_cost(4*MEMORY_REF_COST);
5808 
5809   size(4);
5810   format %{ "MEMBAR-acquire" %}
5811   ins_encode %{
5812     __ membar(MacroAssembler::Membar_mask_bits(MacroAssembler::LoadLoad | MacroAssembler::LoadStore), noreg);
5813   %}
5814   ins_pipe(long_memory_op);
5815 %}
5816 
5817 instruct membar_acquire_lock() %{
5818   match(MemBarAcquireLock);
5819   ins_cost(0);
5820 
5821   size(0);
5822   format %{ "!MEMBAR-acquire (CAS in prior FastLock so empty encoding)" %}
5823   ins_encode( );
5824   ins_pipe(empty);
5825 %}
5826 
5827 instruct membar_release() %{
5828   match(MemBarRelease);
5829   match(StoreFence);
5830   ins_cost(4*MEMORY_REF_COST);
5831 
5832   size(4);
5833   format %{ "MEMBAR-release" %}
5834   ins_encode %{
5835     __ membar(MacroAssembler::Membar_mask_bits(MacroAssembler::StoreStore | MacroAssembler::LoadStore), noreg);
5836   %}
5837   ins_pipe(long_memory_op);
5838 %}
5839 
5840 instruct membar_release_lock() %{
5841   match(MemBarReleaseLock);
5842   ins_cost(0);
5843 
5844   size(0);
5845   format %{ "!MEMBAR-release (CAS in succeeding FastUnlock so empty encoding)" %}
5846   ins_encode( );
5847   ins_pipe(empty);
5848 %}
5849 
5850 instruct membar_volatile() %{
5851   match(MemBarVolatile);
5852   ins_cost(4*MEMORY_REF_COST);
5853 
5854   size(4);
5855   format %{ "MEMBAR-volatile" %}
5856   ins_encode %{
5857     __ membar(MacroAssembler::StoreLoad, noreg);
5858   %}
5859   ins_pipe(long_memory_op);
5860 %}
5861 
5862 instruct unnecessary_membar_volatile() %{
5863   match(MemBarVolatile);
5864   predicate(Matcher::post_store_load_barrier(n));
5865   ins_cost(0);
5866 
5867   size(0);
5868   format %{ "!MEMBAR-volatile (unnecessary so empty encoding)" %}
5869   ins_encode( );
5870   ins_pipe(empty);
5871 %}
5872 
5873 //----------Register Move Instructions-----------------------------------------
5874 // instruct roundDouble_nop(regD dst) %{
5875 //   match(Set dst (RoundDouble dst));
5876 //   ins_pipe(empty);
5877 // %}
5878 
5879 
5880 // instruct roundFloat_nop(regF dst) %{
5881 //   match(Set dst (RoundFloat dst));
5882 //   ins_pipe(empty);
5883 // %}
5884 
5885 
5886 #ifdef AARCH64
5887 // 0 constant in register
5888 instruct zrImmI0(ZRRegI dst, immI0 imm) %{
5889   match(Set dst imm);
5890   size(0);
5891   ins_cost(0);
5892 
5893   format %{ "! ZR (int 0)" %}
5894   ins_encode( /*empty encoding*/ );
5895   ins_pipe(ialu_none);
5896 %}
5897 
5898 // 0 constant in register
5899 instruct zrImmL0(ZRRegL dst, immL0 imm) %{
5900   match(Set dst imm);
5901   size(0);
5902   ins_cost(0);
5903 
5904   format %{ "! ZR (long 0)" %}
5905   ins_encode( /*empty encoding*/ );
5906   ins_pipe(ialu_none);
5907 %}
5908 
5909 #ifdef XXX
5910 // 0 constant in register
5911 instruct zrImmN0(ZRRegN dst, immN0 imm) %{
5912   match(Set dst imm);
5913   size(0);
5914   ins_cost(0);
5915 
5916   format %{ "! ZR (compressed pointer NULL)" %}
5917   ins_encode( /*empty encoding*/ );
5918   ins_pipe(ialu_none);
5919 %}
5920 
5921 // 0 constant in register
5922 instruct zrImmP0(ZRRegP dst, immP0 imm) %{
5923   match(Set dst imm);
5924   size(0);
5925   ins_cost(0);
5926 
5927   format %{ "! ZR (NULL)" %}
5928   ins_encode( /*empty encoding*/ );
5929   ins_pipe(ialu_none);
5930 %}
5931 #endif
5932 #endif // AARCH64
5933 
5934 // Cast Index to Pointer for unsafe natives
5935 instruct castX2P(iRegX src, iRegP dst) %{
5936   match(Set dst (CastX2P src));
5937 
5938   format %{ "MOV    $dst,$src\t! IntX->Ptr if $dst != $src" %}
5939   ins_encode %{
5940     if ($dst$$Register !=  $src$$Register) {
5941       __ mov($dst$$Register, $src$$Register);
5942     }
5943   %}
5944   ins_pipe(ialu_reg);
5945 %}
5946 
5947 // Cast Pointer to Index for unsafe natives
5948 instruct castP2X(iRegP src, iRegX dst) %{
5949   match(Set dst (CastP2X src));
5950 
5951   format %{ "MOV    $dst,$src\t! Ptr->IntX if $dst != $src" %}
5952   ins_encode %{
5953     if ($dst$$Register !=  $src$$Register) {
5954       __ mov($dst$$Register, $src$$Register);
5955     }
5956   %}
5957   ins_pipe(ialu_reg);
5958 %}
5959 
5960 #ifndef AARCH64
5961 //----------Conditional Move---------------------------------------------------
5962 // Conditional move
5963 instruct cmovIP_reg(cmpOpP cmp, flagsRegP pcc, iRegI dst, iRegI src) %{
5964   match(Set dst (CMoveI (Binary cmp pcc) (Binary dst src)));
5965   ins_cost(150);
5966   size(4);
5967   format %{ "MOV$cmp  $dst,$src\t! int" %}
5968   ins_encode %{
5969     __ mov($dst$$Register, $src$$Register, (AsmCondition)($cmp$$cmpcode));
5970   %}
5971   ins_pipe(ialu_reg);
5972 %}
5973 #endif
5974 
5975 #ifdef AARCH64
5976 instruct cmovI_reg3(cmpOp cmp, flagsReg icc, iRegI dst, iRegI src1, iRegI src2) %{
5977   match(Set dst (CMoveI (Binary cmp icc) (Binary src2 src1)));
5978   ins_cost(150);
5979   size(4);
5980   format %{ "CSEL $dst,$src1,$src2,$cmp\t! int" %}
5981   ins_encode %{
5982     __ csel($dst$$Register, $src1$$Register, $src2$$Register, (AsmCondition)($cmp$$cmpcode));
5983   %}
5984   ins_pipe(ialu_reg);
5985 %}
5986 
5987 instruct cmovL_reg3(cmpOp cmp, flagsReg icc, iRegL dst, iRegL src1, iRegL src2) %{
5988   match(Set dst (CMoveL (Binary cmp icc) (Binary src2 src1)));
5989   ins_cost(150);
5990   size(4);
5991   format %{ "CSEL $dst,$src1,$src2,$cmp\t! long" %}
5992   ins_encode %{
5993     __ csel($dst$$Register, $src1$$Register, $src2$$Register, (AsmCondition)($cmp$$cmpcode));
5994   %}
5995   ins_pipe(ialu_reg);
5996 %}
5997 
5998 instruct cmovP_reg3(cmpOp cmp, flagsReg icc, iRegP dst, iRegP src1, iRegP src2) %{
5999   match(Set dst (CMoveP (Binary cmp icc) (Binary src2 src1)));
6000   ins_cost(150);
6001   size(4);
6002   format %{ "CSEL $dst,$src1,$src2,$cmp\t! ptr" %}
6003   ins_encode %{
6004     __ csel($dst$$Register, $src1$$Register, $src2$$Register, (AsmCondition)($cmp$$cmpcode));
6005   %}
6006   ins_pipe(ialu_reg);
6007 %}
6008 
6009 instruct cmovN_reg3(cmpOp cmp, flagsReg icc, iRegN dst, iRegN src1, iRegN src2) %{
6010   match(Set dst (CMoveN (Binary cmp icc) (Binary src2 src1)));
6011   ins_cost(150);
6012   size(4);
6013   format %{ "CSEL $dst,$src1,$src2,$cmp\t! compressed ptr" %}
6014   ins_encode %{
6015     __ csel($dst$$Register, $src1$$Register, $src2$$Register, (AsmCondition)($cmp$$cmpcode));
6016   %}
6017   ins_pipe(ialu_reg);
6018 %}
6019 
6020 instruct cmovIP_reg3(cmpOpP cmp, flagsRegP icc, iRegI dst, iRegI src1, iRegI src2) %{
6021   match(Set dst (CMoveI (Binary cmp icc) (Binary src2 src1)));
6022   ins_cost(150);
6023   size(4);
6024   format %{ "CSEL $dst,$src1,$src2,$cmp\t! int" %}
6025   ins_encode %{
6026     __ csel($dst$$Register, $src1$$Register, $src2$$Register, (AsmCondition)($cmp$$cmpcode));
6027   %}
6028   ins_pipe(ialu_reg);
6029 %}
6030 
6031 instruct cmovLP_reg3(cmpOpP cmp, flagsRegP icc, iRegL dst, iRegL src1, iRegL src2) %{
6032   match(Set dst (CMoveL (Binary cmp icc) (Binary src2 src1)));
6033   ins_cost(150);
6034   size(4);
6035   format %{ "CSEL $dst,$src1,$src2,$cmp\t! long" %}
6036   ins_encode %{
6037     __ csel($dst$$Register, $src1$$Register, $src2$$Register, (AsmCondition)($cmp$$cmpcode));
6038   %}
6039   ins_pipe(ialu_reg);
6040 %}
6041 
6042 instruct cmovPP_reg3(cmpOpP cmp, flagsRegP icc, iRegP dst, iRegP src1, iRegP src2) %{
6043   match(Set dst (CMoveP (Binary cmp icc) (Binary src2 src1)));
6044   ins_cost(150);
6045   size(4);
6046   format %{ "CSEL $dst,$src1,$src2,$cmp\t! ptr" %}
6047   ins_encode %{
6048     __ csel($dst$$Register, $src1$$Register, $src2$$Register, (AsmCondition)($cmp$$cmpcode));
6049   %}
6050   ins_pipe(ialu_reg);
6051 %}
6052 
6053 instruct cmovNP_reg3(cmpOpP cmp, flagsRegP icc, iRegN dst, iRegN src1, iRegN src2) %{
6054   match(Set dst (CMoveN (Binary cmp icc) (Binary src2 src1)));
6055   ins_cost(150);
6056   size(4);
6057   format %{ "CSEL $dst,$src1,$src2,$cmp\t! compressed ptr" %}
6058   ins_encode %{
6059     __ csel($dst$$Register, $src1$$Register, $src2$$Register, (AsmCondition)($cmp$$cmpcode));
6060   %}
6061   ins_pipe(ialu_reg);
6062 %}
6063 
6064 instruct cmovIU_reg3(cmpOpU cmp, flagsRegU icc, iRegI dst, iRegI src1, iRegI src2) %{
6065   match(Set dst (CMoveI (Binary cmp icc) (Binary src2 src1)));
6066   ins_cost(150);
6067   size(4);
6068   format %{ "CSEL $dst,$src1,$src2,$cmp\t! int" %}
6069   ins_encode %{
6070     __ csel($dst$$Register, $src1$$Register, $src2$$Register, (AsmCondition)($cmp$$cmpcode));
6071   %}
6072   ins_pipe(ialu_reg);
6073 %}
6074 
6075 instruct cmovLU_reg3(cmpOpU cmp, flagsRegU icc, iRegL dst, iRegL src1, iRegL src2) %{
6076   match(Set dst (CMoveL (Binary cmp icc) (Binary src2 src1)));
6077   ins_cost(150);
6078   size(4);
6079   format %{ "CSEL $dst,$src1,$src2,$cmp\t! long" %}
6080   ins_encode %{
6081     __ csel($dst$$Register, $src1$$Register, $src2$$Register, (AsmCondition)($cmp$$cmpcode));
6082   %}
6083   ins_pipe(ialu_reg);
6084 %}
6085 
6086 instruct cmovPU_reg3(cmpOpU cmp, flagsRegU icc, iRegP dst, iRegP src1, iRegP src2) %{
6087   match(Set dst (CMoveP (Binary cmp icc) (Binary src2 src1)));
6088   ins_cost(150);
6089   size(4);
6090   format %{ "CSEL $dst,$src1,$src2,$cmp\t! ptr" %}
6091   ins_encode %{
6092     __ csel($dst$$Register, $src1$$Register, $src2$$Register, (AsmCondition)($cmp$$cmpcode));
6093   %}
6094   ins_pipe(ialu_reg);
6095 %}
6096 
6097 instruct cmovNU_reg3(cmpOpU cmp, flagsRegU icc, iRegN dst, iRegN src1, iRegN src2) %{
6098   match(Set dst (CMoveN (Binary cmp icc) (Binary src2 src1)));
6099   ins_cost(150);
6100   size(4);
6101   format %{ "CSEL $dst,$src1,$src2,$cmp\t! compressed ptr" %}
6102   ins_encode %{
6103     __ csel($dst$$Register, $src1$$Register, $src2$$Register, (AsmCondition)($cmp$$cmpcode));
6104   %}
6105   ins_pipe(ialu_reg);
6106 %}
6107 
6108 instruct cmovIZ_reg3(cmpOp0 cmp, flagsReg_EQNELTGE icc, iRegI dst, iRegI src1, iRegI src2) %{
6109   match(Set dst (CMoveI (Binary cmp icc) (Binary src2 src1)));
6110   ins_cost(150);
6111   size(4);
6112   format %{ "CSEL $dst,$src1,$src2,$cmp\t! int" %}
6113   ins_encode %{
6114     __ csel($dst$$Register, $src1$$Register, $src2$$Register, (AsmCondition)($cmp$$cmpcode));
6115   %}
6116   ins_pipe(ialu_reg);
6117 %}
6118 
6119 instruct cmovLZ_reg3(cmpOp0 cmp, flagsReg_EQNELTGE icc, iRegL dst, iRegL src1, iRegL src2) %{
6120   match(Set dst (CMoveL (Binary cmp icc) (Binary src2 src1)));
6121   ins_cost(150);
6122   size(4);
6123   format %{ "CSEL $dst,$src1,$src2,$cmp\t! long" %}
6124   ins_encode %{
6125     __ csel($dst$$Register, $src1$$Register, $src2$$Register, (AsmCondition)($cmp$$cmpcode));
6126   %}
6127   ins_pipe(ialu_reg);
6128 %}
6129 
6130 instruct cmovPZ_reg3(cmpOp0 cmp, flagsReg_EQNELTGE icc, iRegP dst, iRegP src1, iRegP src2) %{
6131   match(Set dst (CMoveP (Binary cmp icc) (Binary src2 src1)));
6132   ins_cost(150);
6133   size(4);
6134   format %{ "CSEL $dst,$src1,$src2,$cmp\t! ptr" %}
6135   ins_encode %{
6136     __ csel($dst$$Register, $src1$$Register, $src2$$Register, (AsmCondition)($cmp$$cmpcode));
6137   %}
6138   ins_pipe(ialu_reg);
6139 %}
6140 
6141 instruct cmovNZ_reg3(cmpOp0 cmp, flagsReg_EQNELTGE icc, iRegN dst, iRegN src1, iRegN src2) %{
6142   match(Set dst (CMoveN (Binary cmp icc) (Binary src2 src1)));
6143   ins_cost(150);
6144   size(4);
6145   format %{ "CSEL $dst,$src1,$src2,$cmp\t! compressed ptr" %}
6146   ins_encode %{
6147     __ csel($dst$$Register, $src1$$Register, $src2$$Register, (AsmCondition)($cmp$$cmpcode));
6148   %}
6149   ins_pipe(ialu_reg);
6150 %}
6151 #endif // AARCH64
6152 
6153 #ifndef AARCH64
6154 instruct cmovIP_immMov(cmpOpP cmp, flagsRegP pcc, iRegI dst, immIMov src) %{
6155   match(Set dst (CMoveI (Binary cmp pcc) (Binary dst src)));
6156   ins_cost(140);
6157   size(4);
6158   format %{ "MOV$cmp  $dst,$src" %}
6159   ins_encode %{
6160     __ mov($dst$$Register, $src$$constant, (AsmCondition)($cmp$$cmpcode));
6161   %}
6162   ins_pipe(ialu_imm);
6163 %}
6164 
6165 instruct cmovIP_imm16(cmpOpP cmp, flagsRegP pcc, iRegI dst, immI16 src) %{
6166   match(Set dst (CMoveI (Binary cmp pcc) (Binary dst src)));
6167   ins_cost(140);
6168   size(4);
6169   format %{ "MOVw$cmp  $dst,$src" %}
6170   ins_encode %{
6171     __ movw($dst$$Register, $src$$constant, (AsmCondition)($cmp$$cmpcode));
6172   %}
6173   ins_pipe(ialu_imm);
6174 %}
6175 #endif
6176 
6177 instruct cmovI_reg(cmpOp cmp, flagsReg icc, iRegI dst, iRegI src) %{
6178   match(Set dst (CMoveI (Binary cmp icc) (Binary dst src)));
6179   ins_cost(150);
6180   size(4);
6181   format %{ "MOV$cmp  $dst,$src" %}
6182   ins_encode %{
6183     __ mov($dst$$Register, $src$$Register, (AsmCondition)($cmp$$cmpcode));
6184   %}
6185   ins_pipe(ialu_reg);
6186 %}
6187 
6188 #ifdef AARCH64
6189 instruct cmovL_reg(cmpOp cmp, flagsReg icc, iRegL dst, iRegL src) %{
6190   match(Set dst (CMoveL (Binary cmp icc) (Binary dst src)));
6191   ins_cost(150);
6192   size(4);
6193   format %{ "MOV$cmp  $dst,$src\t! long" %}
6194   ins_encode %{
6195     __ mov($dst$$Register, $src$$Register, (AsmCondition)($cmp$$cmpcode));
6196   %}
6197   ins_pipe(ialu_reg);
6198 %}
6199 #endif
6200 
6201 #ifndef AARCH64
6202 instruct cmovI_immMov(cmpOp cmp, flagsReg icc, iRegI dst, immIMov src) %{
6203   match(Set dst (CMoveI (Binary cmp icc) (Binary dst src)));
6204   ins_cost(140);
6205   size(4);
6206   format %{ "MOV$cmp  $dst,$src" %}
6207   ins_encode %{
6208     __ mov($dst$$Register, $src$$constant, (AsmCondition)($cmp$$cmpcode));
6209   %}
6210   ins_pipe(ialu_imm);
6211 %}
6212 
6213 instruct cmovII_imm16(cmpOp cmp, flagsReg icc, iRegI dst, immI16 src) %{
6214   match(Set dst (CMoveI (Binary cmp icc) (Binary dst src)));
6215   ins_cost(140);
6216   size(4);
6217   format %{ "MOVw$cmp  $dst,$src" %}
6218   ins_encode %{
6219     __ movw($dst$$Register, $src$$constant, (AsmCondition)($cmp$$cmpcode));
6220   %}
6221   ins_pipe(ialu_imm);
6222 %}
6223 #endif
6224 
6225 instruct cmovII_reg_EQNELTGE(cmpOp0 cmp, flagsReg_EQNELTGE icc, iRegI dst, iRegI src) %{
6226   match(Set dst (CMoveI (Binary cmp icc) (Binary dst src)));
6227   predicate(_kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::eq ||
6228             _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ne ||
6229             _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::lt ||
6230             _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ge);
6231   ins_cost(150);
6232   size(4);
6233   format %{ "MOV$cmp  $dst,$src" %}
6234   ins_encode %{
6235     __ mov($dst$$Register, $src$$Register, (AsmCondition)($cmp$$cmpcode));
6236   %}
6237   ins_pipe(ialu_reg);
6238 %}
6239 
6240 #ifndef AARCH64
6241 instruct cmovII_immMov_EQNELTGE(cmpOp0 cmp, flagsReg_EQNELTGE icc, iRegI dst, immIMov src) %{
6242   match(Set dst (CMoveI (Binary cmp icc) (Binary dst src)));
6243   predicate(_kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::eq ||
6244             _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ne ||
6245             _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::lt ||
6246             _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ge);
6247   ins_cost(140);
6248   size(4);
6249   format %{ "MOV$cmp  $dst,$src" %}
6250   ins_encode %{
6251     __ mov($dst$$Register, $src$$constant, (AsmCondition)($cmp$$cmpcode));
6252   %}
6253   ins_pipe(ialu_imm);
6254 %}
6255 
6256 instruct cmovII_imm16_EQNELTGE(cmpOp0 cmp, flagsReg_EQNELTGE icc, iRegI dst, immI16 src) %{
6257   match(Set dst (CMoveI (Binary cmp icc) (Binary dst src)));
6258   predicate(_kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::eq ||
6259             _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ne ||
6260             _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::lt ||
6261             _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ge);
6262   ins_cost(140);
6263   size(4);
6264   format %{ "MOVW$cmp  $dst,$src" %}
6265   ins_encode %{
6266     __ movw($dst$$Register, $src$$constant, (AsmCondition)($cmp$$cmpcode));
6267   %}
6268   ins_pipe(ialu_imm);
6269 %}
6270 #endif
6271 
6272 instruct cmovIIu_reg(cmpOpU cmp, flagsRegU icc, iRegI dst, iRegI src) %{
6273   match(Set dst (CMoveI (Binary cmp icc) (Binary dst src)));
6274   ins_cost(150);
6275   size(4);
6276   format %{ "MOV$cmp  $dst,$src" %}
6277   ins_encode %{
6278     __ mov($dst$$Register, $src$$Register, (AsmCondition)($cmp$$cmpcode));
6279   %}
6280   ins_pipe(ialu_reg);
6281 %}
6282 
6283 #ifndef AARCH64
6284 instruct cmovIIu_immMov(cmpOpU cmp, flagsRegU icc, iRegI dst, immIMov src) %{
6285   match(Set dst (CMoveI (Binary cmp icc) (Binary dst src)));
6286   ins_cost(140);
6287   size(4);
6288   format %{ "MOV$cmp  $dst,$src" %}
6289   ins_encode %{
6290     __ mov($dst$$Register, $src$$constant, (AsmCondition)($cmp$$cmpcode));
6291   %}
6292   ins_pipe(ialu_imm);
6293 %}
6294 
6295 instruct cmovIIu_imm16(cmpOpU cmp, flagsRegU icc, iRegI dst, immI16 src) %{
6296   match(Set dst (CMoveI (Binary cmp icc) (Binary dst src)));
6297   ins_cost(140);
6298   size(4);
6299   format %{ "MOVW$cmp  $dst,$src" %}
6300   ins_encode %{
6301     __ movw($dst$$Register, $src$$constant, (AsmCondition)($cmp$$cmpcode));
6302   %}
6303   ins_pipe(ialu_imm);
6304 %}
6305 #endif
6306 
6307 // Conditional move
6308 instruct cmovPP_reg(cmpOpP cmp, flagsRegP pcc, iRegP dst, iRegP src) %{
6309   match(Set dst (CMoveP (Binary cmp pcc) (Binary dst src)));
6310   ins_cost(150);
6311   size(4);
6312   format %{ "MOV$cmp  $dst,$src" %}
6313   ins_encode %{
6314     __ mov($dst$$Register, $src$$Register, (AsmCondition)($cmp$$cmpcode));
6315   %}
6316   ins_pipe(ialu_reg);
6317 %}
6318 
6319 instruct cmovPP_imm(cmpOpP cmp, flagsRegP pcc, iRegP dst, immP0 src) %{
6320   match(Set dst (CMoveP (Binary cmp pcc) (Binary dst src)));
6321   ins_cost(140);
6322   size(4);
6323 #ifdef AARCH64
6324   format %{ "MOV$cmp  $dst,ZR" %}
6325 #else
6326   format %{ "MOV$cmp  $dst,$src" %}
6327 #endif
6328   ins_encode %{
6329 #ifdef AARCH64
6330     __ mov($dst$$Register,             ZR, (AsmCondition)($cmp$$cmpcode));
6331 #else
6332     __ mov($dst$$Register, $src$$constant, (AsmCondition)($cmp$$cmpcode));
6333 #endif
6334   %}
6335   ins_pipe(ialu_imm);
6336 %}
6337 
6338 // This instruction also works with CmpN so we don't need cmovPN_reg.
6339 instruct cmovPI_reg(cmpOp cmp, flagsReg icc, iRegP dst, iRegP src) %{
6340   match(Set dst (CMoveP (Binary cmp icc) (Binary dst src)));
6341   ins_cost(150);
6342 
6343   size(4);
6344   format %{ "MOV$cmp  $dst,$src\t! ptr" %}
6345   ins_encode %{
6346     __ mov($dst$$Register, $src$$Register, (AsmCondition)($cmp$$cmpcode));
6347   %}
6348   ins_pipe(ialu_reg);
6349 %}
6350 
6351 instruct cmovPI_reg_EQNELTGE(cmpOp0 cmp, flagsReg_EQNELTGE icc, iRegP dst, iRegP src) %{
6352   match(Set dst (CMoveP (Binary cmp icc) (Binary dst src)));
6353   predicate(_kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::eq ||
6354             _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ne ||
6355             _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::lt ||
6356             _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ge);
6357   ins_cost(150);
6358 
6359   size(4);
6360   format %{ "MOV$cmp  $dst,$src\t! ptr" %}
6361   ins_encode %{
6362     __ mov($dst$$Register, $src$$Register, (AsmCondition)($cmp$$cmpcode));
6363   %}
6364   ins_pipe(ialu_reg);
6365 %}
6366 
6367 instruct cmovPIu_reg(cmpOpU cmp, flagsRegU icc, iRegP dst, iRegP src) %{
6368   match(Set dst (CMoveP (Binary cmp icc) (Binary dst src)));
6369   ins_cost(150);
6370 
6371   size(4);
6372   format %{ "MOV$cmp  $dst,$src\t! ptr" %}
6373   ins_encode %{
6374     __ mov($dst$$Register, $src$$Register, (AsmCondition)($cmp$$cmpcode));
6375   %}
6376   ins_pipe(ialu_reg);
6377 %}
6378 
6379 instruct cmovPI_imm(cmpOp cmp, flagsReg icc, iRegP dst, immP0 src) %{
6380   match(Set dst (CMoveP (Binary cmp icc) (Binary dst src)));
6381   ins_cost(140);
6382 
6383   size(4);
6384 #ifdef AARCH64
6385   format %{ "MOV$cmp  $dst,ZR\t! ptr" %}
6386 #else
6387   format %{ "MOV$cmp  $dst,$src\t! ptr" %}
6388 #endif
6389   ins_encode %{
6390 #ifdef AARCH64
6391     __ mov($dst$$Register,             ZR, (AsmCondition)($cmp$$cmpcode));
6392 #else
6393     __ mov($dst$$Register, $src$$constant, (AsmCondition)($cmp$$cmpcode));
6394 #endif
6395   %}
6396   ins_pipe(ialu_imm);
6397 %}
6398 
6399 instruct cmovPI_imm_EQNELTGE(cmpOp0 cmp, flagsReg_EQNELTGE icc, iRegP dst, immP0 src) %{
6400   match(Set dst (CMoveP (Binary cmp icc) (Binary dst src)));
6401   predicate(_kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::eq ||
6402             _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ne ||
6403             _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::lt ||
6404             _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ge);
6405   ins_cost(140);
6406 
6407   size(4);
6408 #ifdef AARCH64
6409   format %{ "MOV$cmp  $dst,ZR\t! ptr" %}
6410 #else
6411   format %{ "MOV$cmp  $dst,$src\t! ptr" %}
6412 #endif
6413   ins_encode %{
6414 #ifdef AARCH64
6415     __ mov($dst$$Register,             ZR, (AsmCondition)($cmp$$cmpcode));
6416 #else
6417     __ mov($dst$$Register, $src$$constant, (AsmCondition)($cmp$$cmpcode));
6418 #endif
6419   %}
6420   ins_pipe(ialu_imm);
6421 %}
6422 
6423 instruct cmovPIu_imm(cmpOpU cmp, flagsRegU icc, iRegP dst, immP0 src) %{
6424   match(Set dst (CMoveP (Binary cmp icc) (Binary dst src)));
6425   ins_cost(140);
6426 
6427   size(4);
6428 #ifdef AARCH64
6429   format %{ "MOV$cmp  $dst,ZR\t! ptr" %}
6430 #else
6431   format %{ "MOV$cmp  $dst,$src\t! ptr" %}
6432 #endif
6433   ins_encode %{
6434 #ifdef AARCH64
6435     __ mov($dst$$Register,             ZR, (AsmCondition)($cmp$$cmpcode));
6436 #else
6437     __ mov($dst$$Register, $src$$constant, (AsmCondition)($cmp$$cmpcode));
6438 #endif
6439   %}
6440   ins_pipe(ialu_imm);
6441 %}
6442 
6443 #ifdef AARCH64
6444 // Conditional move
6445 instruct cmovF_reg(cmpOp cmp, flagsReg icc, regF dst, regF src1, regF src2) %{
6446   match(Set dst (CMoveF (Binary cmp icc) (Binary src2 src1)));
6447   ins_cost(150);
6448   size(4);
6449   format %{ "FCSEL_s $dst,$src1,$src2,$cmp" %}
6450   ins_encode %{
6451     __ fcsel_s($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister, (AsmCondition)($cmp$$cmpcode));
6452   %}
6453   ins_pipe(int_conditional_float_move);
6454 %}
6455 
6456 instruct cmovD_reg(cmpOp cmp, flagsReg icc, regD dst, regD src1, regD src2) %{
6457   match(Set dst (CMoveD (Binary cmp icc) (Binary src2 src1)));
6458   ins_cost(150);
6459   size(4);
6460   format %{ "FCSEL_d $dst,$src1,$src2,$cmp" %}
6461   ins_encode %{
6462     __ fcsel_d($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister, (AsmCondition)($cmp$$cmpcode));
6463   %}
6464   ins_pipe(int_conditional_float_move);
6465 %}
6466 
6467 instruct cmovFP_reg(cmpOpP cmp, flagsRegP icc, regF dst, regF src1, regF src2) %{
6468   match(Set dst (CMoveF (Binary cmp icc) (Binary src2 src1)));
6469   ins_cost(150);
6470   size(4);
6471   format %{ "FCSEL_s $dst,$src1,$src2,$cmp" %}
6472   ins_encode %{
6473     __ fcsel_s($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister, (AsmCondition)($cmp$$cmpcode));
6474   %}
6475   ins_pipe(int_conditional_float_move);
6476 %}
6477 
6478 instruct cmovDP_reg(cmpOpP cmp, flagsRegP icc, regD dst, regD src1, regD src2) %{
6479   match(Set dst (CMoveD (Binary cmp icc) (Binary src2 src1)));
6480   ins_cost(150);
6481   size(4);
6482   format %{ "FCSEL_d $dst,$src1,$src2,$cmp" %}
6483   ins_encode %{
6484     __ fcsel_d($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister, (AsmCondition)($cmp$$cmpcode));
6485   %}
6486   ins_pipe(int_conditional_float_move);
6487 %}
6488 
6489 instruct cmovFU_reg(cmpOpU cmp, flagsRegU icc, regF dst, regF src1, regF src2) %{
6490   match(Set dst (CMoveF (Binary cmp icc) (Binary src2 src1)));
6491   ins_cost(150);
6492   size(4);
6493   format %{ "FCSEL_s $dst,$src1,$src2,$cmp" %}
6494   ins_encode %{
6495     __ fcsel_s($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister, (AsmCondition)($cmp$$cmpcode));
6496   %}
6497   ins_pipe(int_conditional_float_move);
6498 %}
6499 
6500 instruct cmovDU_reg(cmpOpU cmp, flagsRegU icc, regD dst, regD src1, regD src2) %{
6501   match(Set dst (CMoveD (Binary cmp icc) (Binary src2 src1)));
6502   ins_cost(150);
6503   size(4);
6504   format %{ "FCSEL_d $dst,$src1,$src2,$cmp" %}
6505   ins_encode %{
6506     __ fcsel_d($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister, (AsmCondition)($cmp$$cmpcode));
6507   %}
6508   ins_pipe(int_conditional_float_move);
6509 %}
6510 
6511 instruct cmovFZ_reg(cmpOp0 cmp, flagsReg_EQNELTGE icc, regF dst, regF src1, regF src2) %{
6512   match(Set dst (CMoveF (Binary cmp icc) (Binary src2 src1)));
6513   ins_cost(150);
6514   size(4);
6515   format %{ "FCSEL_s $dst,$src1,$src2,$cmp" %}
6516   ins_encode %{
6517     __ fcsel_s($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister, (AsmCondition)($cmp$$cmpcode));
6518   %}
6519   ins_pipe(int_conditional_float_move);
6520 %}
6521 
6522 instruct cmovDZ_reg(cmpOp0 cmp, flagsReg_EQNELTGE icc, regD dst, regD src1, regD src2) %{
6523   match(Set dst (CMoveD (Binary cmp icc) (Binary src2 src1)));
6524   ins_cost(150);
6525   size(4);
6526   format %{ "FCSEL_d $dst,$src1,$src2,$cmp" %}
6527   ins_encode %{
6528     __ fcsel_d($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister, (AsmCondition)($cmp$$cmpcode));
6529   %}
6530   ins_pipe(int_conditional_float_move);
6531 %}
6532 
6533 #else // !AARCH64
6534 
6535 // Conditional move
6536 instruct cmovFP_reg(cmpOpP cmp, flagsRegP pcc, regF dst, regF src) %{
6537   match(Set dst (CMoveF (Binary cmp pcc) (Binary dst src)));
6538   ins_cost(150);
6539   size(4);
6540   format %{ "FCPYS$cmp $dst,$src" %}
6541   ins_encode %{
6542     __ fcpys($dst$$FloatRegister, $src$$FloatRegister, (AsmCondition)($cmp$$cmpcode));
6543   %}
6544   ins_pipe(int_conditional_float_move);
6545 %}
6546 
6547 instruct cmovFI_reg(cmpOp cmp, flagsReg icc, regF dst, regF src) %{
6548   match(Set dst (CMoveF (Binary cmp icc) (Binary dst src)));
6549   ins_cost(150);
6550 
6551   size(4);
6552   format %{ "FCPYS$cmp $dst,$src" %}
6553   ins_encode %{
6554     __ fcpys($dst$$FloatRegister, $src$$FloatRegister, (AsmCondition)($cmp$$cmpcode));
6555   %}
6556   ins_pipe(int_conditional_float_move);
6557 %}
6558 
6559 instruct cmovFI_reg_EQNELTGE(cmpOp0 cmp, flagsReg_EQNELTGE icc, regF dst, regF src) %{
6560   match(Set dst (CMoveF (Binary cmp icc) (Binary dst src)));
6561   predicate(_kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::eq ||
6562             _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ne ||
6563             _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::lt ||
6564             _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ge);
6565   ins_cost(150);
6566 
6567   size(4);
6568   format %{ "FCPYS$cmp $dst,$src" %}
6569   ins_encode %{
6570     __ fcpys($dst$$FloatRegister, $src$$FloatRegister, (AsmCondition)($cmp$$cmpcode));
6571   %}
6572   ins_pipe(int_conditional_float_move);
6573 %}
6574 
6575 instruct cmovFIu_reg(cmpOpU cmp, flagsRegU icc, regF dst, regF src) %{
6576   match(Set dst (CMoveF (Binary cmp icc) (Binary dst src)));
6577   ins_cost(150);
6578 
6579   size(4);
6580   format %{ "FCPYS$cmp $dst,$src" %}
6581   ins_encode %{
6582     __ fcpys($dst$$FloatRegister, $src$$FloatRegister, (AsmCondition)($cmp$$cmpcode));
6583   %}
6584   ins_pipe(int_conditional_float_move);
6585 %}
6586 
6587 // Conditional move
6588 instruct cmovDP_reg(cmpOpP cmp, flagsRegP pcc, regD dst, regD src) %{
6589   match(Set dst (CMoveD (Binary cmp pcc) (Binary dst src)));
6590   ins_cost(150);
6591   size(4);
6592   format %{ "FCPYD$cmp $dst,$src" %}
6593   ins_encode %{
6594     __ fcpyd($dst$$FloatRegister, $src$$FloatRegister, (AsmCondition)($cmp$$cmpcode));
6595   %}
6596   ins_pipe(int_conditional_double_move);
6597 %}
6598 
6599 instruct cmovDI_reg(cmpOp cmp, flagsReg icc, regD dst, regD src) %{
6600   match(Set dst (CMoveD (Binary cmp icc) (Binary dst src)));
6601   ins_cost(150);
6602 
6603   size(4);
6604   format %{ "FCPYD$cmp $dst,$src" %}
6605   ins_encode %{
6606     __ fcpyd($dst$$FloatRegister, $src$$FloatRegister, (AsmCondition)($cmp$$cmpcode));
6607   %}
6608   ins_pipe(int_conditional_double_move);
6609 %}
6610 
6611 instruct cmovDI_reg_EQNELTGE(cmpOp0 cmp, flagsReg_EQNELTGE icc, regD dst, regD src) %{
6612   match(Set dst (CMoveD (Binary cmp icc) (Binary dst src)));
6613   predicate(_kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::eq || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ne || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::lt || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ge);
6614   ins_cost(150);
6615 
6616   size(4);
6617   format %{ "FCPYD$cmp $dst,$src" %}
6618   ins_encode %{
6619     __ fcpyd($dst$$FloatRegister, $src$$FloatRegister, (AsmCondition)($cmp$$cmpcode));
6620   %}
6621   ins_pipe(int_conditional_double_move);
6622 %}
6623 
6624 instruct cmovDIu_reg(cmpOpU cmp, flagsRegU icc, regD dst, regD src) %{
6625   match(Set dst (CMoveD (Binary cmp icc) (Binary dst src)));
6626   ins_cost(150);
6627 
6628   size(4);
6629   format %{ "FCPYD$cmp $dst,$src" %}
6630   ins_encode %{
6631     __ fcpyd($dst$$FloatRegister, $src$$FloatRegister, (AsmCondition)($cmp$$cmpcode));
6632   %}
6633   ins_pipe(int_conditional_double_move);
6634 %}
6635 
6636 // Conditional move
6637 instruct cmovLP_reg(cmpOpP cmp, flagsRegP pcc, iRegL dst, iRegL src) %{
6638   match(Set dst (CMoveL (Binary cmp pcc) (Binary dst src)));
6639   ins_cost(150);
6640 
6641   size(8);
6642   format %{ "MOV$cmp  $dst.lo,$src.lo\t! long\n\t"
6643             "MOV$cmp  $dst.hi,$src.hi" %}
6644   ins_encode %{
6645     __ mov($dst$$Register, $src$$Register, (AsmCondition)($cmp$$cmpcode));
6646     __ mov($dst$$Register->successor(), $src$$Register->successor(), (AsmCondition)($cmp$$cmpcode));
6647   %}
6648   ins_pipe(ialu_reg);
6649 %}
6650 
6651 // TODO: try immLRot2 instead, (0, $con$$constant) becomes
6652 // (hi($con$$constant), lo($con$$constant)) becomes
6653 instruct cmovLP_immRot(cmpOpP cmp, flagsRegP pcc, iRegL dst, immLlowRot src) %{
6654   match(Set dst (CMoveL (Binary cmp pcc) (Binary dst src)));
6655   ins_cost(140);
6656 
6657   size(8);
6658   format %{ "MOV$cmp  $dst.lo,$src\t! long\n\t"
6659             "MOV$cmp  $dst.hi,0" %}
6660   ins_encode %{
6661     __ mov($dst$$Register, $src$$constant, (AsmCondition)($cmp$$cmpcode));
6662     __ mov($dst$$Register->successor(), 0, (AsmCondition)($cmp$$cmpcode));
6663   %}
6664   ins_pipe(ialu_imm);
6665 %}
6666 
6667 instruct cmovLP_imm16(cmpOpP cmp, flagsRegP pcc, iRegL dst, immL16 src) %{
6668   match(Set dst (CMoveL (Binary cmp pcc) (Binary dst src)));
6669   ins_cost(140);
6670 
6671   size(8);
6672   format %{ "MOV$cmp  $dst.lo,$src\t! long\n\t"
6673             "MOV$cmp  $dst.hi,0" %}
6674   ins_encode %{
6675     __ movw($dst$$Register, $src$$constant, (AsmCondition)($cmp$$cmpcode));
6676     __ mov($dst$$Register->successor(), 0, (AsmCondition)($cmp$$cmpcode));
6677   %}
6678   ins_pipe(ialu_imm);
6679 %}
6680 
6681 instruct cmovLI_reg(cmpOp cmp, flagsReg icc, iRegL dst, iRegL src) %{
6682   match(Set dst (CMoveL (Binary cmp icc) (Binary dst src)));
6683   ins_cost(150);
6684 
6685   size(8);
6686   format %{ "MOV$cmp  $dst.lo,$src.lo\t! long\n\t"
6687             "MOV$cmp  $dst.hi,$src.hi" %}
6688   ins_encode %{
6689     __ mov($dst$$Register, $src$$Register, (AsmCondition)($cmp$$cmpcode));
6690     __ mov($dst$$Register->successor(), $src$$Register->successor(), (AsmCondition)($cmp$$cmpcode));
6691   %}
6692   ins_pipe(ialu_reg);
6693 %}
6694 
6695 instruct cmovLI_reg_EQNELTGE(cmpOp0 cmp, flagsReg_EQNELTGE icc, iRegL dst, iRegL src) %{
6696   match(Set dst (CMoveL (Binary cmp icc) (Binary dst src)));
6697   predicate(_kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::eq ||
6698             _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ne ||
6699             _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::lt ||
6700             _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ge);
6701   ins_cost(150);
6702 
6703   size(8);
6704   format %{ "MOV$cmp  $dst.lo,$src.lo\t! long\n\t"
6705             "MOV$cmp  $dst.hi,$src.hi" %}
6706   ins_encode %{
6707     __ mov($dst$$Register, $src$$Register, (AsmCondition)($cmp$$cmpcode));
6708     __ mov($dst$$Register->successor(), $src$$Register->successor(), (AsmCondition)($cmp$$cmpcode));
6709   %}
6710   ins_pipe(ialu_reg);
6711 %}
6712 
6713 // TODO: try immLRot2 instead, (0, $con$$constant) becomes
6714 // (hi($con$$constant), lo($con$$constant)) becomes
6715 instruct cmovLI_immRot(cmpOp cmp, flagsReg icc, iRegL dst, immLlowRot src) %{
6716   match(Set dst (CMoveL (Binary cmp icc) (Binary dst src)));
6717   ins_cost(140);
6718 
6719   size(8);
6720   format %{ "MOV$cmp  $dst.lo,$src\t! long\n\t"
6721             "MOV$cmp  $dst.hi,0" %}
6722   ins_encode %{
6723     __ mov($dst$$Register, $src$$constant, (AsmCondition)($cmp$$cmpcode));
6724     __ mov($dst$$Register->successor(), 0, (AsmCondition)($cmp$$cmpcode));
6725   %}
6726   ins_pipe(ialu_imm);
6727 %}
6728 
6729 // TODO: try immLRot2 instead, (0, $con$$constant) becomes
6730 // (hi($con$$constant), lo($con$$constant)) becomes
6731 instruct cmovLI_immRot_EQNELTGE(cmpOp0 cmp, flagsReg_EQNELTGE icc, iRegL dst, immLlowRot src) %{
6732   match(Set dst (CMoveL (Binary cmp icc) (Binary dst src)));
6733   predicate(_kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::eq ||
6734             _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ne ||
6735             _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::lt ||
6736             _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ge);
6737   ins_cost(140);
6738 
6739   size(8);
6740   format %{ "MOV$cmp  $dst.lo,$src\t! long\n\t"
6741             "MOV$cmp  $dst.hi,0" %}
6742   ins_encode %{
6743     __ mov($dst$$Register, $src$$constant, (AsmCondition)($cmp$$cmpcode));
6744     __ mov($dst$$Register->successor(), 0, (AsmCondition)($cmp$$cmpcode));
6745   %}
6746   ins_pipe(ialu_imm);
6747 %}
6748 
6749 instruct cmovLI_imm16(cmpOp cmp, flagsReg icc, iRegL dst, immL16 src) %{
6750   match(Set dst (CMoveL (Binary cmp icc) (Binary dst src)));
6751   ins_cost(140);
6752 
6753   size(8);
6754   format %{ "MOV$cmp  $dst.lo,$src\t! long\n\t"
6755             "MOV$cmp  $dst.hi,0" %}
6756   ins_encode %{
6757     __ movw($dst$$Register, $src$$constant, (AsmCondition)($cmp$$cmpcode));
6758     __ movw($dst$$Register->successor(), 0, (AsmCondition)($cmp$$cmpcode));
6759   %}
6760   ins_pipe(ialu_imm);
6761 %}
6762 
6763 instruct cmovLI_imm16_EQNELTGE(cmpOp0 cmp, flagsReg_EQNELTGE icc, iRegL dst, immL16 src) %{
6764   match(Set dst (CMoveL (Binary cmp icc) (Binary dst src)));
6765   predicate(_kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::eq ||
6766             _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ne ||
6767             _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::lt ||
6768             _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ge);
6769   ins_cost(140);
6770 
6771   size(8);
6772   format %{ "MOV$cmp  $dst.lo,$src\t! long\n\t"
6773             "MOV$cmp  $dst.hi,0" %}
6774   ins_encode %{
6775     __ movw($dst$$Register, $src$$constant, (AsmCondition)($cmp$$cmpcode));
6776     __ movw($dst$$Register->successor(), 0, (AsmCondition)($cmp$$cmpcode));
6777   %}
6778   ins_pipe(ialu_imm);
6779 %}
6780 
6781 instruct cmovLIu_reg(cmpOpU cmp, flagsRegU icc, iRegL dst, iRegL src) %{
6782   match(Set dst (CMoveL (Binary cmp icc) (Binary dst src)));
6783   ins_cost(150);
6784 
6785   size(8);
6786   format %{ "MOV$cmp  $dst.lo,$src.lo\t! long\n\t"
6787             "MOV$cmp  $dst.hi,$src.hi" %}
6788   ins_encode %{
6789     __ mov($dst$$Register, $src$$Register, (AsmCondition)($cmp$$cmpcode));
6790     __ mov($dst$$Register->successor(), $src$$Register->successor(), (AsmCondition)($cmp$$cmpcode));
6791   %}
6792   ins_pipe(ialu_reg);
6793 %}
6794 #endif // !AARCH64
6795 
6796 
6797 //----------OS and Locking Instructions----------------------------------------
6798 
6799 // This name is KNOWN by the ADLC and cannot be changed.
6800 // The ADLC forces a 'TypeRawPtr::BOTTOM' output type
6801 // for this guy.
6802 instruct tlsLoadP(RthreadRegP dst) %{
6803   match(Set dst (ThreadLocal));
6804 
6805   size(0);
6806   ins_cost(0);
6807   format %{ "! TLS is in $dst" %}
6808   ins_encode( /*empty encoding*/ );
6809   ins_pipe(ialu_none);
6810 %}
6811 
6812 instruct checkCastPP( iRegP dst ) %{
6813   match(Set dst (CheckCastPP dst));
6814 
6815   size(0);
6816   format %{ "! checkcastPP of $dst" %}
6817   ins_encode( /*empty encoding*/ );
6818   ins_pipe(empty);
6819 %}
6820 
6821 
6822 instruct castPP( iRegP dst ) %{
6823   match(Set dst (CastPP dst));
6824   format %{ "! castPP of $dst" %}
6825   ins_encode( /*empty encoding*/ );
6826   ins_pipe(empty);
6827 %}
6828 
6829 instruct castII( iRegI dst ) %{
6830   match(Set dst (CastII dst));
6831   format %{ "! castII of $dst" %}
6832   ins_encode( /*empty encoding*/ );
6833   ins_cost(0);
6834   ins_pipe(empty);
6835 %}
6836 
6837 //----------Arithmetic Instructions--------------------------------------------
6838 // Addition Instructions
6839 // Register Addition
6840 instruct addI_reg_reg(iRegI dst, iRegI src1, iRegI src2) %{
6841   match(Set dst (AddI src1 src2));
6842 
6843   size(4);
6844   format %{ "add_32 $dst,$src1,$src2\t! int" %}
6845   ins_encode %{
6846     __ add_32($dst$$Register, $src1$$Register, $src2$$Register);
6847   %}
6848   ins_pipe(ialu_reg_reg);
6849 %}
6850 
6851 #ifndef AARCH64
6852 instruct addshlI_reg_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3) %{
6853   match(Set dst (AddI (LShiftI src1 src2) src3));
6854 
6855   size(4);
6856   format %{ "add_32 $dst,$src3,$src1<<$src2\t! int" %}
6857   ins_encode %{
6858     __ add_32($dst$$Register, $src3$$Register, AsmOperand($src1$$Register, lsl, $src2$$Register));
6859   %}
6860   ins_pipe(ialu_reg_reg);
6861 %}
6862 #endif
6863 
6864 #ifdef AARCH64
6865 #ifdef TODO
6866 instruct addshlL_reg_imm_reg(iRegL dst, iRegL src1, immU6 src2, iRegL src3) %{
6867   match(Set dst (AddL (LShiftL src1 src2) src3));
6868 
6869   size(4);
6870   format %{ "ADD    $dst,$src3,$src1<<$src2\t! long" %}
6871   ins_encode %{
6872     __ add($dst$$Register, $src3$$Register, AsmOperand($src1$$Register, lsl, $src2$$constant));
6873   %}
6874   ins_pipe(ialu_reg_reg);
6875 %}
6876 #endif
6877 #endif
6878 
6879 instruct addshlI_reg_imm_reg(iRegI dst, iRegI src1, immU5 src2, iRegI src3) %{
6880   match(Set dst (AddI (LShiftI src1 src2) src3));
6881 
6882   size(4);
6883   format %{ "add_32 $dst,$src3,$src1<<$src2\t! int" %}
6884   ins_encode %{
6885     __ add_32($dst$$Register, $src3$$Register, AsmOperand($src1$$Register, lsl, $src2$$constant));
6886   %}
6887   ins_pipe(ialu_reg_reg);
6888 %}
6889 
6890 #ifndef AARCH64
6891 instruct addsarI_reg_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3) %{
6892   match(Set dst (AddI (RShiftI src1 src2) src3));
6893 
6894   size(4);
6895   format %{ "add_32 $dst,$src3,$src1>>$src2\t! int" %}
6896   ins_encode %{
6897     __ add_32($dst$$Register, $src3$$Register, AsmOperand($src1$$Register, asr, $src2$$Register));
6898   %}
6899   ins_pipe(ialu_reg_reg);
6900 %}
6901 #endif
6902 
6903 instruct addsarI_reg_imm_reg(iRegI dst, iRegI src1, immU5 src2, iRegI src3) %{
6904   match(Set dst (AddI (RShiftI src1 src2) src3));
6905 
6906   size(4);
6907   format %{ "add_32 $dst,$src3,$src1>>$src2\t! int" %}
6908   ins_encode %{
6909     __ add_32($dst$$Register, $src3$$Register, AsmOperand($src1$$Register, asr, $src2$$constant));
6910   %}
6911   ins_pipe(ialu_reg_reg);
6912 %}
6913 
6914 #ifndef AARCH64
6915 instruct addshrI_reg_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3) %{
6916   match(Set dst (AddI (URShiftI src1 src2) src3));
6917 
6918   size(4);
6919   format %{ "add_32 $dst,$src3,$src1>>>$src2\t! int" %}
6920   ins_encode %{
6921     __ add_32($dst$$Register, $src3$$Register, AsmOperand($src1$$Register, lsr, $src2$$Register));
6922   %}
6923   ins_pipe(ialu_reg_reg);
6924 %}
6925 #endif
6926 
6927 instruct addshrI_reg_imm_reg(iRegI dst, iRegI src1, immU5 src2, iRegI src3) %{
6928   match(Set dst (AddI (URShiftI src1 src2) src3));
6929 
6930   size(4);
6931   format %{ "add_32 $dst,$src3,$src1>>>$src2\t! int" %}
6932   ins_encode %{
6933     __ add_32($dst$$Register, $src3$$Register, AsmOperand($src1$$Register, lsr, $src2$$constant));
6934   %}
6935   ins_pipe(ialu_reg_reg);
6936 %}
6937 
6938 // Immediate Addition
6939 instruct addI_reg_aimmI(iRegI dst, iRegI src1, aimmI src2) %{
6940   match(Set dst (AddI src1 src2));
6941 
6942   size(4);
6943   format %{ "add_32 $dst,$src1,$src2\t! int" %}
6944   ins_encode %{
6945     __ add_32($dst$$Register, $src1$$Register, $src2$$constant);
6946   %}
6947   ins_pipe(ialu_reg_imm);
6948 %}
6949 
6950 // Pointer Register Addition
6951 instruct addP_reg_reg(iRegP dst, iRegP src1, iRegX src2) %{
6952   match(Set dst (AddP src1 src2));
6953 
6954   size(4);
6955   format %{ "ADD    $dst,$src1,$src2\t! ptr" %}
6956   ins_encode %{
6957     __ add($dst$$Register, $src1$$Register, $src2$$Register);
6958   %}
6959   ins_pipe(ialu_reg_reg);
6960 %}
6961 
6962 #ifdef AARCH64
6963 // unshifted I2L operand
6964 operand unshiftedI2L(iRegI src2) %{
6965 //constraint(ALLOC_IN_RC(sp_ptr_reg));
6966   match(ConvI2L src2);
6967 
6968   op_cost(1);
6969   format %{ "$src2.w" %}
6970   interface(MEMORY_INTER) %{
6971     base($src2);
6972     index(0xff);
6973     scale(0x0);
6974     disp(0x0);
6975   %}
6976 %}
6977 
6978 // shifted I2L operand
6979 operand shiftedI2L(iRegI src2, immI_0_4 src3) %{
6980 //constraint(ALLOC_IN_RC(sp_ptr_reg));
6981   match(LShiftX (ConvI2L src2) src3);
6982 
6983   op_cost(1);
6984   format %{ "$src2.w << $src3" %}
6985   interface(MEMORY_INTER) %{
6986     base($src2);
6987     index(0xff);
6988     scale($src3);
6989     disp(0x0);
6990   %}
6991 %}
6992 
6993 opclass shiftedRegI(shiftedI2L, unshiftedI2L);
6994 
6995 instruct shlL_reg_regI(iRegL dst, iRegI src1, immU6 src2) %{
6996   match(Set dst (LShiftL (ConvI2L src1) src2));
6997 
6998   size(4);
6999   format %{ "LSL    $dst,$src1.w,$src2\t! ptr" %}
7000   ins_encode %{
7001     int c = $src2$$constant;
7002     int r = 64 - c;
7003     int s = 31;
7004     if (s >= r) {
7005       s = r - 1;
7006     }
7007     __ sbfm($dst$$Register, $src1$$Register, r, s);
7008   %}
7009   ins_pipe(ialu_reg_reg);
7010 %}
7011 
7012 instruct addP_reg_regI(iRegP dst, iRegP src1, shiftedRegI src2) %{
7013   match(Set dst (AddP src1 src2));
7014 
7015   ins_cost(DEFAULT_COST * 3/2);
7016   size(4);
7017   format %{ "ADD    $dst,$src1,$src2, sxtw\t! ptr" %}
7018   ins_encode %{
7019     Register base = reg_to_register_object($src2$$base);
7020     __ add($dst$$Register, $src1$$Register, base, ex_sxtw, $src2$$scale);
7021   %}
7022   ins_pipe(ialu_reg_reg);
7023 %}
7024 #endif
7025 
7026 // shifted iRegX operand
7027 operand shiftedX(iRegX src2, shimmX src3) %{
7028 //constraint(ALLOC_IN_RC(sp_ptr_reg));
7029   match(LShiftX src2 src3);
7030 
7031   op_cost(1);
7032   format %{ "$src2 << $src3" %}
7033   interface(MEMORY_INTER) %{
7034     base($src2);
7035     index(0xff);
7036     scale($src3);
7037     disp(0x0);
7038   %}
7039 %}
7040 
7041 instruct addshlP_reg_reg_imm(iRegP dst, iRegP src1, shiftedX src2) %{
7042   match(Set dst (AddP src1 src2));
7043 
7044   ins_cost(DEFAULT_COST * 3/2);
7045   size(4);
7046   format %{ "ADD    $dst,$src1,$src2\t! ptr" %}
7047   ins_encode %{
7048     Register base = reg_to_register_object($src2$$base);
7049     __ add($dst$$Register, $src1$$Register, AsmOperand(base, lsl, $src2$$scale));
7050   %}
7051   ins_pipe(ialu_reg_reg);
7052 %}
7053 
7054 // Pointer Immediate Addition
7055 instruct addP_reg_aimmX(iRegP dst, iRegP src1, aimmX src2) %{
7056   match(Set dst (AddP src1 src2));
7057 
7058   size(4);
7059   format %{ "ADD    $dst,$src1,$src2\t! ptr" %}
7060   ins_encode %{
7061     __ add($dst$$Register, $src1$$Register, $src2$$constant);
7062   %}
7063   ins_pipe(ialu_reg_imm);
7064 %}
7065 
7066 // Long Addition
7067 #ifdef AARCH64
7068 instruct addL_reg_reg(iRegL dst, iRegL src1, iRegL src2) %{
7069   match(Set dst (AddL src1 src2));
7070   size(4);
7071   format %{ "ADD     $dst,$src1,$src2\t! long" %}
7072   ins_encode %{
7073     __ add($dst$$Register, $src1$$Register, $src2$$Register);
7074   %}
7075   ins_pipe(ialu_reg_reg);
7076 %}
7077 
7078 instruct addL_reg_regI(iRegL dst, iRegL src1, shiftedRegI src2) %{
7079   match(Set dst (AddL src1 src2));
7080 
7081   ins_cost(DEFAULT_COST * 3/2);
7082   size(4);
7083   format %{ "ADD    $dst,$src1,$src2, sxtw\t! long" %}
7084   ins_encode %{
7085     Register base = reg_to_register_object($src2$$base);
7086     __ add($dst$$Register, $src1$$Register, base, ex_sxtw, $src2$$scale);
7087   %}
7088   ins_pipe(ialu_reg_reg);
7089 %}
7090 #else
7091 instruct addL_reg_reg(iRegL dst, iRegL src1, iRegL src2, flagsReg ccr) %{
7092   match(Set dst (AddL src1 src2));
7093   effect(KILL ccr);
7094   size(8);
7095   format %{ "ADDS    $dst.lo,$src1.lo,$src2.lo\t! long\n\t"
7096             "ADC     $dst.hi,$src1.hi,$src2.hi" %}
7097   ins_encode %{
7098     __ adds($dst$$Register, $src1$$Register, $src2$$Register);
7099     __ adc($dst$$Register->successor(), $src1$$Register->successor(), $src2$$Register->successor());
7100   %}
7101   ins_pipe(ialu_reg_reg);
7102 %}
7103 #endif
7104 
7105 #ifdef AARCH64
7106 // Immediate Addition
7107 instruct addL_reg_aimm(iRegL dst, iRegL src1, aimmL src2) %{
7108   match(Set dst (AddL src1 src2));
7109 
7110   size(4);
7111   format %{ "ADD    $dst,$src1,$src2\t! long" %}
7112   ins_encode %{
7113     __ add($dst$$Register, $src1$$Register, $src2$$constant);
7114   %}
7115   ins_pipe(ialu_reg_imm);
7116 %}
7117 
7118 instruct addL_reg_immLneg(iRegL dst, iRegL src1, aimmLneg src2) %{
7119   match(Set dst (SubL src1 src2));
7120 
7121   size(4);
7122   format %{ "ADD    $dst,$src1,-($src2)\t! long" %}
7123   ins_encode %{
7124     __ add($dst$$Register, $src1$$Register, -$src2$$constant);
7125   %}
7126   ins_pipe(ialu_reg_imm);
7127 %}
7128 #else
7129 // TODO
7130 #endif
7131 
7132 #ifndef AARCH64
7133 // TODO: try immLRot2 instead, (0, $con$$constant) becomes
7134 // (hi($con$$constant), lo($con$$constant)) becomes
7135 instruct addL_reg_immRot(iRegL dst, iRegL src1, immLlowRot con, flagsReg ccr) %{
7136   match(Set dst (AddL src1 con));
7137   effect(KILL ccr);
7138   size(8);
7139   format %{ "ADDS    $dst.lo,$src1.lo,$con\t! long\n\t"
7140             "ADC     $dst.hi,$src1.hi,0" %}
7141   ins_encode %{
7142     __ adds($dst$$Register, $src1$$Register, $con$$constant);
7143     __ adc($dst$$Register->successor(), $src1$$Register->successor(), 0);
7144   %}
7145   ins_pipe(ialu_reg_imm);
7146 %}
7147 #endif
7148 
7149 //----------Conditional_store--------------------------------------------------
7150 // Conditional-store of the updated heap-top.
7151 // Used during allocation of the shared heap.
7152 // Sets flags (EQ) on success.
7153 
7154 // TODO: optimize out barriers with AArch64 load-acquire/store-release
7155 // LoadP-locked.
7156 instruct loadPLocked(iRegP dst, memoryex mem) %{
7157   match(Set dst (LoadPLocked mem));
7158   size(4);
7159   format %{ "LDREX  $dst,$mem" %}
7160   ins_encode %{
7161 #ifdef AARCH64
7162     Register base = reg_to_register_object($mem$$base);
7163     __ ldxr($dst$$Register, base);
7164 #else
7165     __ ldrex($dst$$Register,$mem$$Address);
7166 #endif
7167   %}
7168   ins_pipe(iload_mem);
7169 %}
7170 
7171 instruct storePConditional( memoryex heap_top_ptr, iRegP oldval, iRegP newval, iRegI tmp, flagsRegP pcc ) %{
7172   predicate(_kids[1]->_kids[0]->_leaf->Opcode() == Op_LoadPLocked); // only works in conjunction with a LoadPLocked node
7173   match(Set pcc (StorePConditional heap_top_ptr (Binary oldval newval)));
7174   effect( TEMP tmp );
7175   size(8);
7176   format %{ "STREX  $tmp,$newval,$heap_top_ptr\n\t"
7177             "CMP    $tmp, 0" %}
7178   ins_encode %{
7179 #ifdef AARCH64
7180     Register base = reg_to_register_object($heap_top_ptr$$base);
7181     __ stxr($tmp$$Register, $newval$$Register, base);
7182 #else
7183     __ strex($tmp$$Register, $newval$$Register, $heap_top_ptr$$Address);
7184 #endif
7185     __ cmp($tmp$$Register, 0);
7186   %}
7187   ins_pipe( long_memory_op );
7188 %}
7189 
7190 // Conditional-store of an intx value.
7191 instruct storeXConditional( memoryex mem, iRegX oldval, iRegX newval, iRegX tmp, flagsReg icc ) %{
7192 #ifdef AARCH64
7193   match(Set icc (StoreLConditional mem (Binary oldval newval)));
7194   effect( TEMP tmp );
7195   size(28);
7196   format %{ "loop:\n\t"
7197             "LDXR     $tmp, $mem\t! If $oldval==[$mem] Then store $newval into [$mem], DOESN'T set $newval=[$mem] in any case\n\t"
7198             "SUBS     $tmp, $tmp, $oldval\n\t"
7199             "B.ne     done\n\t"
7200             "STXR     $tmp, $newval, $mem\n\t"
7201             "CBNZ_w   $tmp, loop\n\t"
7202             "CMP      $tmp, 0\n\t"
7203             "done:\n\t"
7204             "membar   LoadStore|LoadLoad" %}
7205 #else
7206   match(Set icc (StoreIConditional mem (Binary oldval newval)));
7207   effect( TEMP tmp );
7208   size(28);
7209   format %{ "loop: \n\t"
7210             "LDREX    $tmp, $mem\t! If $oldval==[$mem] Then store $newval into [$mem], DOESN'T set $newval=[$mem] in any case\n\t"
7211             "XORS     $tmp,$tmp, $oldval\n\t"
7212             "STREX.eq $tmp, $newval, $mem\n\t"
7213             "CMP.eq   $tmp, 1 \n\t"
7214             "B.eq     loop \n\t"
7215             "TEQ      $tmp, 0\n\t"
7216             "membar   LoadStore|LoadLoad" %}
7217 #endif
7218   ins_encode %{
7219     Label loop;
7220     __ bind(loop);
7221 #ifdef AARCH64
7222 // FIXME: use load-acquire/store-release, remove membar?
7223     Label done;
7224     Register base = reg_to_register_object($mem$$base);
7225     __ ldxr($tmp$$Register, base);
7226     __ subs($tmp$$Register, $tmp$$Register, $oldval$$Register);
7227     __ b(done, ne);
7228     __ stxr($tmp$$Register, $newval$$Register, base);
7229     __ cbnz_w($tmp$$Register, loop);
7230     __ cmp($tmp$$Register, 0);
7231     __ bind(done);
7232 #else
7233     __ ldrex($tmp$$Register, $mem$$Address);
7234     __ eors($tmp$$Register, $tmp$$Register, $oldval$$Register);
7235     __ strex($tmp$$Register, $newval$$Register, $mem$$Address, eq);
7236     __ cmp($tmp$$Register, 1, eq);
7237     __ b(loop, eq);
7238     __ teq($tmp$$Register, 0);
7239 #endif
7240     // used by biased locking only. Requires a membar.
7241     __ membar(MacroAssembler::Membar_mask_bits(MacroAssembler::LoadStore | MacroAssembler::LoadLoad), noreg);
7242   %}
7243   ins_pipe( long_memory_op );
7244 %}
7245 
7246 // No flag versions for CompareAndSwap{P,I,L} because matcher can't match them
7247 
7248 #ifdef AARCH64
7249 // TODO: if combined with membar, elide membar and use
7250 // load-acquire/store-release if appropriate
7251 instruct compareAndSwapL_bool(memoryex mem, iRegL oldval, iRegL newval, iRegI res, iRegI tmp, flagsReg ccr) %{
7252   match(Set res (CompareAndSwapL mem (Binary oldval newval)));
7253   effect( KILL ccr, TEMP tmp);
7254   size(24);
7255   format %{ "loop:\n\t"
7256             "LDXR     $tmp, $mem\t! If $oldval==[$mem] Then store $newval into [$mem]\n\t"
7257             "CMP      $tmp, $oldval\n\t"
7258             "B.ne     done\n\t"
7259             "STXR     $tmp, $newval, $mem\n\t"
7260             "CBNZ_w   $tmp, loop\n\t"
7261             "done:\n\t"
7262             "CSET_w   $res, eq" %}
7263   ins_encode %{
7264     Register base = reg_to_register_object($mem$$base);
7265     Label loop, done;
7266     __ bind(loop);
7267     __ ldxr($tmp$$Register, base);
7268     __ cmp($tmp$$Register, $oldval$$Register);
7269     __ b(done, ne);
7270     __ stxr($tmp$$Register, $newval$$Register, base);
7271     __ cbnz_w($tmp$$Register, loop);
7272     __ bind(done);
7273     __ cset_w($res$$Register, eq);
7274   %}
7275   ins_pipe( long_memory_op );
7276 %}
7277 
7278 instruct compareAndSwapI_bool(memoryex mem, iRegI oldval, iRegI newval, iRegI res, iRegI tmp, flagsReg ccr) %{
7279   match(Set res (CompareAndSwapI mem (Binary oldval newval)));
7280   effect( KILL ccr, TEMP tmp);
7281   size(24);
7282   format %{ "loop:\n\t"
7283             "LDXR_w   $tmp, $mem\t! If $oldval==[$mem] Then store $newval into [$mem]\n\t"
7284             "CMP_w    $tmp, $oldval\n\t"
7285             "B.ne     done\n\t"
7286             "STXR_w   $tmp, $newval, $mem\n\t"
7287             "CBNZ_w   $tmp, loop\n\t"
7288             "done:\n\t"
7289             "CSET_w   $res, eq" %}
7290   ins_encode %{
7291     Register base = reg_to_register_object($mem$$base);
7292     Label loop, done;
7293     __ bind(loop);
7294     __ ldxr_w($tmp$$Register, base);
7295     __ cmp_w($tmp$$Register, $oldval$$Register);
7296     __ b(done, ne);
7297     __ stxr_w($tmp$$Register, $newval$$Register,  base);
7298     __ cbnz_w($tmp$$Register, loop);
7299     __ bind(done);
7300     __ cset_w($res$$Register, eq);
7301   %}
7302   ins_pipe( long_memory_op );
7303 %}
7304 
7305 // tmp must use iRegI instead of iRegN until 8051805 is fixed.
7306 instruct compareAndSwapN_bool(memoryex mem, iRegN oldval, iRegN newval, iRegI res, iRegI tmp, flagsReg ccr) %{
7307   match(Set res (CompareAndSwapN mem (Binary oldval newval)));
7308   effect( KILL ccr, TEMP tmp);
7309   size(24);
7310   format %{ "loop:\n\t"
7311             "LDXR_w   $tmp, $mem\t! If $oldval==[$mem] Then store $newval into [$mem]\n\t"
7312             "CMP_w    $tmp, $oldval\n\t"
7313             "B.ne     done\n\t"
7314             "STXR_w   $tmp, $newval, $mem\n\t"
7315             "CBNZ_w   $tmp, loop\n\t"
7316             "done:\n\t"
7317             "CSET_w   $res, eq" %}
7318   ins_encode %{
7319     Register base = reg_to_register_object($mem$$base);
7320     Label loop, done;
7321     __ bind(loop);
7322     __ ldxr_w($tmp$$Register, base);
7323     __ cmp_w($tmp$$Register, $oldval$$Register);
7324     __ b(done, ne);
7325     __ stxr_w($tmp$$Register, $newval$$Register,  base);
7326     __ cbnz_w($tmp$$Register, loop);
7327     __ bind(done);
7328     __ cset_w($res$$Register, eq);
7329   %}
7330   ins_pipe( long_memory_op );
7331 %}
7332 
7333 instruct compareAndSwapP_bool(memoryex mem, iRegP oldval, iRegP newval, iRegI res, iRegI tmp, flagsReg ccr) %{
7334   match(Set res (CompareAndSwapP mem (Binary oldval newval)));
7335   effect( KILL ccr, TEMP tmp);
7336   size(24);
7337   format %{ "loop:\n\t"
7338             "LDXR     $tmp, $mem\t! If $oldval==[$mem] Then store $newval into [$mem]\n\t"
7339             "CMP      $tmp, $oldval\n\t"
7340             "B.ne     done\n\t"
7341             "STXR     $tmp, $newval, $mem\n\t"
7342             "CBNZ_w   $tmp, loop\n\t"
7343             "done:\n\t"
7344             "CSET_w   $res, eq" %}
7345   ins_encode %{
7346     Register base = reg_to_register_object($mem$$base);
7347     Label loop, done;
7348     __ bind(loop);
7349     __ ldxr($tmp$$Register, base);
7350     __ cmp($tmp$$Register, $oldval$$Register);
7351     __ b(done, ne);
7352     __ stxr($tmp$$Register, $newval$$Register,  base);
7353     __ cbnz_w($tmp$$Register, loop);
7354     __ bind(done);
7355     __ cset_w($res$$Register, eq);
7356   %}
7357   ins_pipe( long_memory_op );
7358 %}
7359 #else // !AARCH64
7360 instruct compareAndSwapL_bool(memoryex mem, iRegL oldval, iRegLd newval, iRegI res, iRegLd tmp, flagsReg ccr ) %{
7361   match(Set res (CompareAndSwapL mem (Binary oldval newval)));
7362   effect( KILL ccr, TEMP tmp);
7363   size(32);
7364   format %{ "loop: \n\t"
7365             "LDREXD   $tmp, $mem\t! If $oldval==[$mem] Then store $newval into [$mem]\n\t"
7366             "CMP      $tmp.lo, $oldval.lo\n\t"
7367             "CMP.eq   $tmp.hi, $oldval.hi\n\t"
7368             "STREXD.eq $tmp, $newval, $mem\n\t"
7369             "MOV.ne   $tmp, 0 \n\t"
7370             "XORS.eq  $tmp,$tmp, 1 \n\t"
7371             "B.eq     loop \n\t"
7372             "MOV      $res, $tmp" %}
7373   ins_encode %{
7374     Label loop;
7375     __ bind(loop);
7376     __ ldrexd($tmp$$Register, $mem$$Address);
7377     __ cmp($tmp$$Register, $oldval$$Register);
7378     __ cmp($tmp$$Register->successor(), $oldval$$Register->successor(), eq);
7379     __ strexd($tmp$$Register, $newval$$Register, $mem$$Address, eq);
7380     __ mov($tmp$$Register, 0, ne);
7381     __ eors($tmp$$Register, $tmp$$Register, 1, eq);
7382     __ b(loop, eq);
7383     __ mov($res$$Register, $tmp$$Register);
7384   %}
7385   ins_pipe( long_memory_op );
7386 %}
7387 
7388 
7389 instruct compareAndSwapI_bool(memoryex mem, iRegI oldval, iRegI newval, iRegI res, iRegI tmp, flagsReg ccr ) %{
7390   match(Set res (CompareAndSwapI mem (Binary oldval newval)));
7391   effect( KILL ccr, TEMP tmp);
7392   size(28);
7393   format %{ "loop: \n\t"
7394             "LDREX    $tmp, $mem\t! If $oldval==[$mem] Then store $newval into [$mem]\n\t"
7395             "CMP      $tmp, $oldval\n\t"
7396             "STREX.eq $tmp, $newval, $mem\n\t"
7397             "MOV.ne   $tmp, 0 \n\t"
7398             "XORS.eq  $tmp,$tmp, 1 \n\t"
7399             "B.eq     loop \n\t"
7400             "MOV      $res, $tmp" %}
7401 
7402   ins_encode %{
7403     Label loop;
7404     __ bind(loop);
7405     __ ldrex($tmp$$Register,$mem$$Address);
7406     __ cmp($tmp$$Register, $oldval$$Register);
7407     __ strex($tmp$$Register, $newval$$Register, $mem$$Address, eq);
7408     __ mov($tmp$$Register, 0, ne);
7409     __ eors($tmp$$Register, $tmp$$Register, 1, eq);
7410     __ b(loop, eq);
7411     __ mov($res$$Register, $tmp$$Register);
7412   %}
7413   ins_pipe( long_memory_op );
7414 %}
7415 
7416 instruct compareAndSwapP_bool(memoryex mem, iRegP oldval, iRegP newval, iRegI res, iRegI tmp, flagsReg ccr ) %{
7417   match(Set res (CompareAndSwapP mem (Binary oldval newval)));
7418   effect( KILL ccr, TEMP tmp);
7419   size(28);
7420   format %{ "loop: \n\t"
7421             "LDREX    $tmp, $mem\t! If $oldval==[$mem] Then store $newval into [$mem]\n\t"
7422             "CMP      $tmp, $oldval\n\t"
7423             "STREX.eq $tmp, $newval, $mem\n\t"
7424             "MOV.ne   $tmp, 0 \n\t"
7425             "EORS.eq  $tmp,$tmp, 1 \n\t"
7426             "B.eq     loop \n\t"
7427             "MOV      $res, $tmp" %}
7428 
7429   ins_encode %{
7430     Label loop;
7431     __ bind(loop);
7432     __ ldrex($tmp$$Register,$mem$$Address);
7433     __ cmp($tmp$$Register, $oldval$$Register);
7434     __ strex($tmp$$Register, $newval$$Register, $mem$$Address, eq);
7435     __ mov($tmp$$Register, 0, ne);
7436     __ eors($tmp$$Register, $tmp$$Register, 1, eq);
7437     __ b(loop, eq);
7438     __ mov($res$$Register, $tmp$$Register);
7439   %}
7440   ins_pipe( long_memory_op );
7441 %}
7442 #endif // !AARCH64
7443 
7444 #ifdef AARCH64
7445 instruct xaddI_aimmI_no_res(memoryex mem, aimmI add, Universe dummy, iRegI tmp1, iRegI tmp2) %{
7446   predicate(n->as_LoadStore()->result_not_used());
7447   match(Set dummy (GetAndAddI mem add));
7448   effect(TEMP tmp1, TEMP tmp2);
7449   size(16);
7450   format %{ "loop:\n\t"
7451             "LDXR_w   $tmp1, $mem\n\t"
7452             "ADD_w    $tmp1, $tmp1, $add\n\t"
7453             "STXR_w   $tmp2, $tmp1, $mem\n\t"
7454             "CBNZ_w   $tmp2, loop" %}
7455 
7456   ins_encode %{
7457     Label loop;
7458     Register base = reg_to_register_object($mem$$base);
7459     __ bind(loop);
7460     __ ldxr_w($tmp1$$Register, base);
7461     __ add_w($tmp1$$Register, $tmp1$$Register, $add$$constant);
7462     __ stxr_w($tmp2$$Register, $tmp1$$Register, base);
7463     __ cbnz_w($tmp2$$Register, loop);
7464   %}
7465   ins_pipe( long_memory_op );
7466 %}
7467 #else
7468 instruct xaddI_aimmI_no_res(memoryex mem, aimmI add, Universe dummy, iRegI tmp1, iRegI tmp2, flagsReg ccr) %{
7469   predicate(n->as_LoadStore()->result_not_used());
7470   match(Set dummy (GetAndAddI mem add));
7471   effect(KILL ccr, TEMP tmp1, TEMP tmp2);
7472   size(20);
7473   format %{ "loop: \n\t"
7474             "LDREX    $tmp1, $mem\n\t"
7475             "ADD      $tmp1, $tmp1, $add\n\t"
7476             "STREX    $tmp2, $tmp1, $mem\n\t"
7477             "CMP      $tmp2, 0 \n\t"
7478             "B.ne     loop \n\t" %}
7479 
7480   ins_encode %{
7481     Label loop;
7482     __ bind(loop);
7483     __ ldrex($tmp1$$Register,$mem$$Address);
7484     __ add($tmp1$$Register, $tmp1$$Register, $add$$constant);
7485     __ strex($tmp2$$Register, $tmp1$$Register, $mem$$Address);
7486     __ cmp($tmp2$$Register, 0);
7487     __ b(loop, ne);
7488   %}
7489   ins_pipe( long_memory_op );
7490 %}
7491 #endif
7492 
7493 #ifdef AARCH64
7494 instruct xaddI_reg_no_res(memoryex mem, iRegI add, Universe dummy, iRegI tmp1, iRegI tmp2) %{
7495   predicate(n->as_LoadStore()->result_not_used());
7496   match(Set dummy (GetAndAddI mem add));
7497   effect(TEMP tmp1, TEMP tmp2);
7498   size(16);
7499   format %{ "loop:\n\t"
7500             "LDXR_w   $tmp1, $mem\n\t"
7501             "ADD_w    $tmp1, $tmp1, $add\n\t"
7502             "STXR_w   $tmp2, $tmp1, $mem\n\t"
7503             "CBNZ_w   $tmp2, loop" %}
7504 
7505   ins_encode %{
7506     Label loop;
7507     Register base = reg_to_register_object($mem$$base);
7508     __ bind(loop);
7509     __ ldxr_w($tmp1$$Register, base);
7510     __ add_w($tmp1$$Register, $tmp1$$Register, $add$$Register);
7511     __ stxr_w($tmp2$$Register, $tmp1$$Register, base);
7512     __ cbnz_w($tmp2$$Register, loop);
7513   %}
7514   ins_pipe( long_memory_op );
7515 %}
7516 #else
7517 instruct xaddI_reg_no_res(memoryex mem, iRegI add, Universe dummy, iRegI tmp1, iRegI tmp2, flagsReg ccr) %{
7518   predicate(n->as_LoadStore()->result_not_used());
7519   match(Set dummy (GetAndAddI mem add));
7520   effect(KILL ccr, TEMP tmp1, TEMP tmp2);
7521   size(20);
7522   format %{ "loop: \n\t"
7523             "LDREX    $tmp1, $mem\n\t"
7524             "ADD      $tmp1, $tmp1, $add\n\t"
7525             "STREX    $tmp2, $tmp1, $mem\n\t"
7526             "CMP      $tmp2, 0 \n\t"
7527             "B.ne     loop \n\t" %}
7528 
7529   ins_encode %{
7530     Label loop;
7531     __ bind(loop);
7532     __ ldrex($tmp1$$Register,$mem$$Address);
7533     __ add($tmp1$$Register, $tmp1$$Register, $add$$Register);
7534     __ strex($tmp2$$Register, $tmp1$$Register, $mem$$Address);
7535     __ cmp($tmp2$$Register, 0);
7536     __ b(loop, ne);
7537   %}
7538   ins_pipe( long_memory_op );
7539 %}
7540 #endif
7541 
7542 #ifdef AARCH64
7543 instruct xaddI_aimmI(memoryex mem, aimmI add, iRegI res, iRegI tmp1, iRegI tmp2) %{
7544   match(Set res (GetAndAddI mem add));
7545   effect(TEMP tmp1, TEMP tmp2, TEMP res);
7546   size(16);
7547   format %{ "loop:\n\t"
7548             "LDXR_w   $res, $mem\n\t"
7549             "ADD_w    $tmp1, $res, $add\n\t"
7550             "STXR_w   $tmp2, $tmp1, $mem\n\t"
7551             "CBNZ_w   $tmp2, loop" %}
7552 
7553   ins_encode %{
7554     Label loop;
7555     Register base = reg_to_register_object($mem$$base);
7556     __ bind(loop);
7557     __ ldxr_w($res$$Register, base);
7558     __ add_w($tmp1$$Register, $res$$Register, $add$$constant);
7559     __ stxr_w($tmp2$$Register, $tmp1$$Register, base);
7560     __ cbnz_w($tmp2$$Register, loop);
7561   %}
7562   ins_pipe( long_memory_op );
7563 %}
7564 #else
7565 instruct xaddI_aimmI(memoryex mem, aimmI add, iRegI res, iRegI tmp1, iRegI tmp2, flagsReg ccr) %{
7566   match(Set res (GetAndAddI mem add));
7567   effect(KILL ccr, TEMP tmp1, TEMP tmp2, TEMP res);
7568   size(20);
7569   format %{ "loop: \n\t"
7570             "LDREX    $res, $mem\n\t"
7571             "ADD      $tmp1, $res, $add\n\t"
7572             "STREX    $tmp2, $tmp1, $mem\n\t"
7573             "CMP      $tmp2, 0 \n\t"
7574             "B.ne     loop \n\t" %}
7575 
7576   ins_encode %{
7577     Label loop;
7578     __ bind(loop);
7579     __ ldrex($res$$Register,$mem$$Address);
7580     __ add($tmp1$$Register, $res$$Register, $add$$constant);
7581     __ strex($tmp2$$Register, $tmp1$$Register, $mem$$Address);
7582     __ cmp($tmp2$$Register, 0);
7583     __ b(loop, ne);
7584   %}
7585   ins_pipe( long_memory_op );
7586 %}
7587 #endif
7588 
7589 #ifdef AARCH64
7590 instruct xaddI_reg(memoryex mem, iRegI add, iRegI res, iRegI tmp1, iRegI tmp2) %{
7591   match(Set res (GetAndAddI mem add));
7592   effect(TEMP tmp1, TEMP tmp2, TEMP res);
7593   size(16);
7594   format %{ "loop:\n\t"
7595             "LDXR_w   $res, $mem\n\t"
7596             "ADD_w    $tmp1, $res, $add\n\t"
7597             "STXR_w   $tmp2, $tmp1, $mem\n\t"
7598             "CBNZ_w   $tmp2, loop" %}
7599 
7600   ins_encode %{
7601     Label loop;
7602     Register base = reg_to_register_object($mem$$base);
7603     __ bind(loop);
7604     __ ldxr_w($res$$Register, base);
7605     __ add_w($tmp1$$Register, $res$$Register, $add$$Register);
7606     __ stxr_w($tmp2$$Register, $tmp1$$Register, base);
7607     __ cbnz_w($tmp2$$Register, loop);
7608   %}
7609   ins_pipe( long_memory_op );
7610 %}
7611 #else
7612 instruct xaddI_reg(memoryex mem, iRegI add, iRegI res, iRegI tmp1, iRegI tmp2, flagsReg ccr) %{
7613   match(Set res (GetAndAddI mem add));
7614   effect(KILL ccr, TEMP tmp1, TEMP tmp2, TEMP res);
7615   size(20);
7616   format %{ "loop: \n\t"
7617             "LDREX    $res, $mem\n\t"
7618             "ADD      $tmp1, $res, $add\n\t"
7619             "STREX    $tmp2, $tmp1, $mem\n\t"
7620             "CMP      $tmp2, 0 \n\t"
7621             "B.ne     loop \n\t" %}
7622 
7623   ins_encode %{
7624     Label loop;
7625     __ bind(loop);
7626     __ ldrex($res$$Register,$mem$$Address);
7627     __ add($tmp1$$Register, $res$$Register, $add$$Register);
7628     __ strex($tmp2$$Register, $tmp1$$Register, $mem$$Address);
7629     __ cmp($tmp2$$Register, 0);
7630     __ b(loop, ne);
7631   %}
7632   ins_pipe( long_memory_op );
7633 %}
7634 #endif
7635 
7636 #ifdef AARCH64
7637 instruct xaddL_reg_no_res(memoryex mem, iRegL add, Universe dummy, iRegL tmp1, iRegI tmp2) %{
7638   predicate(n->as_LoadStore()->result_not_used());
7639   match(Set dummy (GetAndAddL mem add));
7640   effect(TEMP tmp1, TEMP tmp2);
7641   size(16);
7642   format %{ "loop:\n\t"
7643             "LDXR     $tmp1, $mem\n\t"
7644             "ADD      $tmp1, $tmp1, $add\n\t"
7645             "STXR     $tmp2, $tmp1, $mem\n\t"
7646             "CBNZ_w   $tmp2, loop" %}
7647 
7648   ins_encode %{
7649     Label loop;
7650     Register base = reg_to_register_object($mem$$base);
7651     __ bind(loop);
7652     __ ldxr($tmp1$$Register, base);
7653     __ add($tmp1$$Register, $tmp1$$Register, $add$$Register);
7654     __ stxr($tmp2$$Register, $tmp1$$Register, base);
7655     __ cbnz_w($tmp2$$Register, loop);
7656   %}
7657   ins_pipe( long_memory_op );
7658 %}
7659 #else
7660 instruct xaddL_reg_no_res(memoryex mem, iRegL add, Universe dummy, iRegLd tmp1, iRegI tmp2, flagsReg ccr) %{
7661   predicate(n->as_LoadStore()->result_not_used());
7662   match(Set dummy (GetAndAddL mem add));
7663   effect( KILL ccr, TEMP tmp1, TEMP tmp2);
7664   size(24);
7665   format %{ "loop: \n\t"
7666             "LDREXD   $tmp1, $mem\n\t"
7667             "ADDS     $tmp1.lo, $tmp1.lo, $add.lo\n\t"
7668             "ADC      $tmp1.hi, $tmp1.hi, $add.hi\n\t"
7669             "STREXD   $tmp2, $tmp1, $mem\n\t"
7670             "CMP      $tmp2, 0 \n\t"
7671             "B.ne     loop \n\t" %}
7672 
7673   ins_encode %{
7674     Label loop;
7675     __ bind(loop);
7676     __ ldrexd($tmp1$$Register, $mem$$Address);
7677     __ adds($tmp1$$Register, $tmp1$$Register, $add$$Register);
7678     __ adc($tmp1$$Register->successor(), $tmp1$$Register->successor(), $add$$Register->successor());
7679     __ strexd($tmp2$$Register, $tmp1$$Register, $mem$$Address);
7680     __ cmp($tmp2$$Register, 0);
7681     __ b(loop, ne);
7682   %}
7683   ins_pipe( long_memory_op );
7684 %}
7685 #endif
7686 
7687 #ifdef AARCH64
7688 instruct xaddL_imm_no_res(memoryex mem, aimmL add, Universe dummy, iRegL tmp1, iRegI tmp2) %{
7689   predicate(n->as_LoadStore()->result_not_used());
7690   match(Set dummy (GetAndAddL mem add));
7691   effect(TEMP tmp1, TEMP tmp2);
7692   size(16);
7693   format %{ "loop:\n\t"
7694             "LDXR     $tmp1, $mem\n\t"
7695             "ADD      $tmp1, $tmp1, $add\n\t"
7696             "STXR     $tmp2, $tmp1, $mem\n\t"
7697             "CBNZ_w   $tmp2, loop" %}
7698 
7699   ins_encode %{
7700     Label loop;
7701     Register base = reg_to_register_object($mem$$base);
7702     __ bind(loop);
7703     __ ldxr($tmp1$$Register, base);
7704     __ add($tmp1$$Register, $tmp1$$Register, $add$$constant);
7705     __ stxr($tmp2$$Register, $tmp1$$Register, base);
7706     __ cbnz_w($tmp2$$Register, loop);
7707   %}
7708   ins_pipe( long_memory_op );
7709 %}
7710 #else
7711 // TODO: try immLRot2 instead, (0, $con$$constant) becomes
7712 // (hi($con$$constant), lo($con$$constant)) becomes
7713 instruct xaddL_immRot_no_res(memoryex mem, immLlowRot add, Universe dummy, iRegLd tmp1, iRegI tmp2, flagsReg ccr) %{
7714   predicate(n->as_LoadStore()->result_not_used());
7715   match(Set dummy (GetAndAddL mem add));
7716   effect( KILL ccr, TEMP tmp1, TEMP tmp2);
7717   size(24);
7718   format %{ "loop: \n\t"
7719             "LDREXD   $tmp1, $mem\n\t"
7720             "ADDS     $tmp1.lo, $tmp1.lo, $add\n\t"
7721             "ADC      $tmp1.hi, $tmp1.hi, 0\n\t"
7722             "STREXD   $tmp2, $tmp1, $mem\n\t"
7723             "CMP      $tmp2, 0 \n\t"
7724             "B.ne     loop \n\t" %}
7725 
7726   ins_encode %{
7727     Label loop;
7728     __ bind(loop);
7729     __ ldrexd($tmp1$$Register, $mem$$Address);
7730     __ adds($tmp1$$Register, $tmp1$$Register, $add$$constant);
7731     __ adc($tmp1$$Register->successor(), $tmp1$$Register->successor(), 0);
7732     __ strexd($tmp2$$Register, $tmp1$$Register, $mem$$Address);
7733     __ cmp($tmp2$$Register, 0);
7734     __ b(loop, ne);
7735   %}
7736   ins_pipe( long_memory_op );
7737 %}
7738 #endif
7739 
7740 #ifdef AARCH64
7741 instruct xaddL_reg(memoryex mem, iRegL add, iRegL res, iRegL tmp1, iRegI tmp2) %{
7742   match(Set res (GetAndAddL mem add));
7743   effect(TEMP tmp1, TEMP tmp2, TEMP res);
7744   size(16);
7745   format %{ "loop:\n\t"
7746             "LDXR     $res, $mem\n\t"
7747             "ADD      $tmp1, $res, $add\n\t"
7748             "STXR     $tmp2, $tmp1, $mem\n\t"
7749             "CBNZ_w   $tmp2, loop" %}
7750 
7751   ins_encode %{
7752     Label loop;
7753     Register base = reg_to_register_object($mem$$base);
7754     __ bind(loop);
7755     __ ldxr($res$$Register, base);
7756     __ add($tmp1$$Register, $res$$Register, $add$$Register);
7757     __ stxr($tmp2$$Register, $tmp1$$Register, base);
7758     __ cbnz_w($tmp2$$Register, loop);
7759   %}
7760   ins_pipe( long_memory_op );
7761 %}
7762 #else
7763 instruct xaddL_reg(memoryex mem, iRegL add, iRegLd res, iRegLd tmp1, iRegI tmp2, flagsReg ccr) %{
7764   match(Set res (GetAndAddL mem add));
7765   effect( KILL ccr, TEMP tmp1, TEMP tmp2, TEMP res);
7766   size(24);
7767   format %{ "loop: \n\t"
7768             "LDREXD   $res, $mem\n\t"
7769             "ADDS     $tmp1.lo, $res.lo, $add.lo\n\t"
7770             "ADC      $tmp1.hi, $res.hi, $add.hi\n\t"
7771             "STREXD   $tmp2, $tmp1, $mem\n\t"
7772             "CMP      $tmp2, 0 \n\t"
7773             "B.ne     loop \n\t" %}
7774 
7775   ins_encode %{
7776     Label loop;
7777     __ bind(loop);
7778     __ ldrexd($res$$Register, $mem$$Address);
7779     __ adds($tmp1$$Register, $res$$Register, $add$$Register);
7780     __ adc($tmp1$$Register->successor(), $res$$Register->successor(), $add$$Register->successor());
7781     __ strexd($tmp2$$Register, $tmp1$$Register, $mem$$Address);
7782     __ cmp($tmp2$$Register, 0);
7783     __ b(loop, ne);
7784   %}
7785   ins_pipe( long_memory_op );
7786 %}
7787 #endif
7788 
7789 #ifdef AARCH64
7790 instruct xaddL_imm(memoryex mem, aimmL add, iRegL res, iRegL tmp1, iRegI tmp2) %{
7791   match(Set res (GetAndAddL mem add));
7792   effect(TEMP tmp1, TEMP tmp2, TEMP res);
7793   size(16);
7794   format %{ "loop:\n\t"
7795             "LDXR     $res, $mem\n\t"
7796             "ADD      $tmp1, $res, $add\n\t"
7797             "STXR     $tmp2, $tmp1, $mem\n\t"
7798             "CBNZ_w   $tmp2, loop" %}
7799 
7800   ins_encode %{
7801     Label loop;
7802     Register base = reg_to_register_object($mem$$base);
7803     __ bind(loop);
7804     __ ldxr($res$$Register, base);
7805     __ add($tmp1$$Register, $res$$Register, $add$$constant);
7806     __ stxr($tmp2$$Register, $tmp1$$Register, base);
7807     __ cbnz_w($tmp2$$Register, loop);
7808   %}
7809   ins_pipe( long_memory_op );
7810 %}
7811 #else
7812 // TODO: try immLRot2 instead, (0, $con$$constant) becomes
7813 // (hi($con$$constant), lo($con$$constant)) becomes
7814 instruct xaddL_immRot(memoryex mem, immLlowRot add, iRegLd res, iRegLd tmp1, iRegI tmp2, flagsReg ccr) %{
7815   match(Set res (GetAndAddL mem add));
7816   effect( KILL ccr, TEMP tmp1, TEMP tmp2, TEMP res);
7817   size(24);
7818   format %{ "loop: \n\t"
7819             "LDREXD   $res, $mem\n\t"
7820             "ADDS     $tmp1.lo, $res.lo, $add\n\t"
7821             "ADC      $tmp1.hi, $res.hi, 0\n\t"
7822             "STREXD   $tmp2, $tmp1, $mem\n\t"
7823             "CMP      $tmp2, 0 \n\t"
7824             "B.ne     loop \n\t" %}
7825 
7826   ins_encode %{
7827     Label loop;
7828     __ bind(loop);
7829     __ ldrexd($res$$Register, $mem$$Address);
7830     __ adds($tmp1$$Register, $res$$Register, $add$$constant);
7831     __ adc($tmp1$$Register->successor(), $res$$Register->successor(), 0);
7832     __ strexd($tmp2$$Register, $tmp1$$Register, $mem$$Address);
7833     __ cmp($tmp2$$Register, 0);
7834     __ b(loop, ne);
7835   %}
7836   ins_pipe( long_memory_op );
7837 %}
7838 #endif
7839 
7840 #ifdef AARCH64
7841 instruct xchgI(memoryex mem, iRegI newval, iRegI res, iRegI tmp) %{
7842   match(Set res (GetAndSetI mem newval));
7843   effect(TEMP tmp, TEMP res);
7844   size(12);
7845   format %{ "loop:\n\t"
7846             "LDXR_w   $res, $mem\n\t"
7847             "STXR_w   $tmp, $newval, $mem\n\t"
7848             "CBNZ_w   $tmp, loop" %}
7849 
7850   ins_encode %{
7851     Label loop;
7852     Register base = reg_to_register_object($mem$$base);
7853     __ bind(loop);
7854     __ ldxr_w($res$$Register, base);
7855     __ stxr_w($tmp$$Register, $newval$$Register, base);
7856     __ cbnz_w($tmp$$Register, loop);
7857   %}
7858   ins_pipe( long_memory_op );
7859 %}
7860 
7861 #ifdef XXX
7862 // Disabled until 8051805 is fixed.
7863 instruct xchgN(memoryex mem, iRegN newval, iRegN res, iRegN tmp) %{
7864   match(Set res (GetAndSetN mem newval));
7865   effect(TEMP tmp, TEMP res);
7866   size(12);
7867   format %{ "loop:\n\t"
7868             "LDXR_w   $res, $mem\n\t"
7869             "STXR_w   $tmp, $newval, $mem\n\t"
7870             "CBNZ_w   $tmp, loop" %}
7871 
7872   ins_encode %{
7873     Label loop;
7874     Register base = reg_to_register_object($mem$$base);
7875     __ bind(loop);
7876     __ ldxr_w($res$$Register, base);
7877     __ stxr_w($tmp$$Register, $newval$$Register, base);
7878     __ cbnz_w($tmp$$Register, loop);
7879   %}
7880   ins_pipe( long_memory_op );
7881 %}
7882 #endif
7883 #else
7884 instruct xchgI(memoryex mem, iRegI newval, iRegI res, iRegI tmp, flagsReg ccr) %{
7885   match(Set res (GetAndSetI mem newval));
7886   effect(KILL ccr, TEMP tmp, TEMP res);
7887   size(16);
7888   format %{ "loop: \n\t"
7889             "LDREX    $res, $mem\n\t"
7890             "STREX    $tmp, $newval, $mem\n\t"
7891             "CMP      $tmp, 0 \n\t"
7892             "B.ne     loop \n\t" %}
7893 
7894   ins_encode %{
7895     Label loop;
7896     __ bind(loop);
7897     __ ldrex($res$$Register,$mem$$Address);
7898     __ strex($tmp$$Register, $newval$$Register, $mem$$Address);
7899     __ cmp($tmp$$Register, 0);
7900     __ b(loop, ne);
7901   %}
7902   ins_pipe( long_memory_op );
7903 %}
7904 #endif
7905 
7906 #ifdef AARCH64
7907 instruct xchgL(memoryex mem, iRegL newval, iRegL res, iRegI tmp) %{
7908   match(Set res (GetAndSetL mem newval));
7909   effect(TEMP tmp, TEMP res);
7910   size(12);
7911   format %{ "loop:\n\t"
7912             "LDXR     $res, $mem\n\t"
7913             "STXR     $tmp, $newval, $mem\n\t"
7914             "CBNZ_w   $tmp, loop" %}
7915 
7916   ins_encode %{
7917     Label loop;
7918     Register base = reg_to_register_object($mem$$base);
7919     __ bind(loop);
7920     __ ldxr($res$$Register, base);
7921     __ stxr($tmp$$Register, $newval$$Register, base);
7922     __ cbnz_w($tmp$$Register, loop);
7923   %}
7924   ins_pipe( long_memory_op );
7925 %}
7926 #else
7927 instruct xchgL(memoryex mem, iRegLd newval, iRegLd res, iRegI tmp, flagsReg ccr) %{
7928   match(Set res (GetAndSetL mem newval));
7929   effect( KILL ccr, TEMP tmp, TEMP res);
7930   size(16);
7931   format %{ "loop: \n\t"
7932             "LDREXD   $res, $mem\n\t"
7933             "STREXD   $tmp, $newval, $mem\n\t"
7934             "CMP      $tmp, 0 \n\t"
7935             "B.ne     loop \n\t" %}
7936 
7937   ins_encode %{
7938     Label loop;
7939     __ bind(loop);
7940     __ ldrexd($res$$Register, $mem$$Address);
7941     __ strexd($tmp$$Register, $newval$$Register, $mem$$Address);
7942     __ cmp($tmp$$Register, 0);
7943     __ b(loop, ne);
7944   %}
7945   ins_pipe( long_memory_op );
7946 %}
7947 #endif // !AARCH64
7948 
7949 #ifdef AARCH64
7950 instruct xchgP(memoryex mem, iRegP newval, iRegP res, iRegI tmp) %{
7951   match(Set res (GetAndSetP mem newval));
7952   effect(TEMP tmp, TEMP res);
7953   size(12);
7954   format %{ "loop:\n\t"
7955             "LDREX    $res, $mem\n\t"
7956             "STREX    $tmp, $newval, $mem\n\t"
7957             "CBNZ_w   $tmp, loop" %}
7958 
7959   ins_encode %{
7960     Label loop;
7961     Register base = reg_to_register_object($mem$$base);
7962     __ bind(loop);
7963     __ ldrex($res$$Register, base);
7964     __ strex($tmp$$Register, $newval$$Register, base);
7965     __ cbnz_w($tmp$$Register, loop);
7966   %}
7967   ins_pipe( long_memory_op );
7968 %}
7969 #else
7970 instruct xchgP(memoryex mem, iRegP newval, iRegP res, iRegI tmp, flagsReg ccr) %{
7971   match(Set res (GetAndSetP mem newval));
7972   effect(KILL ccr, TEMP tmp, TEMP res);
7973   size(16);
7974   format %{ "loop: \n\t"
7975             "LDREX    $res, $mem\n\t"
7976             "STREX    $tmp, $newval, $mem\n\t"
7977             "CMP      $tmp, 0 \n\t"
7978             "B.ne     loop \n\t" %}
7979 
7980   ins_encode %{
7981     Label loop;
7982     __ bind(loop);
7983     __ ldrex($res$$Register,$mem$$Address);
7984     __ strex($tmp$$Register, $newval$$Register, $mem$$Address);
7985     __ cmp($tmp$$Register, 0);
7986     __ b(loop, ne);
7987   %}
7988   ins_pipe( long_memory_op );
7989 %}
7990 #endif // !AARCH64
7991 
7992 //---------------------
7993 // Subtraction Instructions
7994 // Register Subtraction
7995 instruct subI_reg_reg(iRegI dst, iRegI src1, iRegI src2) %{
7996   match(Set dst (SubI src1 src2));
7997 
7998   size(4);
7999   format %{ "sub_32 $dst,$src1,$src2\t! int" %}
8000   ins_encode %{
8001     __ sub_32($dst$$Register, $src1$$Register, $src2$$Register);
8002   %}
8003   ins_pipe(ialu_reg_reg);
8004 %}
8005 
8006 #ifndef AARCH64
8007 instruct subshlI_reg_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3) %{
8008   match(Set dst (SubI src1 (LShiftI src2 src3)));
8009 
8010   size(4);
8011   format %{ "SUB    $dst,$src1,$src2<<$src3" %}
8012   ins_encode %{
8013     __ sub($dst$$Register, $src1$$Register, AsmOperand($src2$$Register, lsl, $src3$$Register));
8014   %}
8015   ins_pipe(ialu_reg_reg);
8016 %}
8017 #endif
8018 
8019 instruct subshlI_reg_reg_imm(iRegI dst, iRegI src1, iRegI src2, immU5 src3) %{
8020   match(Set dst (SubI src1 (LShiftI src2 src3)));
8021 
8022   size(4);
8023   format %{ "sub_32 $dst,$src1,$src2<<$src3\t! int" %}
8024   ins_encode %{
8025     __ sub_32($dst$$Register, $src1$$Register, AsmOperand($src2$$Register, lsl, $src3$$constant));
8026   %}
8027   ins_pipe(ialu_reg_reg);
8028 %}
8029 
8030 #ifndef AARCH64
8031 instruct subsarI_reg_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3) %{
8032   match(Set dst (SubI src1 (RShiftI src2 src3)));
8033 
8034   size(4);
8035   format %{ "SUB    $dst,$src1,$src2>>$src3" %}
8036   ins_encode %{
8037     __ sub($dst$$Register, $src1$$Register, AsmOperand($src2$$Register, asr, $src3$$Register));
8038   %}
8039   ins_pipe(ialu_reg_reg);
8040 %}
8041 #endif
8042 
8043 instruct subsarI_reg_reg_imm(iRegI dst, iRegI src1, iRegI src2, immU5 src3) %{
8044   match(Set dst (SubI src1 (RShiftI src2 src3)));
8045 
8046   size(4);
8047   format %{ "sub_32 $dst,$src1,$src2>>$src3\t! int" %}
8048   ins_encode %{
8049     __ sub_32($dst$$Register, $src1$$Register, AsmOperand($src2$$Register, asr, $src3$$constant));
8050   %}
8051   ins_pipe(ialu_reg_reg);
8052 %}
8053 
8054 #ifndef AARCH64
8055 instruct subshrI_reg_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3) %{
8056   match(Set dst (SubI src1 (URShiftI src2 src3)));
8057 
8058   size(4);
8059   format %{ "SUB    $dst,$src1,$src2>>>$src3" %}
8060   ins_encode %{
8061     __ sub($dst$$Register, $src1$$Register, AsmOperand($src2$$Register, lsr, $src3$$Register));
8062   %}
8063   ins_pipe(ialu_reg_reg);
8064 %}
8065 #endif
8066 
8067 instruct subshrI_reg_reg_imm(iRegI dst, iRegI src1, iRegI src2, immU5 src3) %{
8068   match(Set dst (SubI src1 (URShiftI src2 src3)));
8069 
8070   size(4);
8071   format %{ "sub_32 $dst,$src1,$src2>>>$src3\t! int" %}
8072   ins_encode %{
8073     __ sub_32($dst$$Register, $src1$$Register, AsmOperand($src2$$Register, lsr, $src3$$constant));
8074   %}
8075   ins_pipe(ialu_reg_reg);
8076 %}
8077 
8078 #ifndef AARCH64
8079 instruct rsbshlI_reg_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3) %{
8080   match(Set dst (SubI (LShiftI src1 src2) src3));
8081 
8082   size(4);
8083   format %{ "RSB    $dst,$src3,$src1<<$src2" %}
8084   ins_encode %{
8085     __ rsb($dst$$Register, $src3$$Register, AsmOperand($src1$$Register, lsl, $src2$$Register));
8086   %}
8087   ins_pipe(ialu_reg_reg);
8088 %}
8089 
8090 instruct rsbshlI_reg_imm_reg(iRegI dst, iRegI src1, immU5 src2, iRegI src3) %{
8091   match(Set dst (SubI (LShiftI src1 src2) src3));
8092 
8093   size(4);
8094   format %{ "RSB    $dst,$src3,$src1<<$src2" %}
8095   ins_encode %{
8096     __ rsb($dst$$Register, $src3$$Register, AsmOperand($src1$$Register, lsl, $src2$$constant));
8097   %}
8098   ins_pipe(ialu_reg_reg);
8099 %}
8100 
8101 instruct rsbsarI_reg_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3) %{
8102   match(Set dst (SubI (RShiftI src1 src2) src3));
8103 
8104   size(4);
8105   format %{ "RSB    $dst,$src3,$src1>>$src2" %}
8106   ins_encode %{
8107     __ rsb($dst$$Register, $src3$$Register, AsmOperand($src1$$Register, asr, $src2$$Register));
8108   %}
8109   ins_pipe(ialu_reg_reg);
8110 %}
8111 
8112 instruct rsbsarI_reg_imm_reg(iRegI dst, iRegI src1, immU5 src2, iRegI src3) %{
8113   match(Set dst (SubI (RShiftI src1 src2) src3));
8114 
8115   size(4);
8116   format %{ "RSB    $dst,$src3,$src1>>$src2" %}
8117   ins_encode %{
8118     __ rsb($dst$$Register, $src3$$Register, AsmOperand($src1$$Register, asr, $src2$$constant));
8119   %}
8120   ins_pipe(ialu_reg_reg);
8121 %}
8122 
8123 instruct rsbshrI_reg_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3) %{
8124   match(Set dst (SubI (URShiftI src1 src2) src3));
8125 
8126   size(4);
8127   format %{ "RSB    $dst,$src3,$src1>>>$src2" %}
8128   ins_encode %{
8129     __ rsb($dst$$Register, $src3$$Register, AsmOperand($src1$$Register, lsr, $src2$$Register));
8130   %}
8131   ins_pipe(ialu_reg_reg);
8132 %}
8133 
8134 instruct rsbshrI_reg_imm_reg(iRegI dst, iRegI src1, immU5 src2, iRegI src3) %{
8135   match(Set dst (SubI (URShiftI src1 src2) src3));
8136 
8137   size(4);
8138   format %{ "RSB    $dst,$src3,$src1>>>$src2" %}
8139   ins_encode %{
8140     __ rsb($dst$$Register, $src3$$Register, AsmOperand($src1$$Register, lsr, $src2$$constant));
8141   %}
8142   ins_pipe(ialu_reg_reg);
8143 %}
8144 #endif
8145 
8146 // Immediate Subtraction
8147 instruct subI_reg_aimmI(iRegI dst, iRegI src1, aimmI src2) %{
8148   match(Set dst (SubI src1 src2));
8149 
8150   size(4);
8151   format %{ "sub_32 $dst,$src1,$src2\t! int" %}
8152   ins_encode %{
8153     __ sub_32($dst$$Register, $src1$$Register, $src2$$constant);
8154   %}
8155   ins_pipe(ialu_reg_imm);
8156 %}
8157 
8158 instruct subI_reg_immRotneg(iRegI dst, iRegI src1, aimmIneg src2) %{
8159   match(Set dst (AddI src1 src2));
8160 
8161   size(4);
8162   format %{ "sub_32 $dst,$src1,-($src2)\t! int" %}
8163   ins_encode %{
8164     __ sub_32($dst$$Register, $src1$$Register, -$src2$$constant);
8165   %}
8166   ins_pipe(ialu_reg_imm);
8167 %}
8168 
8169 #ifndef AARCH64
8170 instruct subI_immRot_reg(iRegI dst, immIRot src1, iRegI src2) %{
8171   match(Set dst (SubI src1 src2));
8172 
8173   size(4);
8174   format %{ "RSB    $dst,$src2,src1" %}
8175   ins_encode %{
8176     __ rsb($dst$$Register, $src2$$Register, $src1$$constant);
8177   %}
8178   ins_pipe(ialu_zero_reg);
8179 %}
8180 #endif
8181 
8182 // Register Subtraction
8183 #ifdef AARCH64
8184 instruct subL_reg_reg(iRegL dst, iRegL src1, iRegL src2) %{
8185   match(Set dst (SubL src1 src2));
8186 
8187   size(4);
8188   format %{ "SUB    $dst,$src1,$src2\t! long" %}
8189   ins_encode %{
8190     __ sub($dst$$Register, $src1$$Register, $src2$$Register);
8191   %}
8192   ins_pipe(ialu_reg_reg);
8193 %}
8194 #else
8195 instruct subL_reg_reg(iRegL dst, iRegL src1, iRegL src2, flagsReg icc ) %{
8196   match(Set dst (SubL src1 src2));
8197   effect (KILL icc);
8198 
8199   size(8);
8200   format %{ "SUBS   $dst.lo,$src1.lo,$src2.lo\t! long\n\t"
8201             "SBC    $dst.hi,$src1.hi,$src2.hi" %}
8202   ins_encode %{
8203     __ subs($dst$$Register, $src1$$Register, $src2$$Register);
8204     __ sbc($dst$$Register->successor(), $src1$$Register->successor(), $src2$$Register->successor());
8205   %}
8206   ins_pipe(ialu_reg_reg);
8207 %}
8208 #endif
8209 
8210 #ifdef AARCH64
8211 // Immediate Subtraction
8212 instruct subL_reg_aimm(iRegL dst, iRegL src1, aimmL src2) %{
8213   match(Set dst (SubL src1 src2));
8214 
8215   size(4);
8216   format %{ "SUB    $dst,$src1,$src2\t! long" %}
8217   ins_encode %{
8218     __ sub($dst$$Register, $src1$$Register, $src2$$constant);
8219   %}
8220   ins_pipe(ialu_reg_imm);
8221 %}
8222 
8223 instruct subL_reg_immLneg(iRegL dst, iRegL src1, aimmLneg src2) %{
8224   match(Set dst (AddL src1 src2));
8225 
8226   size(4);
8227   format %{ "SUB    $dst,$src1,-($src2)\t! long" %}
8228   ins_encode %{
8229     __ sub($dst$$Register, $src1$$Register, -$src2$$constant);
8230   %}
8231   ins_pipe(ialu_reg_imm);
8232 %}
8233 #else
8234 // TODO
8235 #endif
8236 
8237 #ifndef AARCH64
8238 // Immediate Subtraction
8239 // TODO: try immLRot2 instead, (0, $con$$constant) becomes
8240 // (hi($con$$constant), lo($con$$constant)) becomes
8241 instruct subL_reg_immRot(iRegL dst, iRegL src1, immLlowRot con, flagsReg icc) %{
8242   match(Set dst (SubL src1 con));
8243   effect (KILL icc);
8244 
8245   size(8);
8246   format %{ "SUB    $dst.lo,$src1.lo,$con\t! long\n\t"
8247             "SBC    $dst.hi,$src1.hi,0" %}
8248   ins_encode %{
8249     __ subs($dst$$Register, $src1$$Register, $con$$constant);
8250     __ sbc($dst$$Register->successor(), $src1$$Register->successor(), 0);
8251   %}
8252   ins_pipe(ialu_reg_imm);
8253 %}
8254 
8255 // Long negation
8256 instruct negL_reg_reg(iRegL dst, immL0 zero, iRegL src2, flagsReg icc) %{
8257   match(Set dst (SubL zero src2));
8258   effect (KILL icc);
8259 
8260   size(8);
8261   format %{ "RSBS   $dst.lo,$src2.lo,0\t! long\n\t"
8262             "RSC    $dst.hi,$src2.hi,0" %}
8263   ins_encode %{
8264     __ rsbs($dst$$Register, $src2$$Register, 0);
8265     __ rsc($dst$$Register->successor(), $src2$$Register->successor(), 0);
8266   %}
8267   ins_pipe(ialu_zero_reg);
8268 %}
8269 #endif // !AARCH64
8270 
8271 // Multiplication Instructions
8272 // Integer Multiplication
8273 // Register Multiplication
8274 instruct mulI_reg_reg(iRegI dst, iRegI src1, iRegI src2) %{
8275   match(Set dst (MulI src1 src2));
8276 
8277   size(4);
8278   format %{ "mul_32 $dst,$src1,$src2" %}
8279   ins_encode %{
8280     __ mul_32($dst$$Register, $src1$$Register, $src2$$Register);
8281   %}
8282   ins_pipe(imul_reg_reg);
8283 %}
8284 
8285 #ifdef AARCH64
8286 instruct mulL_reg_reg(iRegL dst, iRegL src1, iRegL src2) %{
8287   match(Set dst (MulL src1 src2));
8288   size(4);
8289   format %{ "MUL  $dst,$src1,$src2\t! long" %}
8290   ins_encode %{
8291     __ mul($dst$$Register, $src1$$Register, $src2$$Register);
8292   %}
8293   ins_pipe(imul_reg_reg);
8294 %}
8295 #else
8296 instruct mulL_lo1_hi2(iRegL dst, iRegL src1, iRegL src2) %{
8297   effect(DEF dst, USE src1, USE src2);
8298   size(4);
8299   format %{ "MUL  $dst.hi,$src1.lo,$src2.hi\t! long" %}
8300   ins_encode %{
8301     __ mul($dst$$Register->successor(), $src1$$Register, $src2$$Register->successor());
8302   %}
8303   ins_pipe(imul_reg_reg);
8304 %}
8305 
8306 instruct mulL_hi1_lo2(iRegL dst, iRegL src1, iRegL src2) %{
8307   effect(USE_DEF dst, USE src1, USE src2);
8308   size(8);
8309   format %{ "MLA  $dst.hi,$src1.hi,$src2.lo,$dst.hi\t! long\n\t"
8310             "MOV  $dst.lo, 0"%}
8311   ins_encode %{
8312     __ mla($dst$$Register->successor(), $src1$$Register->successor(), $src2$$Register, $dst$$Register->successor());
8313     __ mov($dst$$Register, 0);
8314   %}
8315   ins_pipe(imul_reg_reg);
8316 %}
8317 
8318 instruct mulL_lo1_lo2(iRegL dst, iRegL src1, iRegL src2) %{
8319   effect(USE_DEF dst, USE src1, USE src2);
8320   size(4);
8321   format %{ "UMLAL  $dst.lo,$dst.hi,$src1,$src2\t! long" %}
8322   ins_encode %{
8323     __ umlal($dst$$Register, $dst$$Register->successor(), $src1$$Register, $src2$$Register);
8324   %}
8325   ins_pipe(imul_reg_reg);
8326 %}
8327 
8328 instruct mulL_reg_reg(iRegL dst, iRegL src1, iRegL src2) %{
8329   match(Set dst (MulL src1 src2));
8330 
8331   expand %{
8332     mulL_lo1_hi2(dst, src1, src2);
8333     mulL_hi1_lo2(dst, src1, src2);
8334     mulL_lo1_lo2(dst, src1, src2);
8335   %}
8336 %}
8337 #endif // !AARCH64
8338 
8339 // Integer Division
8340 // Register Division
8341 #ifdef AARCH64
8342 instruct divI_reg_reg(iRegI dst, iRegI src1, iRegI src2) %{
8343   match(Set dst (DivI src1 src2));
8344 
8345   size(4);
8346   format %{ "SDIV    $dst,$src1,$src2\t! 32-bit" %}
8347   ins_encode %{
8348     __ sdiv_w($dst$$Register, $src1$$Register, $src2$$Register);
8349   %}
8350   ins_pipe(ialu_reg_reg); // FIXME
8351 %}
8352 #else
8353 instruct divI_reg_reg(R1RegI dst, R0RegI src1, R2RegI src2, LRRegP lr, flagsReg ccr) %{
8354   match(Set dst (DivI src1 src2));
8355   effect( KILL ccr, KILL src1, KILL src2, KILL lr);
8356   ins_cost((2+71)*DEFAULT_COST);
8357 
8358   format %{ "DIV   $dst,$src1,$src2 ! call to StubRoutines::Arm::idiv_irem_entry()" %}
8359   ins_encode %{
8360     __ call(StubRoutines::Arm::idiv_irem_entry(), relocInfo::runtime_call_type);
8361   %}
8362   ins_pipe(sdiv_reg_reg);
8363 %}
8364 #endif
8365 
8366 // Register Long Division
8367 #ifdef AARCH64
8368 instruct divL_reg_reg(iRegL dst, iRegL src1, iRegL src2) %{
8369   match(Set dst (DivL src1 src2));
8370 
8371   size(4);
8372   format %{ "SDIV    $dst,$src1,$src2" %}
8373   ins_encode %{
8374     __ sdiv($dst$$Register, $src1$$Register, $src2$$Register);
8375   %}
8376   ins_pipe(ialu_reg_reg); // FIXME
8377 %}
8378 #else
8379 instruct divL_reg_reg(R0R1RegL dst, R2R3RegL src1, R0R1RegL src2) %{
8380   match(Set dst (DivL src1 src2));
8381   effect(CALL);
8382   ins_cost(DEFAULT_COST*71);
8383   format %{ "DIVL  $src1,$src2,$dst\t! long ! call to SharedRuntime::ldiv" %}
8384   ins_encode %{
8385     address target = CAST_FROM_FN_PTR(address, SharedRuntime::ldiv);
8386     __ call(target, relocInfo::runtime_call_type);
8387   %}
8388   ins_pipe(divL_reg_reg);
8389 %}
8390 #endif
8391 
8392 // Integer Remainder
8393 // Register Remainder
8394 #ifdef AARCH64
8395 #ifdef TODO
8396 instruct msubI_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3) %{
8397   match(Set dst (SubI src1 (MulI src2 src3)));
8398 
8399   size(4);
8400   format %{ "MSUB    $dst,$src2,$src3,$src1\t! 32-bit\n\t" %}
8401   ins_encode %{
8402     __ msub_w($dst$$Register, $src2$$Register, $src3$$Register, $src1$$Register);
8403   %}
8404   ins_pipe(ialu_reg_reg); // FIXME
8405 %}
8406 #endif
8407 
8408 instruct modI_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI temp) %{
8409   match(Set dst (ModI src1 src2));
8410   effect(TEMP temp);
8411 
8412   size(8);
8413   format %{ "SDIV    $temp,$src1,$src2\t! 32-bit\n\t"
8414             "MSUB    $dst,$src2,$temp,$src1\t! 32-bit\n\t" %}
8415   ins_encode %{
8416     __ sdiv_w($temp$$Register, $src1$$Register, $src2$$Register);
8417     __ msub_w($dst$$Register, $src2$$Register, $temp$$Register, $src1$$Register);
8418   %}
8419   ins_pipe(ialu_reg_reg); // FIXME
8420 %}
8421 #else
8422 instruct modI_reg_reg(R0RegI dst, R0RegI src1, R2RegI src2, R1RegI temp, LRRegP lr, flagsReg ccr ) %{
8423   match(Set dst (ModI src1 src2));
8424   effect( KILL ccr, KILL temp, KILL src2, KILL lr);
8425 
8426   format %{ "MODI   $dst,$src1,$src2\t ! call to StubRoutines::Arm::idiv_irem_entry" %}
8427   ins_encode %{
8428     __ call(StubRoutines::Arm::idiv_irem_entry(), relocInfo::runtime_call_type);
8429   %}
8430   ins_pipe(sdiv_reg_reg);
8431 %}
8432 #endif
8433 
8434 // Register Long Remainder
8435 #ifdef AARCH64
8436 instruct modL_reg_reg(iRegL dst, iRegL src1, iRegL src2, iRegL temp) %{
8437   match(Set dst (ModL src1 src2));
8438   effect(TEMP temp);
8439 
8440   size(8);
8441   format %{ "SDIV    $temp,$src1,$src2\n\t"
8442             "MSUB    $dst,$src2,$temp,$src1" %}
8443   ins_encode %{
8444     __ sdiv($temp$$Register, $src1$$Register, $src2$$Register);
8445     __ msub($dst$$Register, $src2$$Register, $temp$$Register, $src1$$Register);
8446   %}
8447   ins_pipe(ialu_reg_reg); // FIXME
8448 %}
8449 #else
8450 instruct modL_reg_reg(R0R1RegL dst, R2R3RegL src1, R0R1RegL src2) %{
8451   match(Set dst (ModL src1 src2));
8452   effect(CALL);
8453   ins_cost(MEMORY_REF_COST); // FIXME
8454   format %{ "modL    $dst,$src1,$src2\t ! call to SharedRuntime::lrem" %}
8455   ins_encode %{
8456     address target = CAST_FROM_FN_PTR(address, SharedRuntime::lrem);
8457     __ call(target, relocInfo::runtime_call_type);
8458   %}
8459   ins_pipe(divL_reg_reg);
8460 %}
8461 #endif
8462 
8463 // Integer Shift Instructions
8464 
8465 // Register Shift Left
8466 instruct shlI_reg_reg(iRegI dst, iRegI src1, iRegI src2) %{
8467   match(Set dst (LShiftI src1 src2));
8468 
8469   size(4);
8470 #ifdef AARCH64
8471   format %{ "LSLV   $dst,$src1,$src2\t! int" %}
8472   ins_encode %{
8473     __ lslv_w($dst$$Register, $src1$$Register, $src2$$Register);
8474   %}
8475 #else
8476   format %{ "LSL  $dst,$src1,$src2 \n\t" %}
8477   ins_encode %{
8478     __ mov($dst$$Register, AsmOperand($src1$$Register, lsl, $src2$$Register));
8479   %}
8480 #endif
8481   ins_pipe(ialu_reg_reg);
8482 %}
8483 
8484 // Register Shift Left Immediate
8485 instruct shlI_reg_imm5(iRegI dst, iRegI src1, immU5 src2) %{
8486   match(Set dst (LShiftI src1 src2));
8487 
8488   size(4);
8489 #ifdef AARCH64
8490   format %{ "LSL_w  $dst,$src1,$src2\t! int" %}
8491   ins_encode %{
8492     __ _lsl($dst$$Register, $src1$$Register, $src2$$constant);
8493   %}
8494 #else
8495   format %{ "LSL    $dst,$src1,$src2\t! int" %}
8496   ins_encode %{
8497     __ logical_shift_left($dst$$Register, $src1$$Register, $src2$$constant);
8498   %}
8499 #endif
8500   ins_pipe(ialu_reg_imm);
8501 %}
8502 
8503 #ifndef AARCH64
8504 instruct shlL_reg_reg_merge_hi(iRegL dst, iRegL src1, iRegI src2) %{
8505   effect(USE_DEF dst, USE src1, USE src2);
8506   size(4);
8507   format %{"OR  $dst.hi,$dst.hi,($src1.hi << $src2)"  %}
8508   ins_encode %{
8509     __ orr($dst$$Register->successor(), $dst$$Register->successor(), AsmOperand($src1$$Register->successor(), lsl, $src2$$Register));
8510   %}
8511   ins_pipe(ialu_reg_reg);
8512 %}
8513 
8514 instruct shlL_reg_reg_merge_lo(iRegL dst, iRegL src1, iRegI src2) %{
8515   effect(USE_DEF dst, USE src1, USE src2);
8516   size(4);
8517   format %{ "LSL  $dst.lo,$src1.lo,$src2 \n\t" %}
8518   ins_encode %{
8519     __ mov($dst$$Register, AsmOperand($src1$$Register, lsl, $src2$$Register));
8520   %}
8521   ins_pipe(ialu_reg_reg);
8522 %}
8523 
8524 instruct shlL_reg_reg_overlap(iRegL dst, iRegL src1, iRegI src2, flagsReg ccr) %{
8525   effect(DEF dst, USE src1, USE src2, KILL ccr);
8526   size(16);
8527   format %{ "SUBS  $dst.hi,$src2,32 \n\t"
8528             "LSLpl $dst.hi,$src1.lo,$dst.hi \n\t"
8529             "RSBmi $dst.hi,$dst.hi,0 \n\t"
8530             "LSRmi $dst.hi,$src1.lo,$dst.hi" %}
8531 
8532   ins_encode %{
8533     // $src1$$Register and $dst$$Register->successor() can't be the same
8534     __ subs($dst$$Register->successor(), $src2$$Register, 32);
8535     __ mov($dst$$Register->successor(), AsmOperand($src1$$Register, lsl, $dst$$Register->successor()), pl);
8536     __ rsb($dst$$Register->successor(), $dst$$Register->successor(), 0, mi);
8537     __ mov($dst$$Register->successor(), AsmOperand($src1$$Register, lsr, $dst$$Register->successor()), mi);
8538   %}
8539   ins_pipe(ialu_reg_reg);
8540 %}
8541 #endif // !AARCH64
8542 
8543 instruct shlL_reg_reg(iRegL dst, iRegL src1, iRegI src2) %{
8544   match(Set dst (LShiftL src1 src2));
8545 
8546 #ifdef AARCH64
8547   size(4);
8548   format %{ "LSLV  $dst,$src1,$src2\t! long" %}
8549   ins_encode %{
8550     __ lslv($dst$$Register, $src1$$Register, $src2$$Register);
8551   %}
8552   ins_pipe(ialu_reg_reg);
8553 #else
8554   expand %{
8555     flagsReg ccr;
8556     shlL_reg_reg_overlap(dst, src1, src2, ccr);
8557     shlL_reg_reg_merge_hi(dst, src1, src2);
8558     shlL_reg_reg_merge_lo(dst, src1, src2);
8559   %}
8560 #endif
8561 %}
8562 
8563 #ifdef AARCH64
8564 instruct shlL_reg_imm6(iRegL dst, iRegL src1, immU6 src2) %{
8565   match(Set dst (LShiftL src1 src2));
8566 
8567   size(4);
8568   format %{ "LSL    $dst,$src1,$src2\t! long" %}
8569   ins_encode %{
8570     __ logical_shift_left($dst$$Register, $src1$$Register, $src2$$constant);
8571   %}
8572   ins_pipe(ialu_reg_imm);
8573 %}
8574 #else
8575 // Register Shift Left Immediate
8576 instruct shlL_reg_imm6(iRegL dst, iRegL src1, immU6Big src2) %{
8577   match(Set dst (LShiftL src1 src2));
8578 
8579   size(8);
8580   format %{ "LSL   $dst.hi,$src1.lo,$src2-32\t! or mov if $src2==32\n\t"
8581             "MOV   $dst.lo, 0" %}
8582   ins_encode %{
8583     if ($src2$$constant == 32) {
8584       __ mov($dst$$Register->successor(), $src1$$Register);
8585     } else {
8586       __ mov($dst$$Register->successor(), AsmOperand($src1$$Register, lsl, $src2$$constant-32));
8587     }
8588     __ mov($dst$$Register, 0);
8589   %}
8590   ins_pipe(ialu_reg_imm);
8591 %}
8592 
8593 instruct shlL_reg_imm5(iRegL dst, iRegL src1, immU5 src2) %{
8594   match(Set dst (LShiftL src1 src2));
8595 
8596   size(12);
8597   format %{ "LSL   $dst.hi,$src1.lo,$src2\n\t"
8598             "OR    $dst.hi, $dst.hi, $src1.lo >> 32-$src2\n\t"
8599             "LSL   $dst.lo,$src1.lo,$src2" %}
8600   ins_encode %{
8601     // The order of the following 3 instructions matters: src1.lo and
8602     // dst.hi can't overlap but src.hi and dst.hi can.
8603     __ mov($dst$$Register->successor(), AsmOperand($src1$$Register->successor(), lsl, $src2$$constant));
8604     __ orr($dst$$Register->successor(), $dst$$Register->successor(), AsmOperand($src1$$Register, lsr, 32-$src2$$constant));
8605     __ mov($dst$$Register, AsmOperand($src1$$Register, lsl, $src2$$constant));
8606   %}
8607   ins_pipe(ialu_reg_imm);
8608 %}
8609 #endif // !AARCH64
8610 
8611 // Register Arithmetic Shift Right
8612 instruct sarI_reg_reg(iRegI dst, iRegI src1, iRegI src2) %{
8613   match(Set dst (RShiftI src1 src2));
8614   size(4);
8615 #ifdef AARCH64
8616   format %{ "ASRV   $dst,$src1,$src2\t! int" %}
8617   ins_encode %{
8618     __ asrv_w($dst$$Register, $src1$$Register, $src2$$Register);
8619   %}
8620 #else
8621   format %{ "ASR    $dst,$src1,$src2\t! int" %}
8622   ins_encode %{
8623     __ mov($dst$$Register, AsmOperand($src1$$Register, asr, $src2$$Register));
8624   %}
8625 #endif
8626   ins_pipe(ialu_reg_reg);
8627 %}
8628 
8629 // Register Arithmetic Shift Right Immediate
8630 instruct sarI_reg_imm5(iRegI dst, iRegI src1, immU5 src2) %{
8631   match(Set dst (RShiftI src1 src2));
8632 
8633   size(4);
8634 #ifdef AARCH64
8635   format %{ "ASR_w  $dst,$src1,$src2" %}
8636   ins_encode %{
8637     __ _asr_w($dst$$Register, $src1$$Register, $src2$$constant);
8638   %}
8639 #else
8640   format %{ "ASR    $dst,$src1,$src2" %}
8641   ins_encode %{
8642     __ mov($dst$$Register, AsmOperand($src1$$Register, asr, $src2$$constant));
8643   %}
8644 #endif
8645   ins_pipe(ialu_reg_imm);
8646 %}
8647 
8648 #ifndef AARCH64
8649 // Register Shift Right Arithmetic Long
8650 instruct sarL_reg_reg_merge_lo(iRegL dst, iRegL src1, iRegI src2) %{
8651   effect(USE_DEF dst, USE src1, USE src2);
8652   size(4);
8653   format %{ "OR  $dst.lo,$dst.lo,($src1.lo >> $src2)"  %}
8654   ins_encode %{
8655     __ orr($dst$$Register, $dst$$Register, AsmOperand($src1$$Register, lsr, $src2$$Register));
8656   %}
8657   ins_pipe(ialu_reg_reg);
8658 %}
8659 
8660 instruct sarL_reg_reg_merge_hi(iRegL dst, iRegL src1, iRegI src2) %{
8661   effect(USE_DEF dst, USE src1, USE src2);
8662   size(4);
8663   format %{ "ASR  $dst.hi,$src1.hi,$src2 \n\t" %}
8664   ins_encode %{
8665     __ mov($dst$$Register->successor(), AsmOperand($src1$$Register->successor(), asr, $src2$$Register));
8666   %}
8667   ins_pipe(ialu_reg_reg);
8668 %}
8669 
8670 instruct sarL_reg_reg_overlap(iRegL dst, iRegL src1, iRegI src2, flagsReg ccr) %{
8671   effect(DEF dst, USE src1, USE src2, KILL ccr);
8672   size(16);
8673   format %{ "SUBS  $dst.lo,$src2,32 \n\t"
8674             "ASRpl $dst.lo,$src1.hi,$dst.lo \n\t"
8675             "RSBmi $dst.lo,$dst.lo,0 \n\t"
8676             "LSLmi $dst.lo,$src1.hi,$dst.lo" %}
8677 
8678   ins_encode %{
8679     // $src1$$Register->successor() and $dst$$Register can't be the same
8680     __ subs($dst$$Register, $src2$$Register, 32);
8681     __ mov($dst$$Register, AsmOperand($src1$$Register->successor(), asr, $dst$$Register), pl);
8682     __ rsb($dst$$Register, $dst$$Register, 0, mi);
8683     __ mov($dst$$Register, AsmOperand($src1$$Register->successor(), lsl, $dst$$Register), mi);
8684   %}
8685   ins_pipe(ialu_reg_reg);
8686 %}
8687 #endif // !AARCH64
8688 
8689 instruct sarL_reg_reg(iRegL dst, iRegL src1, iRegI src2) %{
8690   match(Set dst (RShiftL src1 src2));
8691 
8692 #ifdef AARCH64
8693   size(4);
8694   format %{ "ASRV  $dst,$src1,$src2\t! long" %}
8695   ins_encode %{
8696     __ asrv($dst$$Register, $src1$$Register, $src2$$Register);
8697   %}
8698   ins_pipe(ialu_reg_reg);
8699 #else
8700   expand %{
8701     flagsReg ccr;
8702     sarL_reg_reg_overlap(dst, src1, src2, ccr);
8703     sarL_reg_reg_merge_lo(dst, src1, src2);
8704     sarL_reg_reg_merge_hi(dst, src1, src2);
8705   %}
8706 #endif
8707 %}
8708 
8709 // Register Shift Left Immediate
8710 #ifdef AARCH64
8711 instruct sarL_reg_imm6(iRegL dst, iRegL src1, immU6 src2) %{
8712   match(Set dst (RShiftL src1 src2));
8713 
8714   size(4);
8715   format %{ "ASR    $dst,$src1,$src2\t! long" %}
8716   ins_encode %{
8717     __ _asr($dst$$Register, $src1$$Register, $src2$$constant);
8718   %}
8719   ins_pipe(ialu_reg_imm);
8720 %}
8721 #else
8722 instruct sarL_reg_imm6(iRegL dst, iRegL src1, immU6Big src2) %{
8723   match(Set dst (RShiftL src1 src2));
8724 
8725   size(8);
8726   format %{ "ASR   $dst.lo,$src1.hi,$src2-32\t! or mov if $src2==32\n\t"
8727             "ASR   $dst.hi,$src1.hi, $src2" %}
8728   ins_encode %{
8729     if ($src2$$constant == 32) {
8730       __ mov($dst$$Register, $src1$$Register->successor());
8731     } else{
8732       __ mov($dst$$Register, AsmOperand($src1$$Register->successor(), asr, $src2$$constant-32));
8733     }
8734     __ mov($dst$$Register->successor(), AsmOperand($src1$$Register->successor(), asr, 0));
8735   %}
8736 
8737   ins_pipe(ialu_reg_imm);
8738 %}
8739 
8740 instruct sarL_reg_imm5(iRegL dst, iRegL src1, immU5 src2) %{
8741   match(Set dst (RShiftL src1 src2));
8742   size(12);
8743   format %{ "LSR   $dst.lo,$src1.lo,$src2\n\t"
8744             "OR    $dst.lo, $dst.lo, $src1.hi << 32-$src2\n\t"
8745             "ASR   $dst.hi,$src1.hi,$src2" %}
8746   ins_encode %{
8747     // The order of the following 3 instructions matters: src1.lo and
8748     // dst.hi can't overlap but src.hi and dst.hi can.
8749     __ mov($dst$$Register, AsmOperand($src1$$Register, lsr, $src2$$constant));
8750     __ orr($dst$$Register, $dst$$Register, AsmOperand($src1$$Register->successor(), lsl, 32-$src2$$constant));
8751     __ mov($dst$$Register->successor(), AsmOperand($src1$$Register->successor(), asr, $src2$$constant));
8752   %}
8753   ins_pipe(ialu_reg_imm);
8754 %}
8755 #endif
8756 
8757 // Register Shift Right
8758 instruct shrI_reg_reg(iRegI dst, iRegI src1, iRegI src2) %{
8759   match(Set dst (URShiftI src1 src2));
8760   size(4);
8761 #ifdef AARCH64
8762   format %{ "LSRV   $dst,$src1,$src2\t! int" %}
8763   ins_encode %{
8764     __ lsrv_w($dst$$Register, $src1$$Register, $src2$$Register);
8765   %}
8766 #else
8767   format %{ "LSR    $dst,$src1,$src2\t! int" %}
8768   ins_encode %{
8769     __ mov($dst$$Register, AsmOperand($src1$$Register, lsr, $src2$$Register));
8770   %}
8771 #endif
8772   ins_pipe(ialu_reg_reg);
8773 %}
8774 
8775 // Register Shift Right Immediate
8776 instruct shrI_reg_imm5(iRegI dst, iRegI src1, immU5 src2) %{
8777   match(Set dst (URShiftI src1 src2));
8778 
8779   size(4);
8780 #ifdef AARCH64
8781   format %{ "LSR_w  $dst,$src1,$src2" %}
8782   ins_encode %{
8783     __ _lsr_w($dst$$Register, $src1$$Register, $src2$$constant);
8784   %}
8785 #else
8786   format %{ "LSR    $dst,$src1,$src2" %}
8787   ins_encode %{
8788     __ mov($dst$$Register, AsmOperand($src1$$Register, lsr, $src2$$constant));
8789   %}
8790 #endif
8791   ins_pipe(ialu_reg_imm);
8792 %}
8793 
8794 #ifndef AARCH64
8795 // Register Shift Right
8796 instruct shrL_reg_reg_merge_lo(iRegL dst, iRegL src1, iRegI src2) %{
8797   effect(USE_DEF dst, USE src1, USE src2);
8798   size(4);
8799   format %{ "OR   $dst.lo,$dst,($src1.lo >>> $src2)"  %}
8800   ins_encode %{
8801     __ orr($dst$$Register, $dst$$Register, AsmOperand($src1$$Register, lsr, $src2$$Register));
8802   %}
8803   ins_pipe(ialu_reg_reg);
8804 %}
8805 
8806 instruct shrL_reg_reg_merge_hi(iRegL dst, iRegL src1, iRegI src2) %{
8807   effect(USE_DEF dst, USE src1, USE src2);
8808   size(4);
8809   format %{ "LSR  $dst.hi,$src1.hi,$src2 \n\t" %}
8810   ins_encode %{
8811     __ mov($dst$$Register->successor(), AsmOperand($src1$$Register->successor(), lsr, $src2$$Register));
8812   %}
8813   ins_pipe(ialu_reg_reg);
8814 %}
8815 
8816 instruct shrL_reg_reg_overlap(iRegL dst, iRegL src1, iRegI src2, flagsReg ccr) %{
8817   effect(DEF dst, USE src1, USE src2, KILL ccr);
8818   size(16);
8819   format %{ "SUBS  $dst,$src2,32 \n\t"
8820             "LSRpl $dst,$src1.hi,$dst \n\t"
8821             "RSBmi $dst,$dst,0 \n\t"
8822             "LSLmi $dst,$src1.hi,$dst" %}
8823 
8824   ins_encode %{
8825     // $src1$$Register->successor() and $dst$$Register can't be the same
8826     __ subs($dst$$Register, $src2$$Register, 32);
8827     __ mov($dst$$Register, AsmOperand($src1$$Register->successor(), lsr, $dst$$Register), pl);
8828     __ rsb($dst$$Register, $dst$$Register, 0, mi);
8829     __ mov($dst$$Register, AsmOperand($src1$$Register->successor(), lsl, $dst$$Register), mi);
8830   %}
8831   ins_pipe(ialu_reg_reg);
8832 %}
8833 #endif // !AARCH64
8834 
8835 instruct shrL_reg_reg(iRegL dst, iRegL src1, iRegI src2) %{
8836   match(Set dst (URShiftL src1 src2));
8837 
8838 #ifdef AARCH64
8839   size(4);
8840   format %{ "LSRV  $dst,$src1,$src2\t! long" %}
8841   ins_encode %{
8842     __ lsrv($dst$$Register, $src1$$Register, $src2$$Register);
8843   %}
8844   ins_pipe(ialu_reg_reg);
8845 #else
8846   expand %{
8847     flagsReg ccr;
8848     shrL_reg_reg_overlap(dst, src1, src2, ccr);
8849     shrL_reg_reg_merge_lo(dst, src1, src2);
8850     shrL_reg_reg_merge_hi(dst, src1, src2);
8851   %}
8852 #endif
8853 %}
8854 
8855 // Register Shift Right Immediate
8856 #ifdef AARCH64
8857 instruct shrL_reg_imm6(iRegL dst, iRegL src1, immU6 src2) %{
8858   match(Set dst (URShiftL src1 src2));
8859 
8860   size(4);
8861   format %{ "LSR    $dst,$src1,$src2" %}
8862   ins_encode %{
8863     __ _lsr($dst$$Register, $src1$$Register, $src2$$constant);
8864   %}
8865   ins_pipe(ialu_reg_imm);
8866 %}
8867 #else
8868 instruct shrL_reg_imm6(iRegL dst, iRegL src1, immU6Big src2) %{
8869   match(Set dst (URShiftL src1 src2));
8870 
8871   size(8);
8872   format %{ "LSR   $dst.lo,$src1.hi,$src2-32\t! or mov if $src2==32\n\t"
8873             "MOV   $dst.hi, 0" %}
8874   ins_encode %{
8875     if ($src2$$constant == 32) {
8876       __ mov($dst$$Register, $src1$$Register->successor());
8877     } else {
8878       __ mov($dst$$Register, AsmOperand($src1$$Register->successor(), lsr, $src2$$constant-32));
8879     }
8880     __ mov($dst$$Register->successor(), 0);
8881   %}
8882 
8883   ins_pipe(ialu_reg_imm);
8884 %}
8885 
8886 instruct shrL_reg_imm5(iRegL dst, iRegL src1, immU5 src2) %{
8887   match(Set dst (URShiftL src1 src2));
8888 
8889   size(12);
8890   format %{ "LSR   $dst.lo,$src1.lo,$src2\n\t"
8891             "OR    $dst.lo, $dst.lo, $src1.hi << 32-$src2\n\t"
8892             "LSR   $dst.hi,$src1.hi,$src2" %}
8893   ins_encode %{
8894     // The order of the following 3 instructions matters: src1.lo and
8895     // dst.hi can't overlap but src.hi and dst.hi can.
8896     __ mov($dst$$Register, AsmOperand($src1$$Register, lsr, $src2$$constant));
8897     __ orr($dst$$Register, $dst$$Register, AsmOperand($src1$$Register->successor(), lsl, 32-$src2$$constant));
8898     __ mov($dst$$Register->successor(), AsmOperand($src1$$Register->successor(), lsr, $src2$$constant));
8899   %}
8900   ins_pipe(ialu_reg_imm);
8901 %}
8902 #endif // !AARCH64
8903 
8904 
8905 instruct shrP_reg_imm5(iRegX dst, iRegP src1, immU5 src2) %{
8906   match(Set dst (URShiftI (CastP2X src1) src2));
8907   size(4);
8908   format %{ "LSR    $dst,$src1,$src2\t! Cast ptr $src1 to int and shift" %}
8909   ins_encode %{
8910     __ logical_shift_right($dst$$Register, $src1$$Register, $src2$$constant);
8911   %}
8912   ins_pipe(ialu_reg_imm);
8913 %}
8914 
8915 //----------Floating Point Arithmetic Instructions-----------------------------
8916 
8917 //  Add float single precision
8918 instruct addF_reg_reg(regF dst, regF src1, regF src2) %{
8919   match(Set dst (AddF src1 src2));
8920 
8921   size(4);
8922   format %{ "FADDS  $dst,$src1,$src2" %}
8923   ins_encode %{
8924     __ add_float($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister);
8925   %}
8926 
8927   ins_pipe(faddF_reg_reg);
8928 %}
8929 
8930 //  Add float double precision
8931 instruct addD_reg_reg(regD dst, regD src1, regD src2) %{
8932   match(Set dst (AddD src1 src2));
8933 
8934   size(4);
8935   format %{ "FADDD  $dst,$src1,$src2" %}
8936   ins_encode %{
8937     __ add_double($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister);
8938   %}
8939 
8940   ins_pipe(faddD_reg_reg);
8941 %}
8942 
8943 //  Sub float single precision
8944 instruct subF_reg_reg(regF dst, regF src1, regF src2) %{
8945   match(Set dst (SubF src1 src2));
8946 
8947   size(4);
8948   format %{ "FSUBS  $dst,$src1,$src2" %}
8949   ins_encode %{
8950     __ sub_float($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister);
8951   %}
8952   ins_pipe(faddF_reg_reg);
8953 %}
8954 
8955 //  Sub float double precision
8956 instruct subD_reg_reg(regD dst, regD src1, regD src2) %{
8957   match(Set dst (SubD src1 src2));
8958 
8959   size(4);
8960   format %{ "FSUBD  $dst,$src1,$src2" %}
8961   ins_encode %{
8962     __ sub_double($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister);
8963   %}
8964   ins_pipe(faddD_reg_reg);
8965 %}
8966 
8967 //  Mul float single precision
8968 instruct mulF_reg_reg(regF dst, regF src1, regF src2) %{
8969   match(Set dst (MulF src1 src2));
8970 
8971   size(4);
8972   format %{ "FMULS  $dst,$src1,$src2" %}
8973   ins_encode %{
8974     __ mul_float($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister);
8975   %}
8976 
8977   ins_pipe(fmulF_reg_reg);
8978 %}
8979 
8980 //  Mul float double precision
8981 instruct mulD_reg_reg(regD dst, regD src1, regD src2) %{
8982   match(Set dst (MulD src1 src2));
8983 
8984   size(4);
8985   format %{ "FMULD  $dst,$src1,$src2" %}
8986   ins_encode %{
8987     __ mul_double($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister);
8988   %}
8989 
8990   ins_pipe(fmulD_reg_reg);
8991 %}
8992 
8993 //  Div float single precision
8994 instruct divF_reg_reg(regF dst, regF src1, regF src2) %{
8995   match(Set dst (DivF src1 src2));
8996 
8997   size(4);
8998   format %{ "FDIVS  $dst,$src1,$src2" %}
8999   ins_encode %{
9000     __ div_float($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister);
9001   %}
9002 
9003   ins_pipe(fdivF_reg_reg);
9004 %}
9005 
9006 //  Div float double precision
9007 instruct divD_reg_reg(regD dst, regD src1, regD src2) %{
9008   match(Set dst (DivD src1 src2));
9009 
9010   size(4);
9011   format %{ "FDIVD  $dst,$src1,$src2" %}
9012   ins_encode %{
9013     __ div_double($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister);
9014   %}
9015 
9016   ins_pipe(fdivD_reg_reg);
9017 %}
9018 
9019 //  Absolute float double precision
9020 instruct absD_reg(regD dst, regD src) %{
9021   match(Set dst (AbsD src));
9022 
9023   size(4);
9024   format %{ "FABSd  $dst,$src" %}
9025   ins_encode %{
9026     __ abs_double($dst$$FloatRegister, $src$$FloatRegister);
9027   %}
9028   ins_pipe(faddD_reg);
9029 %}
9030 
9031 //  Absolute float single precision
9032 instruct absF_reg(regF dst, regF src) %{
9033   match(Set dst (AbsF src));
9034   format %{ "FABSs  $dst,$src" %}
9035   ins_encode %{
9036     __ abs_float($dst$$FloatRegister, $src$$FloatRegister);
9037   %}
9038   ins_pipe(faddF_reg);
9039 %}
9040 
9041 instruct negF_reg(regF dst, regF src) %{
9042   match(Set dst (NegF src));
9043 
9044   size(4);
9045   format %{ "FNEGs  $dst,$src" %}
9046   ins_encode %{
9047     __ neg_float($dst$$FloatRegister, $src$$FloatRegister);
9048   %}
9049   ins_pipe(faddF_reg);
9050 %}
9051 
9052 instruct negD_reg(regD dst, regD src) %{
9053   match(Set dst (NegD src));
9054 
9055   format %{ "FNEGd  $dst,$src" %}
9056   ins_encode %{
9057     __ neg_double($dst$$FloatRegister, $src$$FloatRegister);
9058   %}
9059   ins_pipe(faddD_reg);
9060 %}
9061 
9062 //  Sqrt float double precision
9063 instruct sqrtF_reg_reg(regF dst, regF src) %{
9064   match(Set dst (ConvD2F (SqrtD (ConvF2D src))));
9065 
9066   size(4);
9067   format %{ "FSQRTS $dst,$src" %}
9068   ins_encode %{
9069     __ sqrt_float($dst$$FloatRegister, $src$$FloatRegister);
9070   %}
9071   ins_pipe(fdivF_reg_reg);
9072 %}
9073 
9074 //  Sqrt float double precision
9075 instruct sqrtD_reg_reg(regD dst, regD src) %{
9076   match(Set dst (SqrtD src));
9077 
9078   size(4);
9079   format %{ "FSQRTD $dst,$src" %}
9080   ins_encode %{
9081     __ sqrt_double($dst$$FloatRegister, $src$$FloatRegister);
9082   %}
9083   ins_pipe(fdivD_reg_reg);
9084 %}
9085 
9086 //----------Logical Instructions-----------------------------------------------
9087 // And Instructions
9088 // Register And
9089 instruct andI_reg_reg(iRegI dst, iRegI src1, iRegI src2) %{
9090   match(Set dst (AndI src1 src2));
9091 
9092   size(4);
9093   format %{ "and_32 $dst,$src1,$src2" %}
9094   ins_encode %{
9095     __ and_32($dst$$Register, $src1$$Register, $src2$$Register);
9096   %}
9097   ins_pipe(ialu_reg_reg);
9098 %}
9099 
9100 #ifndef AARCH64
9101 instruct andshlI_reg_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3) %{
9102   match(Set dst (AndI src1 (LShiftI src2 src3)));
9103 
9104   size(4);
9105   format %{ "AND    $dst,$src1,$src2<<$src3" %}
9106   ins_encode %{
9107     __ andr($dst$$Register, $src1$$Register, AsmOperand($src2$$Register, lsl, $src3$$Register));
9108   %}
9109   ins_pipe(ialu_reg_reg);
9110 %}
9111 #endif
9112 
9113 instruct andshlI_reg_reg_imm(iRegI dst, iRegI src1, iRegI src2, immU5 src3) %{
9114   match(Set dst (AndI src1 (LShiftI src2 src3)));
9115 
9116   size(4);
9117   format %{ "and_32 $dst,$src1,$src2<<$src3" %}
9118   ins_encode %{
9119     __ and_32($dst$$Register, $src1$$Register, AsmOperand($src2$$Register, lsl, $src3$$constant));
9120   %}
9121   ins_pipe(ialu_reg_reg);
9122 %}
9123 
9124 #ifndef AARCH64
9125 instruct andsarI_reg_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3) %{
9126   match(Set dst (AndI src1 (RShiftI src2 src3)));
9127 
9128   size(4);
9129   format %{ "AND    $dst,$src1,$src2>>$src3" %}
9130   ins_encode %{
9131     __ andr($dst$$Register, $src1$$Register, AsmOperand($src2$$Register, asr, $src3$$Register));
9132   %}
9133   ins_pipe(ialu_reg_reg);
9134 %}
9135 #endif
9136 
9137 instruct andsarI_reg_reg_imm(iRegI dst, iRegI src1, iRegI src2, immU5 src3) %{
9138   match(Set dst (AndI src1 (RShiftI src2 src3)));
9139 
9140   size(4);
9141   format %{ "and_32 $dst,$src1,$src2>>$src3" %}
9142   ins_encode %{
9143     __ and_32($dst$$Register, $src1$$Register, AsmOperand($src2$$Register, asr, $src3$$constant));
9144   %}
9145   ins_pipe(ialu_reg_reg);
9146 %}
9147 
9148 #ifndef AARCH64
9149 instruct andshrI_reg_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3) %{
9150   match(Set dst (AndI src1 (URShiftI src2 src3)));
9151 
9152   size(4);
9153   format %{ "AND    $dst,$src1,$src2>>>$src3" %}
9154   ins_encode %{
9155     __ andr($dst$$Register, $src1$$Register, AsmOperand($src2$$Register, lsr, $src3$$Register));
9156   %}
9157   ins_pipe(ialu_reg_reg);
9158 %}
9159 #endif
9160 
9161 instruct andshrI_reg_reg_imm(iRegI dst, iRegI src1, iRegI src2, immU5 src3) %{
9162   match(Set dst (AndI src1 (URShiftI src2 src3)));
9163 
9164   size(4);
9165   format %{ "and_32 $dst,$src1,$src2>>>$src3" %}
9166   ins_encode %{
9167     __ and_32($dst$$Register, $src1$$Register, AsmOperand($src2$$Register, lsr, $src3$$constant));
9168   %}
9169   ins_pipe(ialu_reg_reg);
9170 %}
9171 
9172 // Immediate And
9173 instruct andI_reg_limm(iRegI dst, iRegI src1, limmI src2) %{
9174   match(Set dst (AndI src1 src2));
9175 
9176   size(4);
9177   format %{ "and_32 $dst,$src1,$src2\t! int" %}
9178   ins_encode %{
9179     __ and_32($dst$$Register, $src1$$Register, $src2$$constant);
9180   %}
9181   ins_pipe(ialu_reg_imm);
9182 %}
9183 
9184 #ifndef AARCH64
9185 instruct andI_reg_limmn(iRegI dst, iRegI src1, limmIn src2) %{
9186   match(Set dst (AndI src1 src2));
9187 
9188   size(4);
9189   format %{ "bic    $dst,$src1,~$src2\t! int" %}
9190   ins_encode %{
9191     __ bic($dst$$Register, $src1$$Register, ~$src2$$constant);
9192   %}
9193   ins_pipe(ialu_reg_imm);
9194 %}
9195 #endif
9196 
9197 // Register And Long
9198 instruct andL_reg_reg(iRegL dst, iRegL src1, iRegL src2) %{
9199   match(Set dst (AndL src1 src2));
9200 
9201   ins_cost(DEFAULT_COST);
9202 #ifdef AARCH64
9203   size(4);
9204   format %{ "AND    $dst,$src1,$src2\t! long" %}
9205   ins_encode %{
9206     __ andr($dst$$Register, $src1$$Register, $src2$$Register);
9207   %}
9208 #else
9209   size(8);
9210   format %{ "AND    $dst,$src1,$src2\t! long" %}
9211   ins_encode %{
9212     __ andr($dst$$Register, $src1$$Register, $src2$$Register);
9213     __ andr($dst$$Register->successor(), $src1$$Register->successor(), $src2$$Register->successor());
9214   %}
9215 #endif
9216   ins_pipe(ialu_reg_reg);
9217 %}
9218 
9219 #ifdef AARCH64
9220 // Immediate And
9221 instruct andL_reg_limm(iRegL dst, iRegL src1, limmL src2) %{
9222   match(Set dst (AndL src1 src2));
9223 
9224   size(4);
9225   format %{ "AND    $dst,$src1,$src2\t! long" %}
9226   ins_encode %{
9227     __ andr($dst$$Register, $src1$$Register, (uintx)$src2$$constant);
9228   %}
9229   ins_pipe(ialu_reg_imm);
9230 %}
9231 #else
9232 // TODO: try immLRot2 instead, (0, $con$$constant) becomes
9233 // (hi($con$$constant), lo($con$$constant)) becomes
9234 instruct andL_reg_immRot(iRegL dst, iRegL src1, immLlowRot con) %{
9235   match(Set dst (AndL src1 con));
9236   ins_cost(DEFAULT_COST);
9237   size(8);
9238   format %{ "AND    $dst,$src1,$con\t! long" %}
9239   ins_encode %{
9240     __ andr($dst$$Register, $src1$$Register, $con$$constant);
9241     __ andr($dst$$Register->successor(), $src1$$Register->successor(), 0);
9242   %}
9243   ins_pipe(ialu_reg_imm);
9244 %}
9245 #endif
9246 
9247 // Or Instructions
9248 // Register Or
9249 instruct orI_reg_reg(iRegI dst, iRegI src1, iRegI src2) %{
9250   match(Set dst (OrI src1 src2));
9251 
9252   size(4);
9253   format %{ "orr_32 $dst,$src1,$src2\t! int" %}
9254   ins_encode %{
9255     __ orr_32($dst$$Register, $src1$$Register, $src2$$Register);
9256   %}
9257   ins_pipe(ialu_reg_reg);
9258 %}
9259 
9260 #ifndef AARCH64
9261 instruct orshlI_reg_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3) %{
9262   match(Set dst (OrI src1 (LShiftI src2 src3)));
9263 
9264   size(4);
9265   format %{ "OR    $dst,$src1,$src2<<$src3" %}
9266   ins_encode %{
9267     __ orr($dst$$Register, $src1$$Register, AsmOperand($src2$$Register, lsl, $src3$$Register));
9268   %}
9269   ins_pipe(ialu_reg_reg);
9270 %}
9271 #endif
9272 
9273 instruct orshlI_reg_reg_imm(iRegI dst, iRegI src1, iRegI src2, immU5 src3) %{
9274   match(Set dst (OrI src1 (LShiftI src2 src3)));
9275 
9276   size(4);
9277   format %{ "orr_32 $dst,$src1,$src2<<$src3" %}
9278   ins_encode %{
9279     __ orr_32($dst$$Register, $src1$$Register, AsmOperand($src2$$Register, lsl, $src3$$constant));
9280   %}
9281   ins_pipe(ialu_reg_reg);
9282 %}
9283 
9284 #ifndef AARCH64
9285 instruct orsarI_reg_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3) %{
9286   match(Set dst (OrI src1 (RShiftI src2 src3)));
9287 
9288   size(4);
9289   format %{ "OR    $dst,$src1,$src2>>$src3" %}
9290   ins_encode %{
9291     __ orr($dst$$Register, $src1$$Register, AsmOperand($src2$$Register, asr, $src3$$Register));
9292   %}
9293   ins_pipe(ialu_reg_reg);
9294 %}
9295 #endif
9296 
9297 instruct orsarI_reg_reg_imm(iRegI dst, iRegI src1, iRegI src2, immU5 src3) %{
9298   match(Set dst (OrI src1 (RShiftI src2 src3)));
9299 
9300   size(4);
9301   format %{ "orr_32 $dst,$src1,$src2>>$src3" %}
9302   ins_encode %{
9303     __ orr_32($dst$$Register, $src1$$Register, AsmOperand($src2$$Register, asr, $src3$$constant));
9304   %}
9305   ins_pipe(ialu_reg_reg);
9306 %}
9307 
9308 #ifndef AARCH64
9309 instruct orshrI_reg_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3) %{
9310   match(Set dst (OrI src1 (URShiftI src2 src3)));
9311 
9312   size(4);
9313   format %{ "OR    $dst,$src1,$src2>>>$src3" %}
9314   ins_encode %{
9315     __ orr($dst$$Register, $src1$$Register, AsmOperand($src2$$Register, lsr, $src3$$Register));
9316   %}
9317   ins_pipe(ialu_reg_reg);
9318 %}
9319 #endif
9320 
9321 instruct orshrI_reg_reg_imm(iRegI dst, iRegI src1, iRegI src2, immU5 src3) %{
9322   match(Set dst (OrI src1 (URShiftI src2 src3)));
9323 
9324   size(4);
9325   format %{ "orr_32 $dst,$src1,$src2>>>$src3" %}
9326   ins_encode %{
9327     __ orr_32($dst$$Register, $src1$$Register, AsmOperand($src2$$Register, lsr, $src3$$constant));
9328   %}
9329   ins_pipe(ialu_reg_reg);
9330 %}
9331 
9332 // Immediate Or
9333 instruct orI_reg_limm(iRegI dst, iRegI src1, limmI src2) %{
9334   match(Set dst (OrI src1 src2));
9335 
9336   size(4);
9337   format %{ "orr_32  $dst,$src1,$src2" %}
9338   ins_encode %{
9339     __ orr_32($dst$$Register, $src1$$Register, $src2$$constant);
9340   %}
9341   ins_pipe(ialu_reg_imm);
9342 %}
9343 // TODO: orn_32 with limmIn
9344 
9345 // Register Or Long
9346 instruct orL_reg_reg(iRegL dst, iRegL src1, iRegL src2) %{
9347   match(Set dst (OrL src1 src2));
9348 
9349   ins_cost(DEFAULT_COST);
9350 #ifdef AARCH64
9351   size(4);
9352   format %{ "OR     $dst,$src1,$src2\t! long" %}
9353   ins_encode %{
9354     __ orr($dst$$Register, $src1$$Register, $src2$$Register);
9355   %}
9356 #else
9357   size(8);
9358   format %{ "OR     $dst.lo,$src1.lo,$src2.lo\t! long\n\t"
9359             "OR     $dst.hi,$src1.hi,$src2.hi" %}
9360   ins_encode %{
9361     __ orr($dst$$Register, $src1$$Register, $src2$$Register);
9362     __ orr($dst$$Register->successor(), $src1$$Register->successor(), $src2$$Register->successor());
9363   %}
9364 #endif
9365   ins_pipe(ialu_reg_reg);
9366 %}
9367 
9368 #ifdef AARCH64
9369 instruct orL_reg_limm(iRegL dst, iRegL src1, limmL src2) %{
9370   match(Set dst (OrL src1 src2));
9371 
9372   size(4);
9373   format %{ "ORR    $dst,$src1,$src2\t! long" %}
9374   ins_encode %{
9375     __ orr($dst$$Register, $src1$$Register, (uintx)$src2$$constant);
9376   %}
9377   ins_pipe(ialu_reg_imm);
9378 %}
9379 #else
9380 // TODO: try immLRot2 instead, (0, $con$$constant) becomes
9381 // (hi($con$$constant), lo($con$$constant)) becomes
9382 instruct orL_reg_immRot(iRegL dst, iRegL src1, immLlowRot con) %{
9383   match(Set dst (OrL src1 con));
9384   ins_cost(DEFAULT_COST);
9385   size(8);
9386   format %{ "OR     $dst.lo,$src1.lo,$con\t! long\n\t"
9387             "OR     $dst.hi,$src1.hi,$con" %}
9388   ins_encode %{
9389     __ orr($dst$$Register, $src1$$Register, $con$$constant);
9390     __ orr($dst$$Register->successor(), $src1$$Register->successor(), 0);
9391   %}
9392   ins_pipe(ialu_reg_imm);
9393 %}
9394 #endif
9395 
9396 #ifdef TODO
9397 // Use SPRegP to match Rthread (TLS register) without spilling.
9398 // Use store_ptr_RegP to match Rthread (TLS register) without spilling.
9399 // Use sp_ptr_RegP to match Rthread (TLS register) without spilling.
9400 instruct orI_reg_castP2X(iRegI dst, iRegI src1, sp_ptr_RegP src2) %{
9401   match(Set dst (OrI src1 (CastP2X src2)));
9402   size(4);
9403   format %{ "OR     $dst,$src1,$src2" %}
9404   ins_encode %{
9405     __ orr($dst$$Register, $src1$$Register, $src2$$Register);
9406   %}
9407   ins_pipe(ialu_reg_reg);
9408 %}
9409 #endif
9410 
9411 // Xor Instructions
9412 // Register Xor
9413 instruct xorI_reg_reg(iRegI dst, iRegI src1, iRegI src2) %{
9414   match(Set dst (XorI src1 src2));
9415 
9416   size(4);
9417   format %{ "eor_32 $dst,$src1,$src2" %}
9418   ins_encode %{
9419     __ eor_32($dst$$Register, $src1$$Register, $src2$$Register);
9420   %}
9421   ins_pipe(ialu_reg_reg);
9422 %}
9423 
9424 #ifndef AARCH64
9425 instruct xorshlI_reg_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3) %{
9426   match(Set dst (XorI src1 (LShiftI src2 src3)));
9427 
9428   size(4);
9429   format %{ "XOR    $dst,$src1,$src2<<$src3" %}
9430   ins_encode %{
9431     __ eor($dst$$Register, $src1$$Register, AsmOperand($src2$$Register, lsl, $src3$$Register));
9432   %}
9433   ins_pipe(ialu_reg_reg);
9434 %}
9435 #endif
9436 
9437 instruct xorshlI_reg_reg_imm(iRegI dst, iRegI src1, iRegI src2, immU5 src3) %{
9438   match(Set dst (XorI src1 (LShiftI src2 src3)));
9439 
9440   size(4);
9441   format %{ "eor_32 $dst,$src1,$src2<<$src3" %}
9442   ins_encode %{
9443     __ eor_32($dst$$Register, $src1$$Register, AsmOperand($src2$$Register, lsl, $src3$$constant));
9444   %}
9445   ins_pipe(ialu_reg_reg);
9446 %}
9447 
9448 #ifndef AARCH64
9449 instruct xorsarI_reg_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3) %{
9450   match(Set dst (XorI src1 (RShiftI src2 src3)));
9451 
9452   size(4);
9453   format %{ "XOR    $dst,$src1,$src2>>$src3" %}
9454   ins_encode %{
9455     __ eor($dst$$Register, $src1$$Register, AsmOperand($src2$$Register, asr, $src3$$Register));
9456   %}
9457   ins_pipe(ialu_reg_reg);
9458 %}
9459 #endif
9460 
9461 instruct xorsarI_reg_reg_imm(iRegI dst, iRegI src1, iRegI src2, immU5 src3) %{
9462   match(Set dst (XorI src1 (RShiftI src2 src3)));
9463 
9464   size(4);
9465   format %{ "eor_32 $dst,$src1,$src2>>$src3" %}
9466   ins_encode %{
9467     __ eor_32($dst$$Register, $src1$$Register, AsmOperand($src2$$Register, asr, $src3$$constant));
9468   %}
9469   ins_pipe(ialu_reg_reg);
9470 %}
9471 
9472 #ifndef AARCH64
9473 instruct xorshrI_reg_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3) %{
9474   match(Set dst (XorI src1 (URShiftI src2 src3)));
9475 
9476   size(4);
9477   format %{ "XOR    $dst,$src1,$src2>>>$src3" %}
9478   ins_encode %{
9479     __ eor($dst$$Register, $src1$$Register, AsmOperand($src2$$Register, lsr, $src3$$Register));
9480   %}
9481   ins_pipe(ialu_reg_reg);
9482 %}
9483 #endif
9484 
9485 instruct xorshrI_reg_reg_imm(iRegI dst, iRegI src1, iRegI src2, immU5 src3) %{
9486   match(Set dst (XorI src1 (URShiftI src2 src3)));
9487 
9488   size(4);
9489   format %{ "eor_32 $dst,$src1,$src2>>>$src3" %}
9490   ins_encode %{
9491     __ eor_32($dst$$Register, $src1$$Register, AsmOperand($src2$$Register, lsr, $src3$$constant));
9492   %}
9493   ins_pipe(ialu_reg_reg);
9494 %}
9495 
9496 // Immediate Xor
9497 instruct xorI_reg_imm(iRegI dst, iRegI src1, limmI src2) %{
9498   match(Set dst (XorI src1 src2));
9499 
9500   size(4);
9501   format %{ "eor_32 $dst,$src1,$src2" %}
9502   ins_encode %{
9503     __ eor_32($dst$$Register, $src1$$Register, $src2$$constant);
9504   %}
9505   ins_pipe(ialu_reg_imm);
9506 %}
9507 
9508 // Register Xor Long
9509 instruct xorL_reg_reg(iRegL dst, iRegL src1, iRegL src2) %{
9510   match(Set dst (XorL src1 src2));
9511   ins_cost(DEFAULT_COST);
9512 #ifdef AARCH64
9513   size(4);
9514   format %{ "XOR     $dst,$src1,$src2\t! long" %}
9515   ins_encode %{
9516     __ eor($dst$$Register, $src1$$Register, $src2$$Register);
9517   %}
9518 #else
9519   size(8);
9520   format %{ "XOR     $dst.hi,$src1.hi,$src2.hi\t! long\n\t"
9521             "XOR     $dst.lo,$src1.lo,$src2.lo\t! long" %}
9522   ins_encode %{
9523     __ eor($dst$$Register, $src1$$Register, $src2$$Register);
9524     __ eor($dst$$Register->successor(), $src1$$Register->successor(), $src2$$Register->successor());
9525   %}
9526 #endif
9527   ins_pipe(ialu_reg_reg);
9528 %}
9529 
9530 #ifdef AARCH64
9531 instruct xorL_reg_limmL(iRegL dst, iRegL src1, limmL con) %{
9532   match(Set dst (XorL src1 con));
9533   ins_cost(DEFAULT_COST);
9534   size(4);
9535   format %{ "EOR     $dst,$src1,$con\t! long" %}
9536   ins_encode %{
9537     __ eor($dst$$Register, $src1$$Register, (uintx)$con$$constant);
9538   %}
9539   ins_pipe(ialu_reg_imm);
9540 %}
9541 #else
9542 // TODO: try immLRot2 instead, (0, $con$$constant) becomes
9543 // (hi($con$$constant), lo($con$$constant)) becomes
9544 instruct xorL_reg_immRot(iRegL dst, iRegL src1, immLlowRot con) %{
9545   match(Set dst (XorL src1 con));
9546   ins_cost(DEFAULT_COST);
9547   size(8);
9548   format %{ "XOR     $dst.hi,$src1.hi,$con\t! long\n\t"
9549             "XOR     $dst.lo,$src1.lo,0\t! long" %}
9550   ins_encode %{
9551     __ eor($dst$$Register, $src1$$Register, $con$$constant);
9552     __ eor($dst$$Register->successor(), $src1$$Register->successor(), 0);
9553   %}
9554   ins_pipe(ialu_reg_imm);
9555 %}
9556 #endif // AARCH64
9557 
9558 //----------Convert to Boolean-------------------------------------------------
9559 instruct convI2B( iRegI dst, iRegI src, flagsReg ccr ) %{
9560   match(Set dst (Conv2B src));
9561   effect(KILL ccr);
9562 #ifdef AARCH64
9563   size(8);
9564   ins_cost(DEFAULT_COST*2);
9565   format %{ "cmp_32 $src,ZR\n\t"
9566             "cset_w $dst, ne" %}
9567   ins_encode %{
9568     __ cmp_32($src$$Register, ZR);
9569     __ cset_w($dst$$Register, ne);
9570   %}
9571 #else
9572   size(12);
9573   ins_cost(DEFAULT_COST*2);
9574   format %{ "TST    $src,$src \n\t"
9575             "MOV    $dst, 0   \n\t"
9576             "MOV.ne $dst, 1" %}
9577   ins_encode %{ // FIXME: can do better?
9578     __ tst($src$$Register, $src$$Register);
9579     __ mov($dst$$Register, 0);
9580     __ mov($dst$$Register, 1, ne);
9581   %}
9582 #endif
9583   ins_pipe(ialu_reg_ialu);
9584 %}
9585 
9586 instruct convP2B( iRegI dst, iRegP src, flagsReg ccr ) %{
9587   match(Set dst (Conv2B src));
9588   effect(KILL ccr);
9589 #ifdef AARCH64
9590   size(8);
9591   ins_cost(DEFAULT_COST*2);
9592   format %{ "CMP    $src,ZR\n\t"
9593             "cset   $dst, ne" %}
9594   ins_encode %{
9595     __ cmp($src$$Register, ZR);
9596     __ cset($dst$$Register, ne);
9597   %}
9598 #else
9599   size(12);
9600   ins_cost(DEFAULT_COST*2);
9601   format %{ "TST    $src,$src \n\t"
9602             "MOV    $dst, 0   \n\t"
9603             "MOV.ne $dst, 1" %}
9604   ins_encode %{
9605     __ tst($src$$Register, $src$$Register);
9606     __ mov($dst$$Register, 0);
9607     __ mov($dst$$Register, 1, ne);
9608   %}
9609 #endif
9610   ins_pipe(ialu_reg_ialu);
9611 %}
9612 
9613 instruct cmpLTMask_reg_reg( iRegI dst, iRegI p, iRegI q, flagsReg ccr ) %{
9614   match(Set dst (CmpLTMask p q));
9615   effect( KILL ccr );
9616 #ifdef AARCH64
9617   size(8);
9618   ins_cost(DEFAULT_COST*2);
9619   format %{ "CMP_w   $p,$q\n\t"
9620             "CSETM_w $dst, lt" %}
9621   ins_encode %{
9622     __ cmp_w($p$$Register, $q$$Register);
9623     __ csetm_w($dst$$Register, lt);
9624   %}
9625 #else
9626   ins_cost(DEFAULT_COST*3);
9627   format %{ "CMP    $p,$q\n\t"
9628             "MOV    $dst, #0\n\t"
9629             "MOV.lt $dst, #-1" %}
9630   ins_encode %{
9631     __ cmp($p$$Register, $q$$Register);
9632     __ mov($dst$$Register, 0);
9633     __ mvn($dst$$Register, 0, lt);
9634   %}
9635 #endif
9636   ins_pipe(ialu_reg_reg_ialu);
9637 %}
9638 
9639 instruct cmpLTMask_reg_imm( iRegI dst, iRegI p, aimmI q, flagsReg ccr ) %{
9640   match(Set dst (CmpLTMask p q));
9641   effect( KILL ccr );
9642 #ifdef AARCH64
9643   size(8);
9644   ins_cost(DEFAULT_COST*2);
9645   format %{ "CMP_w   $p,$q\n\t"
9646             "CSETM_w $dst, lt" %}
9647   ins_encode %{
9648     __ cmp_w($p$$Register, $q$$constant);
9649     __ csetm_w($dst$$Register, lt);
9650   %}
9651 #else
9652   ins_cost(DEFAULT_COST*3);
9653   format %{ "CMP    $p,$q\n\t"
9654             "MOV    $dst, #0\n\t"
9655             "MOV.lt $dst, #-1" %}
9656   ins_encode %{
9657     __ cmp($p$$Register, $q$$constant);
9658     __ mov($dst$$Register, 0);
9659     __ mvn($dst$$Register, 0, lt);
9660   %}
9661 #endif
9662   ins_pipe(ialu_reg_reg_ialu);
9663 %}
9664 
9665 #ifdef AARCH64
9666 instruct cadd_cmpLTMask3( iRegI dst, iRegI p, iRegI q, iRegI y, iRegI x, flagsReg ccr ) %{
9667   match(Set dst (AddI (AndI (CmpLTMask p q) y) x));
9668   effect( TEMP dst, KILL ccr );
9669   size(12);
9670   ins_cost(DEFAULT_COST*3);
9671   format %{ "CMP_w  $p,$q\n\t"
9672             "ADD_w  $dst,$y,$x\n\t"
9673             "CSEL_w $dst,$dst,$x,lt" %}
9674   ins_encode %{
9675     __ cmp_w($p$$Register, $q$$Register);
9676     __ add_w($dst$$Register, $y$$Register, $x$$Register);
9677     __ csel_w($dst$$Register, $dst$$Register, $x$$Register, lt);
9678   %}
9679   ins_pipe( cadd_cmpltmask );
9680 %}
9681 #else
9682 instruct cadd_cmpLTMask3( iRegI p, iRegI q, iRegI y, iRegI z, flagsReg ccr ) %{
9683   match(Set z (AddI (AndI (CmpLTMask p q) y) z));
9684   effect( KILL ccr );
9685   ins_cost(DEFAULT_COST*2);
9686   format %{ "CMP    $p,$q\n\t"
9687             "ADD.lt $z,$y,$z" %}
9688   ins_encode %{
9689     __ cmp($p$$Register, $q$$Register);
9690     __ add($z$$Register, $y$$Register, $z$$Register, lt);
9691   %}
9692   ins_pipe( cadd_cmpltmask );
9693 %}
9694 #endif
9695 
9696 #ifdef AARCH64
9697 instruct cadd_cmpLTMask4( iRegI dst, iRegI p, aimmI q, iRegI y, iRegI x, flagsReg ccr ) %{
9698   match(Set dst (AddI (AndI (CmpLTMask p q) y) x));
9699   effect( TEMP dst, KILL ccr );
9700   size(12);
9701   ins_cost(DEFAULT_COST*3);
9702   format %{ "CMP_w  $p,$q\n\t"
9703             "ADD_w  $dst,$y,$x\n\t"
9704             "CSEL_w $dst,$dst,$x,lt" %}
9705   ins_encode %{
9706     __ cmp_w($p$$Register, $q$$constant);
9707     __ add_w($dst$$Register, $y$$Register, $x$$Register);
9708     __ csel_w($dst$$Register, $dst$$Register, $x$$Register, lt);
9709   %}
9710   ins_pipe( cadd_cmpltmask );
9711 %}
9712 #else
9713 // FIXME: remove unused "dst"
9714 instruct cadd_cmpLTMask4( iRegI dst, iRegI p, aimmI q, iRegI y, iRegI z, flagsReg ccr ) %{
9715   match(Set z (AddI (AndI (CmpLTMask p q) y) z));
9716   effect( KILL ccr );
9717   ins_cost(DEFAULT_COST*2);
9718   format %{ "CMP    $p,$q\n\t"
9719             "ADD.lt $z,$y,$z" %}
9720   ins_encode %{
9721     __ cmp($p$$Register, $q$$constant);
9722     __ add($z$$Register, $y$$Register, $z$$Register, lt);
9723   %}
9724   ins_pipe( cadd_cmpltmask );
9725 %}
9726 #endif // !AARCH64
9727 
9728 #ifdef AARCH64
9729 instruct cadd_cmpLTMask( iRegI dst, iRegI p, iRegI q, iRegI y, flagsReg ccr ) %{
9730   match(Set dst (AddI (AndI (CmpLTMask p q) y) (SubI p q)));
9731   effect( TEMP dst, KILL ccr );
9732   size(12);
9733   ins_cost(DEFAULT_COST*3);
9734   format %{ "SUBS_w $p,$p,$q\n\t"
9735             "ADD_w  $dst,$y,$p\n\t"
9736             "CSEL_w $dst,$dst,$p,lt" %}
9737   ins_encode %{
9738     __ subs_w($p$$Register, $p$$Register, $q$$Register);
9739     __ add_w($dst$$Register, $y$$Register, $p$$Register);
9740     __ csel_w($dst$$Register, $dst$$Register, $p$$Register, lt);
9741   %}
9742   ins_pipe( cadd_cmpltmask ); // FIXME
9743 %}
9744 #else
9745 instruct cadd_cmpLTMask( iRegI p, iRegI q, iRegI y, flagsReg ccr ) %{
9746   match(Set p (AddI (AndI (CmpLTMask p q) y) (SubI p q)));
9747   effect( KILL ccr );
9748   ins_cost(DEFAULT_COST*2);
9749   format %{ "SUBS   $p,$p,$q\n\t"
9750             "ADD.lt $p,$y,$p" %}
9751   ins_encode %{
9752     __ subs($p$$Register, $p$$Register, $q$$Register);
9753     __ add($p$$Register, $y$$Register, $p$$Register, lt);
9754   %}
9755   ins_pipe( cadd_cmpltmask );
9756 %}
9757 #endif
9758 
9759 //----------Arithmetic Conversion Instructions---------------------------------
9760 // The conversions operations are all Alpha sorted.  Please keep it that way!
9761 
9762 instruct convD2F_reg(regF dst, regD src) %{
9763   match(Set dst (ConvD2F src));
9764   size(4);
9765   format %{ "FCVTSD  $dst,$src" %}
9766   ins_encode %{
9767     __ convert_d2f($dst$$FloatRegister, $src$$FloatRegister);
9768   %}
9769   ins_pipe(fcvtD2F);
9770 %}
9771 
9772 // Convert a double to an int in a float register.
9773 // If the double is a NAN, stuff a zero in instead.
9774 
9775 #ifdef AARCH64
9776 instruct convD2I_reg_reg(iRegI dst, regD src) %{
9777   match(Set dst (ConvD2I src));
9778   ins_cost(DEFAULT_COST*2 + MEMORY_REF_COST*2 + BRANCH_COST); // FIXME
9779   format %{ "FCVTZS_wd $dst, $src" %}
9780   ins_encode %{
9781     __ fcvtzs_wd($dst$$Register, $src$$FloatRegister);
9782   %}
9783   ins_pipe(fcvtD2I);
9784 %}
9785 
9786 instruct convD2L_reg_reg(iRegL dst, regD src) %{
9787   match(Set dst (ConvD2L src));
9788   ins_cost(DEFAULT_COST*2 + MEMORY_REF_COST*2 + BRANCH_COST); // FIXME
9789   format %{ "FCVTZS_xd $dst, $src" %}
9790   ins_encode %{
9791     __ fcvtzs_xd($dst$$Register, $src$$FloatRegister);
9792   %}
9793   ins_pipe(fcvtD2L);
9794 %}
9795 #else
9796 instruct convD2I_reg_reg(iRegI dst, regD src, regF tmp) %{
9797   match(Set dst (ConvD2I src));
9798   effect( TEMP tmp );
9799   ins_cost(DEFAULT_COST*2 + MEMORY_REF_COST*2 + BRANCH_COST); // FIXME
9800   format %{ "FTOSIZD  $tmp,$src\n\t"
9801             "FMRS     $dst, $tmp" %}
9802   ins_encode %{
9803     __ ftosizd($tmp$$FloatRegister, $src$$FloatRegister);
9804     __ fmrs($dst$$Register, $tmp$$FloatRegister);
9805   %}
9806   ins_pipe(fcvtD2I);
9807 %}
9808 #endif
9809 
9810 // Convert a double to a long in a double register.
9811 // If the double is a NAN, stuff a zero in instead.
9812 
9813 #ifndef AARCH64
9814 // Double to Long conversion
9815 instruct convD2L_reg(R0R1RegL dst, regD src) %{
9816   match(Set dst (ConvD2L src));
9817   effect(CALL);
9818   ins_cost(MEMORY_REF_COST); // FIXME
9819   format %{ "convD2L    $dst,$src\t ! call to SharedRuntime::d2l" %}
9820   ins_encode %{
9821 #ifndef __ABI_HARD__
9822     __ fmrrd($dst$$Register, $dst$$Register->successor(), $src$$FloatRegister);
9823 #else
9824     if ($src$$FloatRegister != D0) {
9825       __ mov_double(D0, $src$$FloatRegister);
9826     }
9827 #endif
9828     address target = CAST_FROM_FN_PTR(address, SharedRuntime::d2l);
9829     __ call(target, relocInfo::runtime_call_type);
9830   %}
9831   ins_pipe(fcvtD2L);
9832 %}
9833 #endif
9834 
9835 instruct convF2D_reg(regD dst, regF src) %{
9836   match(Set dst (ConvF2D src));
9837   size(4);
9838   format %{ "FCVTDS  $dst,$src" %}
9839   ins_encode %{
9840     __ convert_f2d($dst$$FloatRegister, $src$$FloatRegister);
9841   %}
9842   ins_pipe(fcvtF2D);
9843 %}
9844 
9845 #ifdef AARCH64
9846 instruct convF2I_reg_reg(iRegI dst, regF src) %{
9847   match(Set dst (ConvF2I src));
9848   ins_cost(DEFAULT_COST*2 + MEMORY_REF_COST*2 + BRANCH_COST); // FIXME
9849   size(4);
9850   format %{ "FCVTZS_ws $dst, $src" %}
9851   ins_encode %{
9852     __ fcvtzs_ws($dst$$Register, $src$$FloatRegister);
9853   %}
9854   ins_pipe(fcvtF2I);
9855 %}
9856 
9857 instruct convF2L_reg_reg(iRegL dst, regF src) %{
9858   match(Set dst (ConvF2L src));
9859   ins_cost(DEFAULT_COST*2 + MEMORY_REF_COST*2 + BRANCH_COST); // FIXME
9860   size(4);
9861   format %{ "FCVTZS_xs $dst, $src" %}
9862   ins_encode %{
9863     __ fcvtzs_xs($dst$$Register, $src$$FloatRegister);
9864   %}
9865   ins_pipe(fcvtF2L);
9866 %}
9867 #else
9868 instruct convF2I_reg_reg(iRegI dst, regF src, regF tmp) %{
9869   match(Set dst (ConvF2I src));
9870   effect( TEMP tmp );
9871   ins_cost(DEFAULT_COST*2 + MEMORY_REF_COST*2 + BRANCH_COST); // FIXME
9872   size(8);
9873   format %{ "FTOSIZS  $tmp,$src\n\t"
9874             "FMRS     $dst, $tmp" %}
9875   ins_encode %{
9876     __ ftosizs($tmp$$FloatRegister, $src$$FloatRegister);
9877     __ fmrs($dst$$Register, $tmp$$FloatRegister);
9878   %}
9879   ins_pipe(fcvtF2I);
9880 %}
9881 
9882 // Float to Long conversion
9883 instruct convF2L_reg(R0R1RegL dst, regF src, R0RegI arg1) %{
9884   match(Set dst (ConvF2L src));
9885   ins_cost(DEFAULT_COST*2 + MEMORY_REF_COST*2 + BRANCH_COST); // FIXME
9886   effect(CALL);
9887   format %{ "convF2L  $dst,$src\t! call to SharedRuntime::f2l" %}
9888   ins_encode %{
9889 #ifndef __ABI_HARD__
9890     __ fmrs($arg1$$Register, $src$$FloatRegister);
9891 #else
9892     if($src$$FloatRegister != S0) {
9893       __ mov_float(S0, $src$$FloatRegister);
9894     }
9895 #endif
9896     address target = CAST_FROM_FN_PTR(address, SharedRuntime::f2l);
9897     __ call(target, relocInfo::runtime_call_type);
9898   %}
9899   ins_pipe(fcvtF2L);
9900 %}
9901 #endif
9902 
9903 #ifdef AARCH64
9904 instruct convI2D_reg_reg(iRegI src, regD dst) %{
9905   match(Set dst (ConvI2D src));
9906   ins_cost(DEFAULT_COST + MEMORY_REF_COST); // FIXME
9907   size(4);
9908   format %{ "SCVTF_dw $dst,$src" %}
9909   ins_encode %{
9910       __ scvtf_dw($dst$$FloatRegister, $src$$Register);
9911   %}
9912   ins_pipe(fcvtI2D);
9913 %}
9914 #else
9915 instruct convI2D_reg_reg(iRegI src, regD_low dst) %{
9916   match(Set dst (ConvI2D src));
9917   ins_cost(DEFAULT_COST + MEMORY_REF_COST); // FIXME
9918   size(8);
9919   format %{ "FMSR     $dst,$src \n\t"
9920             "FSITOD   $dst $dst"%}
9921   ins_encode %{
9922       __ fmsr($dst$$FloatRegister, $src$$Register);
9923       __ fsitod($dst$$FloatRegister, $dst$$FloatRegister);
9924   %}
9925   ins_pipe(fcvtI2D);
9926 %}
9927 #endif
9928 
9929 instruct convI2F_reg_reg( regF dst, iRegI src ) %{
9930   match(Set dst (ConvI2F src));
9931   ins_cost(DEFAULT_COST + MEMORY_REF_COST); // FIXME
9932 #ifdef AARCH64
9933   size(4);
9934   format %{ "SCVTF_sw $dst,$src" %}
9935   ins_encode %{
9936       __ scvtf_sw($dst$$FloatRegister, $src$$Register);
9937   %}
9938 #else
9939   size(8);
9940   format %{ "FMSR     $dst,$src \n\t"
9941             "FSITOS   $dst, $dst"%}
9942   ins_encode %{
9943       __ fmsr($dst$$FloatRegister, $src$$Register);
9944       __ fsitos($dst$$FloatRegister, $dst$$FloatRegister);
9945   %}
9946 #endif
9947   ins_pipe(fcvtI2F);
9948 %}
9949 
9950 instruct convI2L_reg(iRegL dst, iRegI src) %{
9951   match(Set dst (ConvI2L src));
9952 #ifdef AARCH64
9953   size(4);
9954   format %{ "SXTW   $dst,$src\t! int->long" %}
9955   ins_encode %{
9956     __ sxtw($dst$$Register, $src$$Register);
9957   %}
9958 #else
9959   size(8);
9960   format %{ "MOV    $dst.lo, $src \n\t"
9961             "ASR    $dst.hi,$src,31\t! int->long" %}
9962   ins_encode %{
9963     __ mov($dst$$Register, $src$$Register);
9964     __ mov($dst$$Register->successor(), AsmOperand($src$$Register, asr, 31));
9965   %}
9966 #endif
9967   ins_pipe(ialu_reg_reg);
9968 %}
9969 
9970 // Zero-extend convert int to long
9971 instruct convI2L_reg_zex(iRegL dst, iRegI src, immL_32bits mask ) %{
9972   match(Set dst (AndL (ConvI2L src) mask) );
9973 #ifdef AARCH64
9974   size(4);
9975   format %{ "mov_w  $dst,$src\t! zero-extend int to long"  %}
9976   ins_encode %{
9977     __ mov_w($dst$$Register, $src$$Register);
9978   %}
9979 #else
9980   size(8);
9981   format %{ "MOV    $dst.lo,$src.lo\t! zero-extend int to long\n\t"
9982             "MOV    $dst.hi, 0"%}
9983   ins_encode %{
9984     __ mov($dst$$Register, $src$$Register);
9985     __ mov($dst$$Register->successor(), 0);
9986   %}
9987 #endif
9988   ins_pipe(ialu_reg_reg);
9989 %}
9990 
9991 // Zero-extend long
9992 instruct zerox_long(iRegL dst, iRegL src, immL_32bits mask ) %{
9993   match(Set dst (AndL src mask) );
9994 #ifdef AARCH64
9995   size(4);
9996   format %{ "mov_w  $dst,$src\t! zero-extend long"  %}
9997   ins_encode %{
9998     __ mov_w($dst$$Register, $src$$Register);
9999   %}
10000 #else
10001   size(8);
10002   format %{ "MOV    $dst.lo,$src.lo\t! zero-extend long\n\t"
10003             "MOV    $dst.hi, 0"%}
10004   ins_encode %{
10005     __ mov($dst$$Register, $src$$Register);
10006     __ mov($dst$$Register->successor(), 0);
10007   %}
10008 #endif
10009   ins_pipe(ialu_reg_reg);
10010 %}
10011 
10012 instruct MoveF2I_reg_reg(iRegI dst, regF src) %{
10013   match(Set dst (MoveF2I src));
10014   effect(DEF dst, USE src);
10015   ins_cost(MEMORY_REF_COST); // FIXME
10016 
10017   size(4);
10018   format %{ "FMRS   $dst,$src\t! MoveF2I" %}
10019   ins_encode %{
10020     __ fmrs($dst$$Register, $src$$FloatRegister);
10021   %}
10022   ins_pipe(iload_mem); // FIXME
10023 %}
10024 
10025 instruct MoveI2F_reg_reg(regF dst, iRegI src) %{
10026   match(Set dst (MoveI2F src));
10027   ins_cost(MEMORY_REF_COST); // FIXME
10028 
10029   size(4);
10030   format %{ "FMSR   $dst,$src\t! MoveI2F" %}
10031   ins_encode %{
10032     __ fmsr($dst$$FloatRegister, $src$$Register);
10033   %}
10034   ins_pipe(iload_mem); // FIXME
10035 %}
10036 
10037 instruct MoveD2L_reg_reg(iRegL dst, regD src) %{
10038   match(Set dst (MoveD2L src));
10039   effect(DEF dst, USE src);
10040   ins_cost(MEMORY_REF_COST); // FIXME
10041 
10042   size(4);
10043 #ifdef AARCH64
10044   format %{ "FMOV_xd  $dst,$src\t! MoveD2L" %}
10045   ins_encode %{
10046     __ fmov_xd($dst$$Register, $src$$FloatRegister);
10047   %}
10048 #else
10049   format %{ "FMRRD    $dst,$src\t! MoveD2L" %}
10050   ins_encode %{
10051     __ fmrrd($dst$$Register, $dst$$Register->successor(), $src$$FloatRegister);
10052   %}
10053 #endif
10054   ins_pipe(iload_mem); // FIXME
10055 %}
10056 
10057 instruct MoveL2D_reg_reg(regD dst, iRegL src) %{
10058   match(Set dst (MoveL2D src));
10059   effect(DEF dst, USE src);
10060   ins_cost(MEMORY_REF_COST); // FIXME
10061 
10062   size(4);
10063 #ifdef AARCH64
10064   format %{ "FMOV_dx $dst,$src\t! MoveL2D" %}
10065   ins_encode %{
10066     __ fmov_dx($dst$$FloatRegister, $src$$Register);
10067   %}
10068 #else
10069   format %{ "FMDRR   $dst,$src\t! MoveL2D" %}
10070   ins_encode %{
10071     __ fmdrr($dst$$FloatRegister, $src$$Register, $src$$Register->successor());
10072   %}
10073 #endif
10074   ins_pipe(ialu_reg_reg); // FIXME
10075 %}
10076 
10077 //-----------
10078 // Long to Double conversion
10079 
10080 #ifdef AARCH64
10081 instruct convL2D(regD dst, iRegL src) %{
10082   match(Set dst (ConvL2D src));
10083   ins_cost(DEFAULT_COST*2 + MEMORY_REF_COST*2 + BRANCH_COST); // FIXME
10084   size(4);
10085   format %{ "SCVTF_dx $dst, $src" %}
10086   ins_encode %{
10087     __ scvtf_dx($dst$$FloatRegister, $src$$Register);
10088   %}
10089   ins_pipe(fcvtL2D);
10090 %}
10091 
10092 instruct convL2F(regF dst, iRegL src) %{
10093   match(Set dst (ConvL2F src));
10094   ins_cost(DEFAULT_COST*2 + MEMORY_REF_COST*2 + BRANCH_COST); // FIXME
10095   size(4);
10096   format %{ "SCVTF_sx $dst, $src" %}
10097   ins_encode %{
10098     __ scvtf_sx($dst$$FloatRegister, $src$$Register);
10099   %}
10100   ins_pipe(fcvtL2F);
10101 %}
10102 #else
10103 // Magic constant, 0x43300000
10104 instruct loadConI_x43300000(iRegI dst) %{
10105   effect(DEF dst);
10106   size(8);
10107   format %{ "MOV_SLOW  $dst,0x43300000\t! 2^52" %}
10108   ins_encode %{
10109     __ mov_slow($dst$$Register, 0x43300000);
10110   %}
10111   ins_pipe(ialu_none);
10112 %}
10113 
10114 // Magic constant, 0x41f00000
10115 instruct loadConI_x41f00000(iRegI dst) %{
10116   effect(DEF dst);
10117   size(8);
10118   format %{ "MOV_SLOW  $dst, 0x41f00000\t! 2^32" %}
10119   ins_encode %{
10120     __ mov_slow($dst$$Register, 0x41f00000);
10121   %}
10122   ins_pipe(ialu_none);
10123 %}
10124 
10125 instruct loadConI_x0(iRegI dst) %{
10126   effect(DEF dst);
10127   size(4);
10128   format %{ "MOV  $dst, 0x0\t! 0" %}
10129   ins_encode %{
10130     __ mov($dst$$Register, 0);
10131   %}
10132   ins_pipe(ialu_none);
10133 %}
10134 
10135 // Construct a double from two float halves
10136 instruct regDHi_regDLo_to_regD(regD_low dst, regD_low src1, regD_low src2) %{
10137   effect(DEF dst, USE src1, USE src2);
10138   size(8);
10139   format %{ "FCPYS  $dst.hi,$src1.hi\n\t"
10140             "FCPYS  $dst.lo,$src2.lo" %}
10141   ins_encode %{
10142     __ fcpys($dst$$FloatRegister->successor(), $src1$$FloatRegister->successor());
10143     __ fcpys($dst$$FloatRegister, $src2$$FloatRegister);
10144   %}
10145   ins_pipe(faddD_reg_reg);
10146 %}
10147 
10148 #ifndef AARCH64
10149 // Convert integer in high half of a double register (in the lower half of
10150 // the double register file) to double
10151 instruct convI2D_regDHi_regD(regD dst, regD_low src) %{
10152   effect(DEF dst, USE src);
10153   size(4);
10154   format %{ "FSITOD  $dst,$src" %}
10155   ins_encode %{
10156     __ fsitod($dst$$FloatRegister, $src$$FloatRegister->successor());
10157   %}
10158   ins_pipe(fcvtLHi2D);
10159 %}
10160 #endif
10161 
10162 // Add float double precision
10163 instruct addD_regD_regD(regD dst, regD src1, regD src2) %{
10164   effect(DEF dst, USE src1, USE src2);
10165   size(4);
10166   format %{ "FADDD  $dst,$src1,$src2" %}
10167   ins_encode %{
10168     __ add_double($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister);
10169   %}
10170   ins_pipe(faddD_reg_reg);
10171 %}
10172 
10173 // Sub float double precision
10174 instruct subD_regD_regD(regD dst, regD src1, regD src2) %{
10175   effect(DEF dst, USE src1, USE src2);
10176   size(4);
10177   format %{ "FSUBD  $dst,$src1,$src2" %}
10178   ins_encode %{
10179     __ sub_double($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister);
10180   %}
10181   ins_pipe(faddD_reg_reg);
10182 %}
10183 
10184 // Mul float double precision
10185 instruct mulD_regD_regD(regD dst, regD src1, regD src2) %{
10186   effect(DEF dst, USE src1, USE src2);
10187   size(4);
10188   format %{ "FMULD  $dst,$src1,$src2" %}
10189   ins_encode %{
10190     __ mul_double($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister);
10191   %}
10192   ins_pipe(fmulD_reg_reg);
10193 %}
10194 
10195 instruct regL_to_regD(regD dst, iRegL src) %{
10196   // No match rule to avoid chain rule match.
10197   effect(DEF dst, USE src);
10198   ins_cost(MEMORY_REF_COST);
10199   size(4);
10200   format %{ "FMDRR   $dst,$src\t! regL to regD" %}
10201   ins_encode %{
10202     __ fmdrr($dst$$FloatRegister, $src$$Register, $src$$Register->successor());
10203   %}
10204   ins_pipe(ialu_reg_reg); // FIXME
10205 %}
10206 
10207 instruct regI_regI_to_regD(regD dst, iRegI src1, iRegI src2) %{
10208   // No match rule to avoid chain rule match.
10209   effect(DEF dst, USE src1, USE src2);
10210   ins_cost(MEMORY_REF_COST);
10211   size(4);
10212   format %{ "FMDRR   $dst,$src1,$src2\t! regI,regI to regD" %}
10213   ins_encode %{
10214     __ fmdrr($dst$$FloatRegister, $src1$$Register, $src2$$Register);
10215   %}
10216   ins_pipe(ialu_reg_reg); // FIXME
10217 %}
10218 
10219 instruct convL2D_reg_slow_fxtof(regD dst, iRegL src) %{
10220   match(Set dst (ConvL2D src));
10221   ins_cost(DEFAULT_COST*8 + MEMORY_REF_COST*6); // FIXME
10222 
10223   expand %{
10224     regD_low   tmpsrc;
10225     iRegI      ix43300000;
10226     iRegI      ix41f00000;
10227     iRegI      ix0;
10228     regD_low   dx43300000;
10229     regD       dx41f00000;
10230     regD       tmp1;
10231     regD_low   tmp2;
10232     regD       tmp3;
10233     regD       tmp4;
10234 
10235     regL_to_regD(tmpsrc, src);
10236 
10237     loadConI_x43300000(ix43300000);
10238     loadConI_x41f00000(ix41f00000);
10239     loadConI_x0(ix0);
10240 
10241     regI_regI_to_regD(dx43300000, ix0, ix43300000);
10242     regI_regI_to_regD(dx41f00000, ix0, ix41f00000);
10243 
10244     convI2D_regDHi_regD(tmp1, tmpsrc);
10245     regDHi_regDLo_to_regD(tmp2, dx43300000, tmpsrc);
10246     subD_regD_regD(tmp3, tmp2, dx43300000);
10247     mulD_regD_regD(tmp4, tmp1, dx41f00000);
10248     addD_regD_regD(dst, tmp3, tmp4);
10249   %}
10250 %}
10251 #endif // !AARCH64
10252 
10253 instruct convL2I_reg(iRegI dst, iRegL src) %{
10254   match(Set dst (ConvL2I src));
10255   size(4);
10256 #ifdef AARCH64
10257   format %{ "MOV_w  $dst,$src\t! long->int" %}
10258   ins_encode %{
10259     __ mov_w($dst$$Register, $src$$Register);
10260   %}
10261 #else
10262   format %{ "MOV    $dst,$src.lo\t! long->int" %}
10263   ins_encode %{
10264     __ mov($dst$$Register, $src$$Register);
10265   %}
10266 #endif
10267   ins_pipe(ialu_move_reg_I_to_L);
10268 %}
10269 
10270 #ifndef AARCH64
10271 // Register Shift Right Immediate
10272 instruct shrL_reg_imm6_L2I(iRegI dst, iRegL src, immI_32_63 cnt) %{
10273   match(Set dst (ConvL2I (RShiftL src cnt)));
10274   size(4);
10275   format %{ "ASR    $dst,$src.hi,($cnt - 32)\t! long->int or mov if $cnt==32" %}
10276   ins_encode %{
10277     if ($cnt$$constant == 32) {
10278       __ mov($dst$$Register, $src$$Register->successor());
10279     } else {
10280       __ mov($dst$$Register, AsmOperand($src$$Register->successor(), asr, $cnt$$constant - 32));
10281     }
10282   %}
10283   ins_pipe(ialu_reg_imm);
10284 %}
10285 #endif
10286 
10287 
10288 //----------Control Flow Instructions------------------------------------------
10289 // Compare Instructions
10290 // Compare Integers
10291 instruct compI_iReg(flagsReg icc, iRegI op1, iRegI op2) %{
10292   match(Set icc (CmpI op1 op2));
10293   effect( DEF icc, USE op1, USE op2 );
10294 
10295   size(4);
10296   format %{ "cmp_32 $op1,$op2\t! int" %}
10297   ins_encode %{
10298     __ cmp_32($op1$$Register, $op2$$Register);
10299   %}
10300   ins_pipe(ialu_cconly_reg_reg);
10301 %}
10302 
10303 #ifdef _LP64
10304 // Compare compressed pointers
10305 instruct compN_reg2(flagsRegU icc, iRegN op1, iRegN op2) %{
10306   match(Set icc (CmpN op1 op2));
10307   effect( DEF icc, USE op1, USE op2 );
10308 
10309   size(4);
10310   format %{ "cmp_32 $op1,$op2\t! int" %}
10311   ins_encode %{
10312     __ cmp_32($op1$$Register, $op2$$Register);
10313   %}
10314   ins_pipe(ialu_cconly_reg_reg);
10315 %}
10316 #endif
10317 
10318 instruct compU_iReg(flagsRegU icc, iRegI op1, iRegI op2) %{
10319   match(Set icc (CmpU op1 op2));
10320 
10321   size(4);
10322   format %{ "cmp_32 $op1,$op2\t! unsigned int" %}
10323   ins_encode %{
10324     __ cmp_32($op1$$Register, $op2$$Register);
10325   %}
10326   ins_pipe(ialu_cconly_reg_reg);
10327 %}
10328 
10329 instruct compI_iReg_immneg(flagsReg icc, iRegI op1, aimmIneg op2) %{
10330   match(Set icc (CmpI op1 op2));
10331   effect( DEF icc, USE op1 );
10332 
10333   size(4);
10334   format %{ "cmn_32 $op1,-$op2\t! int" %}
10335   ins_encode %{
10336     __ cmn_32($op1$$Register, -$op2$$constant);
10337   %}
10338   ins_pipe(ialu_cconly_reg_imm);
10339 %}
10340 
10341 instruct compI_iReg_imm(flagsReg icc, iRegI op1, aimmI op2) %{
10342   match(Set icc (CmpI op1 op2));
10343   effect( DEF icc, USE op1 );
10344 
10345   size(4);
10346   format %{ "cmp_32 $op1,$op2\t! int" %}
10347   ins_encode %{
10348     __ cmp_32($op1$$Register, $op2$$constant);
10349   %}
10350   ins_pipe(ialu_cconly_reg_imm);
10351 %}
10352 
10353 instruct testI_reg_reg( flagsReg_EQNELTGE icc, iRegI op1, iRegI op2, immI0 zero ) %{
10354   match(Set icc (CmpI (AndI op1 op2) zero));
10355   size(4);
10356   format %{ "tst_32 $op2,$op1" %}
10357 
10358   ins_encode %{
10359     __ tst_32($op1$$Register, $op2$$Register);
10360   %}
10361   ins_pipe(ialu_cconly_reg_reg_zero);
10362 %}
10363 
10364 #ifndef AARCH64
10365 instruct testshlI_reg_reg_reg( flagsReg_EQNELTGE icc, iRegI op1, iRegI op2, iRegI op3, immI0 zero ) %{
10366   match(Set icc (CmpI (AndI op1 (LShiftI op2 op3)) zero));
10367   size(4);
10368   format %{ "TST   $op2,$op1<<$op3" %}
10369 
10370   ins_encode %{
10371     __ tst($op1$$Register, AsmOperand($op2$$Register, lsl, $op3$$Register));
10372   %}
10373   ins_pipe(ialu_cconly_reg_reg_zero);
10374 %}
10375 #endif
10376 
10377 instruct testshlI_reg_reg_imm( flagsReg_EQNELTGE icc, iRegI op1, iRegI op2, immU5 op3, immI0 zero ) %{
10378   match(Set icc (CmpI (AndI op1 (LShiftI op2 op3)) zero));
10379   size(4);
10380   format %{ "tst_32 $op2,$op1<<$op3" %}
10381 
10382   ins_encode %{
10383     __ tst_32($op1$$Register, AsmOperand($op2$$Register, lsl, $op3$$constant));
10384   %}
10385   ins_pipe(ialu_cconly_reg_reg_zero);
10386 %}
10387 
10388 #ifndef AARCH64
10389 instruct testsarI_reg_reg_reg( flagsReg_EQNELTGE icc, iRegI op1, iRegI op2, iRegI op3, immI0 zero ) %{
10390   match(Set icc (CmpI (AndI op1 (RShiftI op2 op3)) zero));
10391   size(4);
10392   format %{ "TST   $op2,$op1<<$op3" %}
10393 
10394   ins_encode %{
10395     __ tst($op1$$Register, AsmOperand($op2$$Register, asr, $op3$$Register));
10396   %}
10397   ins_pipe(ialu_cconly_reg_reg_zero);
10398 %}
10399 #endif
10400 
10401 instruct testsarI_reg_reg_imm( flagsReg_EQNELTGE icc, iRegI op1, iRegI op2, immU5 op3, immI0 zero ) %{
10402   match(Set icc (CmpI (AndI op1 (RShiftI op2 op3)) zero));
10403   size(4);
10404   format %{ "tst_32 $op2,$op1<<$op3" %}
10405 
10406   ins_encode %{
10407     __ tst_32($op1$$Register, AsmOperand($op2$$Register, asr, $op3$$constant));
10408   %}
10409   ins_pipe(ialu_cconly_reg_reg_zero);
10410 %}
10411 
10412 #ifndef AARCH64
10413 instruct testshrI_reg_reg_reg( flagsReg_EQNELTGE icc, iRegI op1, iRegI op2, iRegI op3, immI0 zero ) %{
10414   match(Set icc (CmpI (AndI op1 (URShiftI op2 op3)) zero));
10415   size(4);
10416   format %{ "TST   $op2,$op1<<$op3" %}
10417 
10418   ins_encode %{
10419     __ tst($op1$$Register, AsmOperand($op2$$Register, lsr, $op3$$Register));
10420   %}
10421   ins_pipe(ialu_cconly_reg_reg_zero);
10422 %}
10423 #endif
10424 
10425 instruct testshrI_reg_reg_imm( flagsReg_EQNELTGE icc, iRegI op1, iRegI op2, immU5 op3, immI0 zero ) %{
10426   match(Set icc (CmpI (AndI op1 (URShiftI op2 op3)) zero));
10427   size(4);
10428   format %{ "tst_32 $op2,$op1<<$op3" %}
10429 
10430   ins_encode %{
10431     __ tst_32($op1$$Register, AsmOperand($op2$$Register, lsr, $op3$$constant));
10432   %}
10433   ins_pipe(ialu_cconly_reg_reg_zero);
10434 %}
10435 
10436 instruct testI_reg_imm( flagsReg_EQNELTGE icc, iRegI op1, limmI op2, immI0 zero ) %{
10437   match(Set icc (CmpI (AndI op1 op2) zero));
10438   size(4);
10439   format %{ "tst_32 $op2,$op1" %}
10440 
10441   ins_encode %{
10442     __ tst_32($op1$$Register, $op2$$constant);
10443   %}
10444   ins_pipe(ialu_cconly_reg_imm_zero);
10445 %}
10446 
10447 #ifdef AARCH64
10448 instruct compL_reg_reg(flagsReg xcc, iRegL op1, iRegL op2)
10449 %{
10450   match(Set xcc (CmpL op1 op2));
10451   effect( DEF xcc, USE op1, USE op2 );
10452 
10453   size(4);
10454   format %{ "CMP     $op1,$op2\t! long" %}
10455   ins_encode %{
10456     __ cmp($op1$$Register, $op2$$Register);
10457   %}
10458   ins_pipe(ialu_cconly_reg_reg);
10459 %}
10460 #else
10461 instruct compL_reg_reg_LTGE(flagsRegL_LTGE xcc, iRegL op1, iRegL op2, iRegL tmp) %{
10462   match(Set xcc (CmpL op1 op2));
10463   effect( DEF xcc, USE op1, USE op2, TEMP tmp );
10464 
10465   size(8);
10466   format %{ "SUBS    $tmp,$op1.low,$op2.low\t\t! long\n\t"
10467             "SBCS    $tmp,$op1.hi,$op2.hi" %}
10468   ins_encode %{
10469     __ subs($tmp$$Register, $op1$$Register, $op2$$Register);
10470     __ sbcs($tmp$$Register->successor(), $op1$$Register->successor(), $op2$$Register->successor());
10471   %}
10472   ins_pipe(ialu_cconly_reg_reg);
10473 %}
10474 #endif
10475 
10476 #ifdef AARCH64
10477 instruct compL_reg_con(flagsReg xcc, iRegL op1, aimmL con) %{
10478   match(Set xcc (CmpL op1 con));
10479   effect( DEF xcc, USE op1, USE con );
10480 
10481   size(8);
10482   format %{ "CMP     $op1,$con\t\t! long"  %}
10483   ins_encode %{
10484     __ cmp($op1$$Register, $con$$constant);
10485   %}
10486 
10487   ins_pipe(ialu_cconly_reg_imm);
10488 %}
10489 #else
10490 instruct compL_reg_reg_EQNE(flagsRegL_EQNE xcc, iRegL op1, iRegL op2) %{
10491   match(Set xcc (CmpL op1 op2));
10492   effect( DEF xcc, USE op1, USE op2 );
10493 
10494   size(8);
10495   format %{ "TEQ    $op1.hi,$op2.hi\t\t! long\n\t"
10496             "TEQ.eq $op1.lo,$op2.lo" %}
10497   ins_encode %{
10498     __ teq($op1$$Register->successor(), $op2$$Register->successor());
10499     __ teq($op1$$Register, $op2$$Register, eq);
10500   %}
10501   ins_pipe(ialu_cconly_reg_reg);
10502 %}
10503 
10504 instruct compL_reg_reg_LEGT(flagsRegL_LEGT xcc, iRegL op1, iRegL op2, iRegL tmp) %{
10505   match(Set xcc (CmpL op1 op2));
10506   effect( DEF xcc, USE op1, USE op2, TEMP tmp );
10507 
10508   size(8);
10509   format %{ "SUBS    $tmp,$op2.low,$op1.low\t\t! long\n\t"
10510             "SBCS    $tmp,$op2.hi,$op1.hi" %}
10511   ins_encode %{
10512     __ subs($tmp$$Register, $op2$$Register, $op1$$Register);
10513     __ sbcs($tmp$$Register->successor(), $op2$$Register->successor(), $op1$$Register->successor());
10514   %}
10515   ins_pipe(ialu_cconly_reg_reg);
10516 %}
10517 
10518 // TODO: try immLRot2 instead, (0, $con$$constant) becomes
10519 // (hi($con$$constant), lo($con$$constant)) becomes
10520 instruct compL_reg_con_LTGE(flagsRegL_LTGE xcc, iRegL op1, immLlowRot con, iRegL tmp) %{
10521   match(Set xcc (CmpL op1 con));
10522   effect( DEF xcc, USE op1, USE con, TEMP tmp );
10523 
10524   size(8);
10525   format %{ "SUBS    $tmp,$op1.low,$con\t\t! long\n\t"
10526             "SBCS    $tmp,$op1.hi,0" %}
10527   ins_encode %{
10528     __ subs($tmp$$Register, $op1$$Register, $con$$constant);
10529     __ sbcs($tmp$$Register->successor(), $op1$$Register->successor(), 0);
10530   %}
10531 
10532   ins_pipe(ialu_cconly_reg_reg);
10533 %}
10534 
10535 // TODO: try immLRot2 instead, (0, $con$$constant) becomes
10536 // (hi($con$$constant), lo($con$$constant)) becomes
10537 instruct compL_reg_con_EQNE(flagsRegL_EQNE xcc, iRegL op1, immLlowRot con) %{
10538   match(Set xcc (CmpL op1 con));
10539   effect( DEF xcc, USE op1, USE con );
10540 
10541   size(8);
10542   format %{ "TEQ    $op1.hi,0\t\t! long\n\t"
10543             "TEQ.eq $op1.lo,$con" %}
10544   ins_encode %{
10545     __ teq($op1$$Register->successor(), 0);
10546     __ teq($op1$$Register, $con$$constant, eq);
10547   %}
10548 
10549   ins_pipe(ialu_cconly_reg_reg);
10550 %}
10551 
10552 // TODO: try immLRot2 instead, (0, $con$$constant) becomes
10553 // (hi($con$$constant), lo($con$$constant)) becomes
10554 instruct compL_reg_con_LEGT(flagsRegL_LEGT xcc, iRegL op1, immLlowRot con, iRegL tmp) %{
10555   match(Set xcc (CmpL op1 con));
10556   effect( DEF xcc, USE op1, USE con, TEMP tmp );
10557 
10558   size(8);
10559   format %{ "RSBS    $tmp,$op1.low,$con\t\t! long\n\t"
10560             "RSCS    $tmp,$op1.hi,0" %}
10561   ins_encode %{
10562     __ rsbs($tmp$$Register, $op1$$Register, $con$$constant);
10563     __ rscs($tmp$$Register->successor(), $op1$$Register->successor(), 0);
10564   %}
10565 
10566   ins_pipe(ialu_cconly_reg_reg);
10567 %}
10568 #endif
10569 
10570 /* instruct testL_reg_reg(flagsRegL xcc, iRegL op1, iRegL op2, immL0 zero) %{ */
10571 /*   match(Set xcc (CmpL (AndL op1 op2) zero)); */
10572 /*   ins_encode %{ */
10573 /*     __ stop("testL_reg_reg unimplemented"); */
10574 /*   %} */
10575 /*   ins_pipe(ialu_cconly_reg_reg); */
10576 /* %} */
10577 
10578 /* // useful for checking the alignment of a pointer: */
10579 /* instruct testL_reg_con(flagsRegL xcc, iRegL op1, immLlowRot con, immL0 zero) %{ */
10580 /*   match(Set xcc (CmpL (AndL op1 con) zero)); */
10581 /*   ins_encode %{ */
10582 /*     __ stop("testL_reg_con unimplemented"); */
10583 /*   %} */
10584 /*   ins_pipe(ialu_cconly_reg_reg); */
10585 /* %} */
10586 
10587 instruct compU_iReg_imm(flagsRegU icc, iRegI op1, aimmU31 op2 ) %{
10588   match(Set icc (CmpU op1 op2));
10589 
10590   size(4);
10591   format %{ "cmp_32 $op1,$op2\t! unsigned" %}
10592   ins_encode %{
10593     __ cmp_32($op1$$Register, $op2$$constant);
10594   %}
10595   ins_pipe(ialu_cconly_reg_imm);
10596 %}
10597 
10598 // Compare Pointers
10599 instruct compP_iRegP(flagsRegP pcc, iRegP op1, iRegP op2 ) %{
10600   match(Set pcc (CmpP op1 op2));
10601 
10602   size(4);
10603   format %{ "CMP    $op1,$op2\t! ptr" %}
10604   ins_encode %{
10605     __ cmp($op1$$Register, $op2$$Register);
10606   %}
10607   ins_pipe(ialu_cconly_reg_reg);
10608 %}
10609 
10610 instruct compP_iRegP_imm(flagsRegP pcc, iRegP op1, aimmP op2 ) %{
10611   match(Set pcc (CmpP op1 op2));
10612 
10613   size(4);
10614   format %{ "CMP    $op1,$op2\t! ptr" %}
10615   ins_encode %{
10616     assert($op2$$constant == 0 || _opnds[2]->constant_reloc() == relocInfo::none, "reloc in cmp?");
10617     __ cmp($op1$$Register, $op2$$constant);
10618   %}
10619   ins_pipe(ialu_cconly_reg_imm);
10620 %}
10621 
10622 //----------Max and Min--------------------------------------------------------
10623 // Min Instructions
10624 // Conditional move for min
10625 instruct cmovI_reg_lt( iRegI op2, iRegI op1, flagsReg icc ) %{
10626   effect( USE_DEF op2, USE op1, USE icc );
10627 
10628   size(4);
10629   format %{ "MOV.lt  $op2,$op1\t! min" %}
10630   ins_encode %{
10631     __ mov($op2$$Register, $op1$$Register, lt);
10632   %}
10633   ins_pipe(ialu_reg_flags);
10634 %}
10635 
10636 // Min Register with Register.
10637 instruct minI_eReg(iRegI op1, iRegI op2) %{
10638   match(Set op2 (MinI op1 op2));
10639   ins_cost(DEFAULT_COST*2);
10640   expand %{
10641     flagsReg icc;
10642     compI_iReg(icc,op1,op2);
10643     cmovI_reg_lt(op2,op1,icc);
10644   %}
10645 %}
10646 
10647 // Max Instructions
10648 // Conditional move for max
10649 instruct cmovI_reg_gt( iRegI op2, iRegI op1, flagsReg icc ) %{
10650   effect( USE_DEF op2, USE op1, USE icc );
10651   format %{ "MOV.gt  $op2,$op1\t! max" %}
10652   ins_encode %{
10653     __ mov($op2$$Register, $op1$$Register, gt);
10654   %}
10655   ins_pipe(ialu_reg_flags);
10656 %}
10657 
10658 // Max Register with Register
10659 instruct maxI_eReg(iRegI op1, iRegI op2) %{
10660   match(Set op2 (MaxI op1 op2));
10661   ins_cost(DEFAULT_COST*2);
10662   expand %{
10663     flagsReg icc;
10664     compI_iReg(icc,op1,op2);
10665     cmovI_reg_gt(op2,op1,icc);
10666   %}
10667 %}
10668 
10669 
10670 //----------Float Compares----------------------------------------------------
10671 // Compare floating, generate condition code
10672 instruct cmpF_cc(flagsRegF fcc, flagsReg icc, regF src1, regF src2) %{
10673   match(Set icc (CmpF src1 src2));
10674   effect(KILL fcc);
10675 
10676 #ifdef AARCH64
10677   size(4);
10678   format %{ "FCMP_s  $src1,$src2" %}
10679   ins_encode %{
10680     __ fcmp_s($src1$$FloatRegister, $src2$$FloatRegister);
10681   %}
10682 #else
10683   size(8);
10684   format %{ "FCMPs  $src1,$src2\n\t"
10685             "FMSTAT" %}
10686   ins_encode %{
10687     __ fcmps($src1$$FloatRegister, $src2$$FloatRegister);
10688     __ fmstat();
10689   %}
10690 #endif
10691   ins_pipe(faddF_fcc_reg_reg_zero);
10692 %}
10693 
10694 instruct cmpF0_cc(flagsRegF fcc, flagsReg icc, regF src1, immF0 src2) %{
10695   match(Set icc (CmpF src1 src2));
10696   effect(KILL fcc);
10697 
10698 #ifdef AARCH64
10699   size(4);
10700   format %{ "FCMP0_s $src1" %}
10701   ins_encode %{
10702     __ fcmp0_s($src1$$FloatRegister);
10703   %}
10704 #else
10705   size(8);
10706   format %{ "FCMPs  $src1,$src2\n\t"
10707             "FMSTAT" %}
10708   ins_encode %{
10709     __ fcmpzs($src1$$FloatRegister);
10710     __ fmstat();
10711   %}
10712 #endif
10713   ins_pipe(faddF_fcc_reg_reg_zero);
10714 %}
10715 
10716 instruct cmpD_cc(flagsRegF fcc, flagsReg icc, regD src1, regD src2) %{
10717   match(Set icc (CmpD src1 src2));
10718   effect(KILL fcc);
10719 
10720 #ifdef AARCH64
10721   size(4);
10722   format %{ "FCMP_d $src1,$src2" %}
10723   ins_encode %{
10724     __ fcmp_d($src1$$FloatRegister, $src2$$FloatRegister);
10725   %}
10726 #else
10727   size(8);
10728   format %{ "FCMPd  $src1,$src2 \n\t"
10729             "FMSTAT" %}
10730   ins_encode %{
10731     __ fcmpd($src1$$FloatRegister, $src2$$FloatRegister);
10732     __ fmstat();
10733   %}
10734 #endif
10735   ins_pipe(faddD_fcc_reg_reg_zero);
10736 %}
10737 
10738 instruct cmpD0_cc(flagsRegF fcc, flagsReg icc, regD src1, immD0 src2) %{
10739   match(Set icc (CmpD src1 src2));
10740   effect(KILL fcc);
10741 
10742 #ifdef AARCH64
10743   size(8);
10744   format %{ "FCMP0_d $src1" %}
10745   ins_encode %{
10746     __ fcmp0_d($src1$$FloatRegister);
10747   %}
10748 #else
10749   size(8);
10750   format %{ "FCMPZd  $src1,$src2 \n\t"
10751             "FMSTAT" %}
10752   ins_encode %{
10753     __ fcmpzd($src1$$FloatRegister);
10754     __ fmstat();
10755   %}
10756 #endif
10757   ins_pipe(faddD_fcc_reg_reg_zero);
10758 %}
10759 
10760 #ifdef AARCH64
10761 // Compare floating, generate -1,0,1
10762 instruct cmpF_reg(iRegI dst, regF src1, regF src2, flagsReg icc) %{
10763   match(Set dst (CmpF3 src1 src2));
10764   // effect(KILL fcc); // nobody cares if flagsRegF is killed
10765   effect(KILL icc);
10766   ins_cost(DEFAULT_COST*3); // FIXME
10767   size(12);
10768   format %{ "FCMP_s $src1,$src2\n\t"
10769             "CSET   $dst, gt\n\t"
10770             "CSINV  $dst, $dst, ZR, ge" %}
10771   ins_encode %{
10772     Register dst = $dst$$Register;
10773     __ fcmp_s($src1$$FloatRegister, $src2$$FloatRegister);
10774     __ cset(dst, gt);            // 1 if '>', else 0
10775     __ csinv(dst, dst, ZR, ge);  // previous value if '>=', else -1
10776   %}
10777   ins_pipe( floating_cmp ); // FIXME
10778 %}
10779 
10780 // Compare floating, generate -1,0,1
10781 instruct cmpD_reg(iRegI dst, regD src1, regD src2, flagsReg icc) %{
10782   match(Set dst (CmpD3 src1 src2));
10783   // effect(KILL fcc); // nobody cares if flagsRegF is killed
10784   effect(KILL icc);
10785   ins_cost(DEFAULT_COST*3); // FIXME
10786   size(12);
10787   format %{ "FCMP_d $src1,$src2\n\t"
10788             "CSET   $dst, gt\n\t"
10789             "CSINV  $dst, $dst, ZR, ge" %}
10790   ins_encode %{
10791     Register dst = $dst$$Register;
10792     __ fcmp_d($src1$$FloatRegister, $src2$$FloatRegister);
10793     __ cset(dst, gt);            // 1 if '>', else 0
10794     __ csinv(dst, dst, ZR, ge);  // previous value if '>=', else -1
10795   %}
10796   ins_pipe( floating_cmp ); // FIXME
10797 %}
10798 
10799 // Compare floating, generate -1,0,1
10800 instruct cmpF0_reg(iRegI dst, regF src1, immF0 src2, flagsReg icc) %{
10801   match(Set dst (CmpF3 src1 src2));
10802   // effect(KILL fcc); // nobody cares if flagsRegF is killed
10803   effect(KILL icc);
10804   ins_cost(DEFAULT_COST*3); // FIXME
10805   size(12);
10806   format %{ "FCMP0_s $src1\n\t"
10807             "CSET   $dst, gt\n\t"
10808             "CSINV  $dst, $dst, ZR, ge" %}
10809   ins_encode %{
10810     Register dst = $dst$$Register;
10811     __ fcmp0_s($src1$$FloatRegister);
10812     __ cset(dst, gt);            // 1 if '>', else 0
10813     __ csinv(dst, dst, ZR, ge);  // previous value if '>=', else -1
10814   %}
10815   ins_pipe( floating_cmp ); // FIXME
10816 %}
10817 
10818 // Compare floating, generate -1,0,1
10819 instruct cmpD0_reg(iRegI dst, regD src1, immD0 src2, flagsReg icc) %{
10820   match(Set dst (CmpD3 src1 src2));
10821   // effect(KILL fcc); // nobody cares if flagsRegF is killed
10822   effect(KILL icc);
10823   ins_cost(DEFAULT_COST*3); // FIXME
10824   size(12);
10825   format %{ "FCMP0_d $src1\n\t"
10826             "CSET   $dst, gt\n\t"
10827             "CSINV  $dst, $dst, ZR, ge" %}
10828   ins_encode %{
10829     Register dst = $dst$$Register;
10830     __ fcmp0_d($src1$$FloatRegister);
10831     __ cset(dst, gt);            // 1 if '>', else 0
10832     __ csinv(dst, dst, ZR, ge);  // previous value if '>=', else -1
10833   %}
10834   ins_pipe( floating_cmp ); // FIXME
10835 %}
10836 #else
10837 // Compare floating, generate -1,0,1
10838 instruct cmpF_reg(iRegI dst, regF src1, regF src2, flagsRegF fcc) %{
10839   match(Set dst (CmpF3 src1 src2));
10840   effect(KILL fcc);
10841   ins_cost(DEFAULT_COST*3+BRANCH_COST*3); // FIXME
10842   size(20);
10843   // same number of instructions as code using conditional moves but
10844   // doesn't kill integer condition register
10845   format %{ "FCMPs  $dst,$src1,$src2 \n\t"
10846             "VMRS   $dst, FPSCR \n\t"
10847             "OR     $dst, $dst, 0x08000000 \n\t"
10848             "EOR    $dst, $dst, $dst << 3 \n\t"
10849             "MOV    $dst, $dst >> 30" %}
10850   ins_encode %{
10851     __ fcmps($src1$$FloatRegister, $src2$$FloatRegister);
10852     __ floating_cmp($dst$$Register);
10853   %}
10854   ins_pipe( floating_cmp );
10855 %}
10856 
10857 instruct cmpF0_reg(iRegI dst, regF src1, immF0 src2, flagsRegF fcc) %{
10858   match(Set dst (CmpF3 src1 src2));
10859   effect(KILL fcc);
10860   ins_cost(DEFAULT_COST*3+BRANCH_COST*3); // FIXME
10861   size(20);
10862   // same number of instructions as code using conditional moves but
10863   // doesn't kill integer condition register
10864   format %{ "FCMPZs $dst,$src1,$src2 \n\t"
10865             "VMRS   $dst, FPSCR \n\t"
10866             "OR     $dst, $dst, 0x08000000 \n\t"
10867             "EOR    $dst, $dst, $dst << 3 \n\t"
10868             "MOV    $dst, $dst >> 30" %}
10869   ins_encode %{
10870     __ fcmpzs($src1$$FloatRegister);
10871     __ floating_cmp($dst$$Register);
10872   %}
10873   ins_pipe( floating_cmp );
10874 %}
10875 
10876 instruct cmpD_reg(iRegI dst, regD src1, regD src2, flagsRegF fcc) %{
10877   match(Set dst (CmpD3 src1 src2));
10878   effect(KILL fcc);
10879   ins_cost(DEFAULT_COST*3+BRANCH_COST*3); // FIXME
10880   size(20);
10881   // same number of instructions as code using conditional moves but
10882   // doesn't kill integer condition register
10883   format %{ "FCMPd  $dst,$src1,$src2 \n\t"
10884             "VMRS   $dst, FPSCR \n\t"
10885             "OR     $dst, $dst, 0x08000000 \n\t"
10886             "EOR    $dst, $dst, $dst << 3 \n\t"
10887             "MOV    $dst, $dst >> 30" %}
10888   ins_encode %{
10889     __ fcmpd($src1$$FloatRegister, $src2$$FloatRegister);
10890     __ floating_cmp($dst$$Register);
10891   %}
10892   ins_pipe( floating_cmp );
10893 %}
10894 
10895 instruct cmpD0_reg(iRegI dst, regD src1, immD0 src2, flagsRegF fcc) %{
10896   match(Set dst (CmpD3 src1 src2));
10897   effect(KILL fcc);
10898   ins_cost(DEFAULT_COST*3+BRANCH_COST*3); // FIXME
10899   size(20);
10900   // same number of instructions as code using conditional moves but
10901   // doesn't kill integer condition register
10902   format %{ "FCMPZd $dst,$src1,$src2 \n\t"
10903             "VMRS   $dst, FPSCR \n\t"
10904             "OR     $dst, $dst, 0x08000000 \n\t"
10905             "EOR    $dst, $dst, $dst << 3 \n\t"
10906             "MOV    $dst, $dst >> 30" %}
10907   ins_encode %{
10908     __ fcmpzd($src1$$FloatRegister);
10909     __ floating_cmp($dst$$Register);
10910   %}
10911   ins_pipe( floating_cmp );
10912 %}
10913 #endif // !AARCH64
10914 
10915 //----------Branches---------------------------------------------------------
10916 // Jump
10917 // (compare 'operand indIndex' and 'instruct addP_reg_reg' above)
10918 // FIXME
10919 instruct jumpXtnd(iRegX switch_val, iRegP tmp) %{
10920   match(Jump switch_val);
10921   effect(TEMP tmp);
10922   ins_cost(350);
10923   format %{  "ADD    $tmp, $constanttablebase, $switch_val\n\t"
10924              "LDR    $tmp,[$tmp + $constantoffset]\n\t"
10925              "BX     $tmp" %}
10926   size(20);
10927   ins_encode %{
10928     Register table_reg;
10929     Register label_reg = $tmp$$Register;
10930     if (constant_offset() == 0) {
10931       table_reg = $constanttablebase;
10932       __ ldr(label_reg, Address(table_reg, $switch_val$$Register));
10933     } else {
10934       table_reg = $tmp$$Register;
10935       int offset = $constantoffset;
10936       if (is_memoryP(offset)) {
10937         __ add(table_reg, $constanttablebase, $switch_val$$Register);
10938         __ ldr(label_reg, Address(table_reg, offset));
10939       } else {
10940         __ mov_slow(table_reg, $constantoffset);
10941         __ add(table_reg, $constanttablebase, table_reg);
10942         __ ldr(label_reg, Address(table_reg, $switch_val$$Register));
10943       }
10944     }
10945     __ jump(label_reg); // ldr + b better than ldr to PC for branch predictor?
10946     //    __ ldr(PC, Address($table$$Register, $switch_val$$Register));
10947   %}
10948   ins_pipe(ialu_reg_reg);
10949 %}
10950 
10951 // // Direct Branch.
10952 instruct branch(label labl) %{
10953   match(Goto);
10954   effect(USE labl);
10955 
10956   size(4);
10957   ins_cost(BRANCH_COST);
10958   format %{ "B     $labl" %}
10959   ins_encode %{
10960     __ b(*($labl$$label));
10961   %}
10962   ins_pipe(br);
10963 %}
10964 
10965 // Conditional Direct Branch
10966 instruct branchCon(cmpOp cmp, flagsReg icc, label labl) %{
10967   match(If cmp icc);
10968   effect(USE labl);
10969 
10970   size(4);
10971   ins_cost(BRANCH_COST);
10972   format %{ "B$cmp   $icc,$labl" %}
10973   ins_encode %{
10974     __ b(*($labl$$label), (AsmCondition)($cmp$$cmpcode));
10975   %}
10976   ins_pipe(br_cc);
10977 %}
10978 
10979 #ifdef ARM
10980 instruct branchCon_EQNELTGE(cmpOp0 cmp, flagsReg_EQNELTGE icc, label labl) %{
10981   match(If cmp icc);
10982   effect(USE labl);
10983   predicate( _kids[0]->_leaf->as_Bool()->_test._test == BoolTest::eq || _kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ne || _kids[0]->_leaf->as_Bool()->_test._test == BoolTest::lt || _kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ge);
10984 
10985   size(4);
10986   ins_cost(BRANCH_COST);
10987   format %{ "B$cmp   $icc,$labl" %}
10988   ins_encode %{
10989     __ b(*($labl$$label), (AsmCondition)($cmp$$cmpcode));
10990   %}
10991   ins_pipe(br_cc);
10992 %}
10993 #endif
10994 
10995 #ifdef AARCH64
10996 instruct cbzI(cmpOp cmp, iRegI op1, immI0 op2, label labl) %{
10997   match(If cmp (CmpI op1 op2));
10998   effect(USE labl);
10999   predicate(_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::eq ||
11000             _kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ne);
11001   size(4);
11002   ins_cost(BRANCH_COST);
11003   format %{ "CB{N}Z $op1, $labl\t! int $cmp" %}
11004   ins_encode %{
11005     if ($cmp$$cmpcode == eq) {
11006       __ cbz_w($op1$$Register, *($labl$$label));
11007     } else {
11008       __ cbnz_w($op1$$Register, *($labl$$label));
11009     }
11010   %}
11011   ins_pipe(br_cc); // FIXME
11012 %}
11013 
11014 instruct cbzP(cmpOpP cmp, iRegP op1, immP0 op2, label labl) %{
11015   match(If cmp (CmpP op1 op2));
11016   effect(USE labl);
11017   predicate(_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::eq ||
11018             _kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ne);
11019   size(4);
11020   ins_cost(BRANCH_COST);
11021   format %{ "CB{N}Z $op1, $labl\t! ptr $cmp" %}
11022   ins_encode %{
11023     if ($cmp$$cmpcode == eq) {
11024       __ cbz($op1$$Register, *($labl$$label));
11025     } else {
11026       __ cbnz($op1$$Register, *($labl$$label));
11027     }
11028   %}
11029   ins_pipe(br_cc); // FIXME
11030 %}
11031 
11032 instruct cbzL(cmpOpL cmp, iRegL op1, immL0 op2, label labl) %{
11033   match(If cmp (CmpL op1 op2));
11034   effect(USE labl);
11035   predicate(_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::eq ||
11036             _kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ne);
11037   size(4);
11038   ins_cost(BRANCH_COST);
11039   format %{ "CB{N}Z $op1, $labl\t! long $cmp" %}
11040   ins_encode %{
11041     if ($cmp$$cmpcode == eq) {
11042       __ cbz($op1$$Register, *($labl$$label));
11043     } else {
11044       __ cbnz($op1$$Register, *($labl$$label));
11045     }
11046   %}
11047   ins_pipe(br_cc); // FIXME
11048 %}
11049 #endif
11050 
11051 instruct branchConU(cmpOpU cmp, flagsRegU icc, label labl) %{
11052   match(If cmp icc);
11053   effect(USE labl);
11054 
11055   size(4);
11056   ins_cost(BRANCH_COST);
11057   format %{ "B$cmp  $icc,$labl" %}
11058   ins_encode %{
11059     __ b(*($labl$$label), (AsmCondition)($cmp$$cmpcode));
11060   %}
11061   ins_pipe(br_cc);
11062 %}
11063 
11064 instruct branchConP(cmpOpP cmp, flagsRegP pcc, label labl) %{
11065   match(If cmp pcc);
11066   effect(USE labl);
11067 
11068   size(4);
11069   ins_cost(BRANCH_COST);
11070   format %{ "B$cmp  $pcc,$labl" %}
11071   ins_encode %{
11072     __ b(*($labl$$label), (AsmCondition)($cmp$$cmpcode));
11073   %}
11074   ins_pipe(br_cc);
11075 %}
11076 
11077 #ifndef AARCH64
11078 instruct branchConL_LTGE(cmpOpL cmp, flagsRegL_LTGE xcc, label labl) %{
11079   match(If cmp xcc);
11080   effect(USE labl);
11081   predicate( _kids[0]->_leaf->as_Bool()->_test._test == BoolTest::lt || _kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ge );
11082 
11083   size(4);
11084   ins_cost(BRANCH_COST);
11085   format %{ "B$cmp  $xcc,$labl" %}
11086   ins_encode %{
11087     __ b(*($labl$$label), (AsmCondition)($cmp$$cmpcode));
11088   %}
11089   ins_pipe(br_cc);
11090 %}
11091 
11092 instruct branchConL_EQNE(cmpOpL cmp, flagsRegL_EQNE xcc, label labl) %{
11093   match(If cmp xcc);
11094   effect(USE labl);
11095   predicate( _kids[0]->_leaf->as_Bool()->_test._test == BoolTest::eq || _kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ne );
11096 
11097   size(4);
11098   ins_cost(BRANCH_COST);
11099   format %{ "B$cmp  $xcc,$labl" %}
11100   ins_encode %{
11101     __ b(*($labl$$label), (AsmCondition)($cmp$$cmpcode));
11102   %}
11103   ins_pipe(br_cc);
11104 %}
11105 
11106 instruct branchConL_LEGT(cmpOpL_commute cmp, flagsRegL_LEGT xcc, label labl) %{
11107   match(If cmp xcc);
11108   effect(USE labl);
11109   predicate( _kids[0]->_leaf->as_Bool()->_test._test == BoolTest::gt || _kids[0]->_leaf->as_Bool()->_test._test == BoolTest::le );
11110 
11111   size(4);
11112   ins_cost(BRANCH_COST);
11113   format %{ "B$cmp  $xcc,$labl" %}
11114   ins_encode %{
11115     __ b(*($labl$$label), (AsmCondition)($cmp$$cmpcode));
11116   %}
11117   ins_pipe(br_cc);
11118 %}
11119 #endif
11120 
11121 instruct branchLoopEnd(cmpOp cmp, flagsReg icc, label labl) %{
11122   match(CountedLoopEnd cmp icc);
11123   effect(USE labl);
11124 
11125   size(4);
11126   ins_cost(BRANCH_COST);
11127   format %{ "B$cmp   $icc,$labl\t! Loop end" %}
11128   ins_encode %{
11129     __ b(*($labl$$label), (AsmCondition)($cmp$$cmpcode));
11130   %}
11131   ins_pipe(br_cc);
11132 %}
11133 
11134 // instruct branchLoopEndU(cmpOpU cmp, flagsRegU icc, label labl) %{
11135 //   match(CountedLoopEnd cmp icc);
11136 //   ins_pipe(br_cc);
11137 // %}
11138 
11139 // ============================================================================
11140 // Long Compare
11141 //
11142 // Currently we hold longs in 2 registers.  Comparing such values efficiently
11143 // is tricky.  The flavor of compare used depends on whether we are testing
11144 // for LT, LE, or EQ.  For a simple LT test we can check just the sign bit.
11145 // The GE test is the negated LT test.  The LE test can be had by commuting
11146 // the operands (yielding a GE test) and then negating; negate again for the
11147 // GT test.  The EQ test is done by ORcc'ing the high and low halves, and the
11148 // NE test is negated from that.
11149 
11150 // Due to a shortcoming in the ADLC, it mixes up expressions like:
11151 // (foo (CmpI (CmpL X Y) 0)) and (bar (CmpI (CmpL X 0L) 0)).  Note the
11152 // difference between 'Y' and '0L'.  The tree-matches for the CmpI sections
11153 // are collapsed internally in the ADLC's dfa-gen code.  The match for
11154 // (CmpI (CmpL X Y) 0) is silently replaced with (CmpI (CmpL X 0L) 0) and the
11155 // foo match ends up with the wrong leaf.  One fix is to not match both
11156 // reg-reg and reg-zero forms of long-compare.  This is unfortunate because
11157 // both forms beat the trinary form of long-compare and both are very useful
11158 // on Intel which has so few registers.
11159 
11160 // instruct branchCon_long(cmpOp cmp, flagsRegL xcc, label labl) %{
11161 //   match(If cmp xcc);
11162 //   ins_pipe(br_cc);
11163 // %}
11164 
11165 // Manifest a CmpL3 result in an integer register.  Very painful.
11166 // This is the test to avoid.
11167 #ifdef AARCH64
11168 instruct cmpL3_reg_reg(iRegI dst, iRegL src1, iRegL src2, flagsReg ccr) %{
11169   match(Set dst (CmpL3 src1 src2));
11170   // effect(KILL fcc); // nobody cares if flagsRegF is killed
11171   effect(KILL ccr);
11172   ins_cost(DEFAULT_COST*3); // FIXME
11173   size(12);
11174   format %{ "CMP    $src1,$src2\n\t"
11175             "CSET   $dst, gt\n\t"
11176             "CSINV  $dst, $dst, ZR, ge" %}
11177   ins_encode %{
11178     Register dst = $dst$$Register;
11179     __ cmp($src1$$Register, $src2$$Register);
11180     __ cset(dst, gt);            // 1 if '>', else 0
11181     __ csinv(dst, dst, ZR, ge);  // previous value if '>=', else -1
11182   %}
11183   ins_pipe( ialu_cconly_reg_reg ); // FIXME
11184 %}
11185 // TODO cmpL3_reg_imm
11186 #else
11187 instruct cmpL3_reg_reg(iRegI dst, iRegL src1, iRegL src2, flagsReg ccr ) %{
11188   match(Set dst (CmpL3 src1 src2) );
11189   effect( KILL ccr );
11190   ins_cost(6*DEFAULT_COST); // FIXME
11191   size(32);
11192   format %{
11193       "CMP    $src1.hi, $src2.hi\t\t! long\n"
11194     "\tMOV.gt $dst, 1\n"
11195     "\tmvn.lt $dst, 0\n"
11196     "\tB.ne   done\n"
11197     "\tSUBS   $dst, $src1.lo, $src2.lo\n"
11198     "\tMOV.hi $dst, 1\n"
11199     "\tmvn.lo $dst, 0\n"
11200     "done:"     %}
11201   ins_encode %{
11202     Label done;
11203     __ cmp($src1$$Register->successor(), $src2$$Register->successor());
11204     __ mov($dst$$Register, 1, gt);
11205     __ mvn($dst$$Register, 0, lt);
11206     __ b(done, ne);
11207     __ subs($dst$$Register, $src1$$Register, $src2$$Register);
11208     __ mov($dst$$Register, 1, hi);
11209     __ mvn($dst$$Register, 0, lo);
11210     __ bind(done);
11211   %}
11212   ins_pipe(cmpL_reg);
11213 %}
11214 #endif
11215 
11216 #ifndef AARCH64
11217 // Conditional move
11218 instruct cmovLL_reg_LTGE(cmpOpL cmp, flagsRegL_LTGE xcc, iRegL dst, iRegL src) %{
11219   match(Set dst (CMoveL (Binary cmp xcc) (Binary dst src)));
11220   predicate(_kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::lt || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ge );
11221 
11222   ins_cost(150);
11223   size(8);
11224   format %{ "MOV$cmp  $dst.lo,$src.lo\t! long\n\t"
11225             "MOV$cmp  $dst,$src.hi" %}
11226   ins_encode %{
11227     __ mov($dst$$Register, $src$$Register, (AsmCondition)($cmp$$cmpcode));
11228     __ mov($dst$$Register->successor(), $src$$Register->successor(), (AsmCondition)($cmp$$cmpcode));
11229   %}
11230   ins_pipe(ialu_reg);
11231 %}
11232 
11233 instruct cmovLL_reg_EQNE(cmpOpL cmp, flagsRegL_EQNE xcc, iRegL dst, iRegL src) %{
11234   match(Set dst (CMoveL (Binary cmp xcc) (Binary dst src)));
11235   predicate(_kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::eq || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ne );
11236 
11237   ins_cost(150);
11238   size(8);
11239   format %{ "MOV$cmp  $dst.lo,$src.lo\t! long\n\t"
11240             "MOV$cmp  $dst,$src.hi" %}
11241   ins_encode %{
11242     __ mov($dst$$Register, $src$$Register, (AsmCondition)($cmp$$cmpcode));
11243     __ mov($dst$$Register->successor(), $src$$Register->successor(), (AsmCondition)($cmp$$cmpcode));
11244   %}
11245   ins_pipe(ialu_reg);
11246 %}
11247 
11248 instruct cmovLL_reg_LEGT(cmpOpL_commute cmp, flagsRegL_LEGT xcc, iRegL dst, iRegL src) %{
11249   match(Set dst (CMoveL (Binary cmp xcc) (Binary dst src)));
11250   predicate(_kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::le || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::gt );
11251 
11252   ins_cost(150);
11253   size(8);
11254   format %{ "MOV$cmp  $dst.lo,$src.lo\t! long\n\t"
11255             "MOV$cmp  $dst,$src.hi" %}
11256   ins_encode %{
11257     __ mov($dst$$Register, $src$$Register, (AsmCondition)($cmp$$cmpcode));
11258     __ mov($dst$$Register->successor(), $src$$Register->successor(), (AsmCondition)($cmp$$cmpcode));
11259   %}
11260   ins_pipe(ialu_reg);
11261 %}
11262 
11263 instruct cmovLL_imm_LTGE(cmpOpL cmp, flagsRegL_LTGE xcc, iRegL dst, immL0 src) %{
11264   match(Set dst (CMoveL (Binary cmp xcc) (Binary dst src)));
11265   predicate(_kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::lt || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ge );
11266   ins_cost(140);
11267   size(8);
11268   format %{ "MOV$cmp  $dst.lo,0\t! long\n\t"
11269             "MOV$cmp  $dst,0" %}
11270   ins_encode %{
11271     __ mov($dst$$Register, 0, (AsmCondition)($cmp$$cmpcode));
11272     __ mov($dst$$Register->successor(), 0, (AsmCondition)($cmp$$cmpcode));
11273   %}
11274   ins_pipe(ialu_imm);
11275 %}
11276 
11277 instruct cmovLL_imm_EQNE(cmpOpL cmp, flagsRegL_EQNE xcc, iRegL dst, immL0 src) %{
11278   match(Set dst (CMoveL (Binary cmp xcc) (Binary dst src)));
11279   predicate(_kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::eq || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ne );
11280   ins_cost(140);
11281   size(8);
11282   format %{ "MOV$cmp  $dst.lo,0\t! long\n\t"
11283             "MOV$cmp  $dst,0" %}
11284   ins_encode %{
11285     __ mov($dst$$Register, 0, (AsmCondition)($cmp$$cmpcode));
11286     __ mov($dst$$Register->successor(), 0, (AsmCondition)($cmp$$cmpcode));
11287   %}
11288   ins_pipe(ialu_imm);
11289 %}
11290 
11291 instruct cmovLL_imm_LEGT(cmpOpL_commute cmp, flagsRegL_LEGT xcc, iRegL dst, immL0 src) %{
11292   match(Set dst (CMoveL (Binary cmp xcc) (Binary dst src)));
11293   predicate(_kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::le || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::gt );
11294   ins_cost(140);
11295   size(8);
11296   format %{ "MOV$cmp  $dst.lo,0\t! long\n\t"
11297             "MOV$cmp  $dst,0" %}
11298   ins_encode %{
11299     __ mov($dst$$Register, 0, (AsmCondition)($cmp$$cmpcode));
11300     __ mov($dst$$Register->successor(), 0, (AsmCondition)($cmp$$cmpcode));
11301   %}
11302   ins_pipe(ialu_imm);
11303 %}
11304 #endif // !AARCH64
11305 
11306 #ifndef AARCH64
11307 instruct cmovIL_reg_LTGE(cmpOpL cmp, flagsRegL_LTGE xcc, iRegI dst, iRegI src) %{
11308   match(Set dst (CMoveI (Binary cmp xcc) (Binary dst src)));
11309   predicate(_kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::lt || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ge );
11310 
11311   ins_cost(150);
11312   size(4);
11313   format %{ "MOV$cmp  $dst,$src" %}
11314   ins_encode %{
11315     __ mov($dst$$Register, $src$$Register, (AsmCondition)($cmp$$cmpcode));
11316   %}
11317   ins_pipe(ialu_reg);
11318 %}
11319 
11320 instruct cmovIL_reg_EQNE(cmpOpL cmp, flagsRegL_EQNE xcc, iRegI dst, iRegI src) %{
11321   match(Set dst (CMoveI (Binary cmp xcc) (Binary dst src)));
11322   predicate(_kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::eq || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ne );
11323 
11324   ins_cost(150);
11325   size(4);
11326   format %{ "MOV$cmp  $dst,$src" %}
11327   ins_encode %{
11328     __ mov($dst$$Register, $src$$Register, (AsmCondition)($cmp$$cmpcode));
11329   %}
11330   ins_pipe(ialu_reg);
11331 %}
11332 
11333 instruct cmovIL_reg_LEGT(cmpOpL_commute cmp, flagsRegL_LEGT xcc, iRegI dst, iRegI src) %{
11334   match(Set dst (CMoveI (Binary cmp xcc) (Binary dst src)));
11335   predicate(_kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::le || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::gt );
11336 
11337   ins_cost(150);
11338   size(4);
11339   format %{ "MOV$cmp  $dst,$src" %}
11340   ins_encode %{
11341     __ mov($dst$$Register, $src$$Register, (AsmCondition)($cmp$$cmpcode));
11342   %}
11343   ins_pipe(ialu_reg);
11344 %}
11345 #endif // !AARCH64
11346 
11347 #ifndef AARCH64
11348 instruct cmovIL_imm_LTGE(cmpOpL cmp, flagsRegL_LTGE xcc, iRegI dst, immI16 src) %{
11349   match(Set dst (CMoveI (Binary cmp xcc) (Binary dst src)));
11350   predicate(_kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::lt || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ge );
11351 
11352   ins_cost(140);
11353   format %{ "MOVW$cmp  $dst,$src" %}
11354   ins_encode %{
11355     __ movw($dst$$Register, $src$$constant, (AsmCondition)($cmp$$cmpcode));
11356   %}
11357   ins_pipe(ialu_imm);
11358 %}
11359 
11360 instruct cmovIL_imm_EQNE(cmpOpL cmp, flagsRegL_EQNE xcc, iRegI dst, immI16 src) %{
11361   match(Set dst (CMoveI (Binary cmp xcc) (Binary dst src)));
11362   predicate(_kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::eq || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ne );
11363 
11364   ins_cost(140);
11365   format %{ "MOVW$cmp  $dst,$src" %}
11366   ins_encode %{
11367     __ movw($dst$$Register, $src$$constant, (AsmCondition)($cmp$$cmpcode));
11368   %}
11369   ins_pipe(ialu_imm);
11370 %}
11371 
11372 instruct cmovIL_imm_LEGT(cmpOpL_commute cmp, flagsRegL_LEGT xcc, iRegI dst, immI16 src) %{
11373   match(Set dst (CMoveI (Binary cmp xcc) (Binary dst src)));
11374   predicate(_kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::le || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::gt );
11375 
11376   ins_cost(140);
11377   format %{ "MOVW$cmp  $dst,$src" %}
11378   ins_encode %{
11379     __ movw($dst$$Register, $src$$constant, (AsmCondition)($cmp$$cmpcode));
11380   %}
11381   ins_pipe(ialu_imm);
11382 %}
11383 
11384 instruct cmovPL_reg_LTGE(cmpOpL cmp, flagsRegL_LTGE xcc, iRegP dst, iRegP src) %{
11385   match(Set dst (CMoveP (Binary cmp xcc) (Binary dst src)));
11386   predicate(_kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::lt || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ge );
11387 
11388   ins_cost(150);
11389   size(4);
11390   format %{ "MOV$cmp  $dst,$src" %}
11391   ins_encode %{
11392     __ mov($dst$$Register, $src$$Register, (AsmCondition)($cmp$$cmpcode));
11393   %}
11394   ins_pipe(ialu_reg);
11395 %}
11396 
11397 instruct cmovPL_reg_EQNE(cmpOpL cmp, flagsRegL_EQNE xcc, iRegP dst, iRegP src) %{
11398   match(Set dst (CMoveP (Binary cmp xcc) (Binary dst src)));
11399   predicate(_kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::eq || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ne );
11400 
11401   ins_cost(150);
11402   size(4);
11403   format %{ "MOV$cmp  $dst,$src" %}
11404   ins_encode %{
11405     __ mov($dst$$Register, $src$$Register, (AsmCondition)($cmp$$cmpcode));
11406   %}
11407   ins_pipe(ialu_reg);
11408 %}
11409 
11410 instruct cmovPL_reg_LEGT(cmpOpL_commute cmp, flagsRegL_LEGT xcc, iRegP dst, iRegP src) %{
11411   match(Set dst (CMoveP (Binary cmp xcc) (Binary dst src)));
11412   predicate(_kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::le || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::gt );
11413 
11414   ins_cost(150);
11415   size(4);
11416   format %{ "MOV$cmp  $dst,$src" %}
11417   ins_encode %{
11418     __ mov($dst$$Register, $src$$Register, (AsmCondition)($cmp$$cmpcode));
11419   %}
11420   ins_pipe(ialu_reg);
11421 %}
11422 
11423 instruct cmovPL_imm_LTGE(cmpOpL cmp, flagsRegL_LTGE xcc, iRegP dst, immP0 src) %{
11424   match(Set dst (CMoveP (Binary cmp xcc) (Binary dst src)));
11425   predicate(_kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::lt || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ge );
11426 
11427   ins_cost(140);
11428   format %{ "MOVW$cmp  $dst,$src" %}
11429   ins_encode %{
11430     __ movw($dst$$Register, $src$$constant, (AsmCondition)($cmp$$cmpcode));
11431   %}
11432   ins_pipe(ialu_imm);
11433 %}
11434 
11435 instruct cmovPL_imm_EQNE(cmpOpL cmp, flagsRegL_EQNE xcc, iRegP dst, immP0 src) %{
11436   match(Set dst (CMoveP (Binary cmp xcc) (Binary dst src)));
11437   predicate(_kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::eq || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ne );
11438 
11439   ins_cost(140);
11440   format %{ "MOVW$cmp  $dst,$src" %}
11441   ins_encode %{
11442     __ movw($dst$$Register, $src$$constant, (AsmCondition)($cmp$$cmpcode));
11443   %}
11444   ins_pipe(ialu_imm);
11445 %}
11446 
11447 instruct cmovPL_imm_LEGT(cmpOpL_commute cmp, flagsRegL_LEGT xcc, iRegP dst, immP0 src) %{
11448   match(Set dst (CMoveP (Binary cmp xcc) (Binary dst src)));
11449   predicate(_kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::le || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::gt );
11450 
11451   ins_cost(140);
11452   format %{ "MOVW$cmp  $dst,$src" %}
11453   ins_encode %{
11454     __ movw($dst$$Register, $src$$constant, (AsmCondition)($cmp$$cmpcode));
11455   %}
11456   ins_pipe(ialu_imm);
11457 %}
11458 
11459 instruct cmovFL_reg_LTGE(cmpOpL cmp, flagsRegL_LTGE xcc, regF dst, regF src) %{
11460   match(Set dst (CMoveF (Binary cmp xcc) (Binary dst src)));
11461   predicate(_kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::lt || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ge );
11462   ins_cost(150);
11463   size(4);
11464   format %{ "FCPYS$cmp $dst,$src" %}
11465   ins_encode %{
11466     __ fcpys($dst$$FloatRegister, $src$$FloatRegister, (AsmCondition)($cmp$$cmpcode));
11467   %}
11468   ins_pipe(int_conditional_float_move);
11469 %}
11470 
11471 instruct cmovFL_reg_EQNE(cmpOpL cmp, flagsRegL_EQNE xcc, regF dst, regF src) %{
11472   match(Set dst (CMoveF (Binary cmp xcc) (Binary dst src)));
11473   predicate(_kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::eq || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ne );
11474   ins_cost(150);
11475   size(4);
11476   format %{ "FCPYS$cmp $dst,$src" %}
11477   ins_encode %{
11478     __ fcpys($dst$$FloatRegister, $src$$FloatRegister, (AsmCondition)($cmp$$cmpcode));
11479   %}
11480   ins_pipe(int_conditional_float_move);
11481 %}
11482 
11483 instruct cmovFL_reg_LEGT(cmpOpL_commute cmp, flagsRegL_LEGT xcc, regF dst, regF src) %{
11484   match(Set dst (CMoveF (Binary cmp xcc) (Binary dst src)));
11485   predicate(_kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::le || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::gt );
11486   ins_cost(150);
11487   size(4);
11488   format %{ "FCPYS$cmp $dst,$src" %}
11489   ins_encode %{
11490     __ fcpys($dst$$FloatRegister, $src$$FloatRegister, (AsmCondition)($cmp$$cmpcode));
11491   %}
11492   ins_pipe(int_conditional_float_move);
11493 %}
11494 
11495 instruct cmovDL_reg_LTGE(cmpOpL cmp, flagsRegL_LTGE xcc, regD dst, regD src) %{
11496   match(Set dst (CMoveD (Binary cmp xcc) (Binary dst src)));
11497   predicate(_kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::lt || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ge );
11498 
11499   ins_cost(150);
11500   size(4);
11501   format %{ "FCPYD$cmp $dst,$src" %}
11502   ins_encode %{
11503     __ fcpyd($dst$$FloatRegister, $src$$FloatRegister, (AsmCondition)($cmp$$cmpcode));
11504   %}
11505   ins_pipe(int_conditional_float_move);
11506 %}
11507 
11508 instruct cmovDL_reg_EQNE(cmpOpL cmp, flagsRegL_EQNE xcc, regD dst, regD src) %{
11509   match(Set dst (CMoveD (Binary cmp xcc) (Binary dst src)));
11510   predicate(_kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::eq || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ne );
11511 
11512   ins_cost(150);
11513   size(4);
11514   format %{ "FCPYD$cmp $dst,$src" %}
11515   ins_encode %{
11516     __ fcpyd($dst$$FloatRegister, $src$$FloatRegister, (AsmCondition)($cmp$$cmpcode));
11517   %}
11518   ins_pipe(int_conditional_float_move);
11519 %}
11520 
11521 instruct cmovDL_reg_LEGT(cmpOpL_commute cmp, flagsRegL_LEGT xcc, regD dst, regD src) %{
11522   match(Set dst (CMoveD (Binary cmp xcc) (Binary dst src)));
11523   predicate(_kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::le || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::gt );
11524 
11525   ins_cost(150);
11526   size(4);
11527   format %{ "FCPYD$cmp $dst,$src" %}
11528   ins_encode %{
11529     __ fcpyd($dst$$FloatRegister, $src$$FloatRegister, (AsmCondition)($cmp$$cmpcode));
11530   %}
11531   ins_pipe(int_conditional_float_move);
11532 %}
11533 #endif // !AARCH64
11534 
11535 // ============================================================================
11536 // Safepoint Instruction
11537 #ifdef AARCH64
11538 instruct safePoint_poll(iRegP poll, flagsReg icc, RtempRegP tmp) %{
11539   match(SafePoint poll);
11540   // The handler stub kills Rtemp
11541   effect(USE poll, KILL tmp, KILL icc);
11542 
11543   size(4);
11544   format %{ "LDR   ZR,[$poll]\t! Safepoint: poll for GC" %}
11545   ins_encode %{
11546     __ relocate(relocInfo::poll_type);
11547     __ ldr(ZR, Address($poll$$Register));
11548   %}
11549   ins_pipe(loadPollP);
11550 %}
11551 #else
11552 // rather than KILL R12, it would be better to use any reg as
11553 // TEMP. Can't do that at this point because it crashes the compiler
11554 instruct safePoint_poll(iRegP poll, R12RegI tmp, flagsReg icc) %{
11555   match(SafePoint poll);
11556   effect(USE poll, KILL tmp, KILL icc);
11557 
11558   size(4);
11559   format %{ "LDR   $tmp,[$poll]\t! Safepoint: poll for GC" %}
11560   ins_encode %{
11561     __ relocate(relocInfo::poll_type);
11562     __ ldr($tmp$$Register, Address($poll$$Register));
11563   %}
11564   ins_pipe(loadPollP);
11565 %}
11566 #endif
11567 
11568 
11569 // ============================================================================
11570 // Call Instructions
11571 // Call Java Static Instruction
11572 instruct CallStaticJavaDirect( method meth ) %{
11573   match(CallStaticJava);
11574   predicate(! ((CallStaticJavaNode*)n)->is_method_handle_invoke());
11575   effect(USE meth);
11576 
11577   ins_cost(CALL_COST);
11578   format %{ "CALL,static ==> " %}
11579   ins_encode( Java_Static_Call( meth ), call_epilog );
11580   ins_pipe(simple_call);
11581 %}
11582 
11583 // Call Java Static Instruction (method handle version)
11584 instruct CallStaticJavaHandle( method meth ) %{
11585   match(CallStaticJava);
11586   predicate(((CallStaticJavaNode*)n)->is_method_handle_invoke());
11587   effect(USE meth);
11588   // FP is saved by all callees (for interpreter stack correction).
11589   // We use it here for a similar purpose, in {preserve,restore}_FP.
11590 
11591   ins_cost(CALL_COST);
11592   format %{ "CALL,static/MethodHandle ==> " %}
11593   ins_encode( preserve_SP, Java_Static_Call( meth ), restore_SP, call_epilog );
11594   ins_pipe(simple_call);
11595 %}
11596 
11597 // Call Java Dynamic Instruction
11598 instruct CallDynamicJavaDirect( method meth ) %{
11599   match(CallDynamicJava);
11600   effect(USE meth);
11601 
11602   ins_cost(CALL_COST);
11603   format %{ "MOV_OOP    (empty),R_R8\n\t"
11604             "CALL,dynamic  ; NOP ==> " %}
11605   ins_encode( Java_Dynamic_Call( meth ), call_epilog );
11606   ins_pipe(call);
11607 %}
11608 
11609 // Call Runtime Instruction
11610 instruct CallRuntimeDirect(method meth) %{
11611   match(CallRuntime);
11612   effect(USE meth);
11613   ins_cost(CALL_COST);
11614   format %{ "CALL,runtime" %}
11615 #ifdef AARCH64
11616   ins_encode( save_last_PC, Java_To_Runtime( meth ),
11617               call_epilog );
11618 #else
11619   ins_encode( Java_To_Runtime( meth ),
11620               call_epilog );
11621 #endif
11622   ins_pipe(simple_call);
11623 %}
11624 
11625 // Call runtime without safepoint - same as CallRuntime
11626 instruct CallLeafDirect(method meth) %{
11627   match(CallLeaf);
11628   effect(USE meth);
11629   ins_cost(CALL_COST);
11630   format %{ "CALL,runtime leaf" %}
11631   // TODO: ned save_last_PC here?
11632   ins_encode( Java_To_Runtime( meth ),
11633               call_epilog );
11634   ins_pipe(simple_call);
11635 %}
11636 
11637 // Call runtime without safepoint - same as CallLeaf
11638 instruct CallLeafNoFPDirect(method meth) %{
11639   match(CallLeafNoFP);
11640   effect(USE meth);
11641   ins_cost(CALL_COST);
11642   format %{ "CALL,runtime leaf nofp" %}
11643   // TODO: ned save_last_PC here?
11644   ins_encode( Java_To_Runtime( meth ),
11645               call_epilog );
11646   ins_pipe(simple_call);
11647 %}
11648 
11649 // Tail Call; Jump from runtime stub to Java code.
11650 // Also known as an 'interprocedural jump'.
11651 // Target of jump will eventually return to caller.
11652 // TailJump below removes the return address.
11653 instruct TailCalljmpInd(IPRegP jump_target, inline_cache_regP method_oop) %{
11654   match(TailCall jump_target method_oop );
11655 
11656   ins_cost(CALL_COST);
11657   format %{ "MOV    Rexception_pc, LR\n\t"
11658             "jump   $jump_target  \t! $method_oop holds method oop" %}
11659   ins_encode %{
11660     __ mov(Rexception_pc, LR);   // this is used only to call
11661                                  // StubRoutines::forward_exception_entry()
11662                                  // which expects PC of exception in
11663                                  // R5. FIXME?
11664     __ jump($jump_target$$Register);
11665   %}
11666   ins_pipe(tail_call);
11667 %}
11668 
11669 
11670 // Return Instruction
11671 instruct Ret() %{
11672   match(Return);
11673 
11674   format %{ "ret LR" %}
11675 
11676   ins_encode %{
11677     __ ret(LR);
11678   %}
11679 
11680   ins_pipe(br);
11681 %}
11682 
11683 
11684 // Tail Jump; remove the return address; jump to target.
11685 // TailCall above leaves the return address around.
11686 // TailJump is used in only one place, the rethrow_Java stub (fancy_jump=2).
11687 // ex_oop (Exception Oop) is needed in %o0 at the jump. As there would be a
11688 // "restore" before this instruction (in Epilogue), we need to materialize it
11689 // in %i0.
11690 instruct tailjmpInd(IPRegP jump_target, RExceptionRegP ex_oop) %{
11691   match( TailJump jump_target ex_oop );
11692   ins_cost(CALL_COST);
11693   format %{ "MOV    Rexception_pc, LR\n\t"
11694             "jump   $jump_target \t! $ex_oop holds exc. oop" %}
11695   ins_encode %{
11696     __ mov(Rexception_pc, LR);
11697     __ jump($jump_target$$Register);
11698   %}
11699   ins_pipe(tail_call);
11700 %}
11701 
11702 // Create exception oop: created by stack-crawling runtime code.
11703 // Created exception is now available to this handler, and is setup
11704 // just prior to jumping to this handler.  No code emitted.
11705 instruct CreateException( RExceptionRegP ex_oop )
11706 %{
11707   match(Set ex_oop (CreateEx));
11708   ins_cost(0);
11709 
11710   size(0);
11711   // use the following format syntax
11712   format %{ "! exception oop is in Rexception_obj; no code emitted" %}
11713   ins_encode();
11714   ins_pipe(empty);
11715 %}
11716 
11717 
11718 // Rethrow exception:
11719 // The exception oop will come in the first argument position.
11720 // Then JUMP (not call) to the rethrow stub code.
11721 instruct RethrowException()
11722 %{
11723   match(Rethrow);
11724   ins_cost(CALL_COST);
11725 
11726   // use the following format syntax
11727   format %{ "b    rethrow_stub" %}
11728   ins_encode %{
11729     Register scratch = R1_tmp;
11730     assert_different_registers(scratch, c_rarg0, LR);
11731     __ jump(OptoRuntime::rethrow_stub(), relocInfo::runtime_call_type, scratch);
11732   %}
11733   ins_pipe(tail_call);
11734 %}
11735 
11736 
11737 // Die now
11738 instruct ShouldNotReachHere( )
11739 %{
11740   match(Halt);
11741   ins_cost(CALL_COST);
11742 
11743   size(4);
11744   // Use the following format syntax
11745   format %{ "breakpoint   ; ShouldNotReachHere" %}
11746   ins_encode %{
11747     __ breakpoint();
11748   %}
11749   ins_pipe(tail_call);
11750 %}
11751 
11752 // ============================================================================
11753 // The 2nd slow-half of a subtype check.  Scan the subklass's 2ndary superklass
11754 // array for an instance of the superklass.  Set a hidden internal cache on a
11755 // hit (cache is checked with exposed code in gen_subtype_check()).  Return
11756 // not zero for a miss or zero for a hit.  The encoding ALSO sets flags.
11757 instruct partialSubtypeCheck( R0RegP index, R1RegP sub, R2RegP super, flagsRegP pcc, LRRegP lr ) %{
11758   match(Set index (PartialSubtypeCheck sub super));
11759   effect( KILL pcc, KILL lr );
11760   ins_cost(DEFAULT_COST*10);
11761   format %{ "CALL   PartialSubtypeCheck" %}
11762   ins_encode %{
11763     __ call(StubRoutines::Arm::partial_subtype_check(), relocInfo::runtime_call_type);
11764   %}
11765   ins_pipe(partial_subtype_check_pipe);
11766 %}
11767 
11768 /* instruct partialSubtypeCheck_vs_zero( flagsRegP pcc, o1RegP sub, o2RegP super, immP0 zero, o0RegP idx, o7RegP o7 ) %{ */
11769 /*   match(Set pcc (CmpP (PartialSubtypeCheck sub super) zero)); */
11770 /*   ins_pipe(partial_subtype_check_pipe); */
11771 /* %} */
11772 
11773 
11774 // ============================================================================
11775 // inlined locking and unlocking
11776 
11777 #ifdef AARCH64
11778 instruct cmpFastLock(flagsRegP pcc, iRegP object, iRegP box, iRegP scratch2, iRegP scratch, iRegP scratch3 )
11779 #else
11780 instruct cmpFastLock(flagsRegP pcc, iRegP object, iRegP box, iRegP scratch2, iRegP scratch )
11781 #endif
11782 %{
11783   match(Set pcc (FastLock object box));
11784 
11785 #ifdef AARCH64
11786   effect(TEMP scratch, TEMP scratch2, TEMP scratch3);
11787 #else
11788   effect(TEMP scratch, TEMP scratch2);
11789 #endif
11790   ins_cost(100);
11791 
11792 #ifdef AARCH64
11793   format %{ "FASTLOCK  $object, $box; KILL $scratch, $scratch2, $scratch3" %}
11794   ins_encode %{
11795     __ fast_lock($object$$Register, $box$$Register, $scratch$$Register, $scratch2$$Register, $scratch3$$Register);
11796   %}
11797 #else
11798   format %{ "FASTLOCK  $object, $box; KILL $scratch, $scratch2" %}
11799   ins_encode %{
11800     __ fast_lock($object$$Register, $box$$Register, $scratch$$Register, $scratch2$$Register);
11801   %}
11802 #endif
11803   ins_pipe(long_memory_op);
11804 %}
11805 
11806 
11807 #ifdef AARCH64
11808 instruct cmpFastUnlock(flagsRegP pcc, iRegP object, iRegP box, iRegP scratch2, iRegP scratch, iRegP scratch3 ) %{
11809   match(Set pcc (FastUnlock object box));
11810   effect(TEMP scratch, TEMP scratch2, TEMP scratch3);
11811   ins_cost(100);
11812 
11813   format %{ "FASTUNLOCK  $object, $box; KILL $scratch, $scratch2, $scratch3" %}
11814   ins_encode %{
11815     __ fast_unlock($object$$Register, $box$$Register, $scratch$$Register, $scratch2$$Register, $scratch3$$Register);
11816   %}
11817   ins_pipe(long_memory_op);
11818 %}
11819 #else
11820 instruct cmpFastUnlock(flagsRegP pcc, iRegP object, iRegP box, iRegP scratch2, iRegP scratch ) %{
11821   match(Set pcc (FastUnlock object box));
11822   effect(TEMP scratch, TEMP scratch2);
11823   ins_cost(100);
11824 
11825   format %{ "FASTUNLOCK  $object, $box; KILL $scratch, $scratch2" %}
11826   ins_encode %{
11827     __ fast_unlock($object$$Register, $box$$Register, $scratch$$Register, $scratch2$$Register);
11828   %}
11829   ins_pipe(long_memory_op);
11830 %}
11831 #endif
11832 
11833 #ifdef AARCH64
11834 // TODO: add version that takes immI cnt?
11835 instruct clear_array(iRegX cnt, iRegP base, iRegP ptr, iRegX temp, Universe dummy, flagsReg cpsr) %{
11836   match(Set dummy (ClearArray cnt base));
11837   effect(TEMP temp, TEMP ptr, KILL cpsr);
11838   ins_cost(300);
11839   format %{
11840       "        MOV    $temp,$cnt\n"
11841       "        ADD    $ptr,$base,$cnt\n"
11842       "        SUBS   $temp,$temp,16\t! Count down dword pair in bytes\n"
11843       "        B.lt   done16\n"
11844       "loop:   STP    ZR,ZR,[$ptr,-16]!\n"
11845       "        SUBS   $temp,$temp,16\t! Count down dword pair in bytes\n"
11846       "        B.ge   loop\t! Clearing loop\n"
11847       "done16: ADDS   $temp,$temp,8\t! Room for 1 more long?\n"
11848       "        B.lt   done\n"
11849       "        STR    ZR,[$base+$temp]\n"
11850       "done:"
11851   %}
11852   ins_encode %{
11853     // TODO: preload?
11854     __ mov($temp$$Register, $cnt$$Register);
11855     __ add($ptr$$Register, $base$$Register, $cnt$$Register);
11856     Label loop, done, done16;
11857     __ subs($temp$$Register, $temp$$Register, 16);
11858     __ b(done16, lt);
11859     __ bind(loop);
11860     __ stp(ZR, ZR, Address($ptr$$Register, -16, pre_indexed));
11861     __ subs($temp$$Register, $temp$$Register, 16);
11862     __ b(loop, ge);
11863     __ bind(done16);
11864     __ adds($temp$$Register, $temp$$Register, 8);
11865     __ b(done, lt);
11866     // $temp should be 0 here
11867     __ str(ZR, Address($base$$Register, $temp$$Register));
11868     __ bind(done);
11869   %}
11870   ins_pipe(long_memory_op);
11871 %}
11872 #else
11873 // Count and Base registers are fixed because the allocator cannot
11874 // kill unknown registers.  The encodings are generic.
11875 instruct clear_array(iRegX cnt, iRegP base, iRegI temp, iRegX zero, Universe dummy, flagsReg cpsr) %{
11876   match(Set dummy (ClearArray cnt base));
11877   effect(TEMP temp, TEMP zero, KILL cpsr);
11878   ins_cost(300);
11879   format %{ "MOV    $zero,0\n"
11880       "        MOV    $temp,$cnt\n"
11881       "loop:   SUBS   $temp,$temp,4\t! Count down a dword of bytes\n"
11882       "        STR.ge $zero,[$base+$temp]\t! delay slot"
11883       "        B.gt   loop\t\t! Clearing loop\n" %}
11884   ins_encode %{
11885     __ mov($zero$$Register, 0);
11886     __ mov($temp$$Register, $cnt$$Register);
11887     Label(loop);
11888     __ bind(loop);
11889     __ subs($temp$$Register, $temp$$Register, 4);
11890     __ str($zero$$Register, Address($base$$Register, $temp$$Register), ge);
11891     __ b(loop, gt);
11892   %}
11893   ins_pipe(long_memory_op);
11894 %}
11895 #endif
11896 
11897 #ifdef XXX
11898 // FIXME: Why R0/R1/R2/R3?
11899 instruct string_compare(R0RegP str1, R1RegP str2, R2RegI cnt1, R3RegI cnt2, iRegI result,
11900                         iRegI tmp1, iRegI tmp2, flagsReg ccr) %{
11901   predicate(!CompactStrings);
11902   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
11903   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL ccr, TEMP tmp1, TEMP tmp2);
11904   ins_cost(300);
11905   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   // TEMP $tmp1, $tmp2" %}
11906   ins_encode( enc_String_Compare(str1, str2, cnt1, cnt2, result, tmp1, tmp2) );
11907 
11908   ins_pipe(long_memory_op);
11909 %}
11910 
11911 // FIXME: Why R0/R1/R2?
11912 instruct string_equals(R0RegP str1, R1RegP str2, R2RegI cnt, iRegI result, iRegI tmp1, iRegI tmp2,
11913                        flagsReg ccr) %{
11914   predicate(!CompactStrings);
11915   match(Set result (StrEquals (Binary str1 str2) cnt));
11916   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt, TEMP tmp1, TEMP tmp2, TEMP result, KILL ccr);
11917 
11918   ins_cost(300);
11919   format %{ "String Equals $str1,$str2,$cnt -> $result   // TEMP $tmp1, $tmp2" %}
11920   ins_encode( enc_String_Equals(str1, str2, cnt, result, tmp1, tmp2) );
11921   ins_pipe(long_memory_op);
11922 %}
11923 
11924 // FIXME: Why R0/R1?
11925 instruct array_equals(R0RegP ary1, R1RegP ary2, iRegI tmp1, iRegI tmp2, iRegI tmp3, iRegI result,
11926                       flagsReg ccr) %{
11927   predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
11928   match(Set result (AryEq ary1 ary2));
11929   effect(USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP result, KILL ccr);
11930 
11931   ins_cost(300);
11932   format %{ "Array Equals $ary1,$ary2 -> $result   // TEMP $tmp1,$tmp2,$tmp3" %}
11933   ins_encode( enc_Array_Equals(ary1, ary2, tmp1, tmp2, tmp3, result));
11934   ins_pipe(long_memory_op);
11935 %}
11936 #endif
11937 
11938 //---------- Zeros Count Instructions ------------------------------------------
11939 
11940 instruct countLeadingZerosI(iRegI dst, iRegI src) %{
11941   match(Set dst (CountLeadingZerosI src));
11942   size(4);
11943   format %{ "CLZ_32 $dst,$src" %}
11944   ins_encode %{
11945     __ clz_32($dst$$Register, $src$$Register);
11946   %}
11947   ins_pipe(ialu_reg);
11948 %}
11949 
11950 #ifdef AARCH64
11951 instruct countLeadingZerosL(iRegI dst, iRegL src) %{
11952   match(Set dst (CountLeadingZerosL src));
11953   size(4);
11954   format %{ "CLZ $dst,$src" %}
11955   ins_encode %{
11956     __ clz($dst$$Register, $src$$Register);
11957   %}
11958   ins_pipe(ialu_reg);
11959 %}
11960 #else
11961 instruct countLeadingZerosL(iRegI dst, iRegL src, iRegI tmp, flagsReg ccr) %{
11962   match(Set dst (CountLeadingZerosL src));
11963   effect(TEMP tmp, TEMP dst, KILL ccr);
11964   size(16);
11965   format %{ "CLZ    $dst,$src.hi\n\t"
11966             "TEQ    $dst,32\n\t"
11967             "CLZ.eq $tmp,$src.lo\n\t"
11968             "ADD.eq $dst, $dst, $tmp\n\t" %}
11969   ins_encode %{
11970     __ clz($dst$$Register, $src$$Register->successor());
11971     __ teq($dst$$Register, 32);
11972     __ clz($tmp$$Register, $src$$Register, eq);
11973     __ add($dst$$Register, $dst$$Register, $tmp$$Register, eq);
11974   %}
11975   ins_pipe(ialu_reg);
11976 %}
11977 #endif
11978 
11979 instruct countTrailingZerosI(iRegI dst, iRegI src, iRegI tmp) %{
11980   match(Set dst (CountTrailingZerosI src));
11981   effect(TEMP tmp);
11982   size(8);
11983   format %{ "RBIT_32 $tmp, $src\n\t"
11984             "CLZ_32  $dst,$tmp" %}
11985   ins_encode %{
11986     __ rbit_32($tmp$$Register, $src$$Register);
11987     __ clz_32($dst$$Register, $tmp$$Register);
11988   %}
11989   ins_pipe(ialu_reg);
11990 %}
11991 
11992 #ifdef AARCH64
11993 instruct countTrailingZerosL(iRegI dst, iRegL src, iRegL tmp) %{
11994   match(Set dst (CountTrailingZerosL src));
11995   effect(TEMP tmp);
11996   size(8);
11997   format %{ "RBIT $tmp, $src\n\t"
11998             "CLZ  $dst,$tmp" %}
11999   ins_encode %{
12000     __ rbit($tmp$$Register, $src$$Register);
12001     __ clz($dst$$Register, $tmp$$Register);
12002   %}
12003   ins_pipe(ialu_reg);
12004 %}
12005 #else
12006 instruct countTrailingZerosL(iRegI dst, iRegL src, iRegI tmp, flagsReg ccr) %{
12007   match(Set dst (CountTrailingZerosL src));
12008   effect(TEMP tmp, TEMP dst, KILL ccr);
12009   size(24);
12010   format %{ "RBIT   $tmp,$src.lo\n\t"
12011             "CLZ    $dst,$tmp\n\t"
12012             "TEQ    $dst,32\n\t"
12013             "RBIT   $tmp,$src.hi\n\t"
12014             "CLZ.eq $tmp,$tmp\n\t"
12015             "ADD.eq $dst,$dst,$tmp\n\t" %}
12016   ins_encode %{
12017     __ rbit($tmp$$Register, $src$$Register);
12018     __ clz($dst$$Register, $tmp$$Register);
12019     __ teq($dst$$Register, 32);
12020     __ rbit($tmp$$Register, $src$$Register->successor());
12021     __ clz($tmp$$Register, $tmp$$Register, eq);
12022     __ add($dst$$Register, $dst$$Register, $tmp$$Register, eq);
12023   %}
12024   ins_pipe(ialu_reg);
12025 %}
12026 #endif
12027 
12028 
12029 //---------- Population Count Instructions -------------------------------------
12030 
12031 #ifdef AARCH64
12032 instruct popCountI(iRegI dst, iRegI src, regD_low tmp) %{
12033   predicate(UsePopCountInstruction);
12034   match(Set dst (PopCountI src));
12035   effect(TEMP tmp);
12036   size(20);
12037 
12038   format %{ "MOV_W      $dst,$src\n\t"
12039             "FMOV_dx    $tmp,$dst\n\t"
12040             "VCNT       $tmp.8B,$tmp.8B\n\t"
12041             "ADDV       $tmp.B,$tmp.8B\n\t"
12042             "FMRS       $dst,$tmp" %}
12043 
12044   ins_encode %{
12045     __ mov_w($dst$$Register, $src$$Register);
12046     __ fmov_dx($tmp$$FloatRegister, $dst$$Register);
12047     int quad = 0;
12048     int cnt_size = 0; // VELEM_SIZE_8
12049     __ vcnt($tmp$$FloatRegister, $tmp$$FloatRegister, quad, cnt_size);
12050     int add_size = 0; // VELEM_SIZE_8
12051     __ addv($tmp$$FloatRegister, $tmp$$FloatRegister, quad, add_size);
12052     __ fmrs($dst$$Register, $tmp$$FloatRegister);
12053   %}
12054   ins_pipe(ialu_reg); // FIXME
12055 %}
12056 #else
12057 instruct popCountI(iRegI dst, iRegI src, regD_low tmp) %{
12058   predicate(UsePopCountInstruction);
12059   match(Set dst (PopCountI src));
12060   effect(TEMP tmp);
12061 
12062   format %{ "FMSR       $tmp,$src\n\t"
12063             "VCNT.8     $tmp,$tmp\n\t"
12064             "VPADDL.U8  $tmp,$tmp\n\t"
12065             "VPADDL.U16 $tmp,$tmp\n\t"
12066             "FMRS       $dst,$tmp" %}
12067   size(20);
12068 
12069   ins_encode %{
12070     __ fmsr($tmp$$FloatRegister, $src$$Register);
12071     __ vcnt($tmp$$FloatRegister, $tmp$$FloatRegister);
12072     __ vpaddl($tmp$$FloatRegister, $tmp$$FloatRegister, 8, 0);
12073     __ vpaddl($tmp$$FloatRegister, $tmp$$FloatRegister, 16, 0);
12074     __ fmrs($dst$$Register, $tmp$$FloatRegister);
12075   %}
12076   ins_pipe(ialu_reg); // FIXME
12077 %}
12078 #endif
12079 
12080 #ifdef AARCH64
12081 instruct popCountL(iRegI dst, iRegL src, regD tmp) %{
12082   predicate(UsePopCountInstruction);
12083   match(Set dst (PopCountL src));
12084   effect(TEMP tmp);
12085   size(16);
12086 
12087   format %{ "FMOV_dx    $tmp,$src\n\t"
12088             "VCNT       $tmp.8B,$tmp.8B\n\t"
12089             "ADDV       $tmp.B,$tmp.8B\n\t"
12090             "FMOV_ws    $dst,$tmp" %}
12091 
12092   ins_encode %{
12093     __ fmov_dx($tmp$$FloatRegister, $src$$Register);
12094     int quad = 0;
12095     int cnt_size = 0;
12096     __ vcnt($tmp$$FloatRegister, $tmp$$FloatRegister, quad, cnt_size);
12097     int add_size = 0;
12098     __ addv($tmp$$FloatRegister, $tmp$$FloatRegister, quad, add_size);
12099     __ fmov_ws($dst$$Register, $tmp$$FloatRegister);
12100   %}
12101   ins_pipe(ialu_reg); // FIXME
12102 %}
12103 #else
12104 // Note: Long.bitCount(long) returns an int.
12105 instruct popCountL(iRegI dst, iRegL src, regD_low tmp) %{
12106   predicate(UsePopCountInstruction);
12107   match(Set dst (PopCountL src));
12108   effect(TEMP tmp);
12109 
12110   format %{ "FMDRR       $tmp,$src.lo,$src.hi\n\t"
12111             "VCNT.8      $tmp,$tmp\n\t"
12112             "VPADDL.U8   $tmp,$tmp\n\t"
12113             "VPADDL.U16  $tmp,$tmp\n\t"
12114             "VPADDL.U32  $tmp,$tmp\n\t"
12115             "FMRS        $dst,$tmp" %}
12116 
12117   size(32);
12118 
12119   ins_encode %{
12120     __ fmdrr($tmp$$FloatRegister, $src$$Register, $src$$Register->successor());
12121     __ vcnt($tmp$$FloatRegister, $tmp$$FloatRegister);
12122     __ vpaddl($tmp$$FloatRegister, $tmp$$FloatRegister, 8, 0);
12123     __ vpaddl($tmp$$FloatRegister, $tmp$$FloatRegister, 16, 0);
12124     __ vpaddl($tmp$$FloatRegister, $tmp$$FloatRegister, 32, 0);
12125     __ fmrs($dst$$Register, $tmp$$FloatRegister);
12126   %}
12127   ins_pipe(ialu_reg);
12128 %}
12129 #endif
12130 
12131 
12132 // ============================================================================
12133 //------------Bytes reverse--------------------------------------------------
12134 
12135 instruct bytes_reverse_int(iRegI dst, iRegI src) %{
12136   match(Set dst (ReverseBytesI src));
12137 
12138   size(4);
12139   format %{ "REV32 $dst,$src" %}
12140   ins_encode %{
12141 #ifdef AARCH64
12142     __ rev_w($dst$$Register, $src$$Register);
12143     // high 32 bits zeroed, not sign extended
12144 #else
12145     __ rev($dst$$Register, $src$$Register);
12146 #endif
12147   %}
12148   ins_pipe( iload_mem ); // FIXME
12149 %}
12150 
12151 instruct bytes_reverse_long(iRegL dst, iRegL src) %{
12152   match(Set dst (ReverseBytesL src));
12153 #ifdef AARCH64
12154 //size(4);
12155   format %{ "REV $dst,$src"  %}
12156   ins_encode %{
12157     __ rev($dst$$Register, $src$$Register);
12158   %}
12159   ins_pipe(ialu_reg_reg); // FIXME
12160 #else
12161   effect(TEMP dst);
12162   size(8);
12163   format %{ "REV $dst.lo,$src.lo\n\t"
12164             "REV $dst.hi,$src.hi" %}
12165   ins_encode %{
12166     __ rev($dst$$Register, $src$$Register->successor());
12167     __ rev($dst$$Register->successor(), $src$$Register);
12168   %}
12169   ins_pipe( iload_mem ); // FIXME
12170 #endif
12171 %}
12172 
12173 instruct bytes_reverse_unsigned_short(iRegI dst, iRegI src) %{
12174   match(Set dst (ReverseBytesUS src));
12175 #ifdef AARCH64
12176   size(4);
12177   format %{ "REV16_W $dst,$src" %}
12178   ins_encode %{
12179     __ rev16_w($dst$$Register, $src$$Register);
12180     // high 32 bits zeroed
12181   %}
12182 #else
12183   size(4);
12184   format %{ "REV16 $dst,$src" %}
12185   ins_encode %{
12186     __ rev16($dst$$Register, $src$$Register);
12187   %}
12188 #endif
12189   ins_pipe( iload_mem ); // FIXME
12190 %}
12191 
12192 instruct bytes_reverse_short(iRegI dst, iRegI src) %{
12193   match(Set dst (ReverseBytesS src));
12194 #ifdef AARCH64
12195   size(8);
12196   format %{ "REV16_W $dst,$src\n\t"
12197             "SIGN_EXT16 $dst" %}
12198   ins_encode %{
12199     __ rev16_w($dst$$Register, $src$$Register);
12200     __ sign_extend($dst$$Register, $dst$$Register, 16);
12201   %}
12202 #else
12203   size(4);
12204   format %{ "REVSH $dst,$src" %}
12205   ins_encode %{
12206     __ revsh($dst$$Register, $src$$Register);
12207   %}
12208 #endif
12209   ins_pipe( iload_mem ); // FIXME
12210 %}
12211 
12212 
12213 // ====================VECTOR INSTRUCTIONS=====================================
12214 
12215 // Load Aligned Packed values into a Double Register
12216 instruct loadV8(vecD dst, memoryD mem) %{
12217   predicate(n->as_LoadVector()->memory_size() == 8);
12218   match(Set dst (LoadVector mem));
12219   ins_cost(MEMORY_REF_COST);
12220   size(4);
12221   format %{ "FLDD   $mem,$dst\t! load vector (8 bytes)" %}
12222   ins_encode %{
12223     __ ldr_double($dst$$FloatRegister, $mem$$Address);
12224   %}
12225   ins_pipe(floadD_mem);
12226 %}
12227 
12228 // Load Aligned Packed values into a Double Register Pair
12229 instruct loadV16(vecX dst, memoryvld mem) %{
12230   predicate(n->as_LoadVector()->memory_size() == 16);
12231   match(Set dst (LoadVector mem));
12232   ins_cost(MEMORY_REF_COST);
12233   size(4);
12234   format %{ "VLD1   $mem,$dst.Q\t! load vector (16 bytes)" %}
12235   ins_encode %{
12236     __ vld1($dst$$FloatRegister, $mem$$Address, MacroAssembler::VELEM_SIZE_16, 128);
12237   %}
12238   ins_pipe(floadD_mem); // FIXME
12239 %}
12240 
12241 // Store Vector in Double register to memory
12242 instruct storeV8(memoryD mem, vecD src) %{
12243   predicate(n->as_StoreVector()->memory_size() == 8);
12244   match(Set mem (StoreVector mem src));
12245   ins_cost(MEMORY_REF_COST);
12246   size(4);
12247   format %{ "FSTD   $src,$mem\t! store vector (8 bytes)" %}
12248   ins_encode %{
12249     __ str_double($src$$FloatRegister, $mem$$Address);
12250   %}
12251   ins_pipe(fstoreD_mem_reg);
12252 %}
12253 
12254 // Store Vector in Double Register Pair to memory
12255 instruct storeV16(memoryvld mem, vecX src) %{
12256   predicate(n->as_StoreVector()->memory_size() == 16);
12257   match(Set mem (StoreVector mem src));
12258   ins_cost(MEMORY_REF_COST);
12259   size(4);
12260   format %{ "VST1   $src,$mem\t! store vector (16 bytes)" %}
12261   ins_encode %{
12262     __ vst1($src$$FloatRegister, $mem$$Address, MacroAssembler::VELEM_SIZE_16, 128);
12263   %}
12264   ins_pipe(fstoreD_mem_reg); // FIXME
12265 %}
12266 
12267 #ifndef AARCH64
12268 // Replicate scalar to packed byte values in Double register
12269 instruct Repl8B_reg(vecD dst, iRegI src, iRegI tmp) %{
12270   predicate(n->as_Vector()->length() == 8);
12271   match(Set dst (ReplicateB src));
12272   ins_cost(DEFAULT_COST*4);
12273   effect(TEMP tmp);
12274   size(16);
12275 
12276   // FIXME: could use PKH instruction instead?
12277   format %{ "LSL      $tmp, $src, 24 \n\t"
12278             "OR       $tmp, $tmp, ($tmp >> 8) \n\t"
12279             "OR       $tmp, $tmp, ($tmp >> 16) \n\t"
12280             "FMDRR    $dst,$tmp,$tmp\t" %}
12281   ins_encode %{
12282     __ mov($tmp$$Register, AsmOperand($src$$Register, lsl, 24));
12283     __ orr($tmp$$Register, $tmp$$Register, AsmOperand($tmp$$Register, lsr, 8));
12284     __ orr($tmp$$Register, $tmp$$Register, AsmOperand($tmp$$Register, lsr, 16));
12285     __ fmdrr($dst$$FloatRegister, $tmp$$Register, $tmp$$Register);
12286   %}
12287   ins_pipe(ialu_reg); // FIXME
12288 %}
12289 #endif /* !AARCH64 */
12290 
12291 // Replicate scalar to packed byte values in Double register
12292 instruct Repl8B_reg_simd(vecD dst, iRegI src) %{
12293   predicate(n->as_Vector()->length_in_bytes() == 8 && VM_Version::has_simd());
12294   match(Set dst (ReplicateB src));
12295   size(4);
12296 
12297   format %{ "VDUP.8 $dst,$src\t" %}
12298   ins_encode %{
12299     bool quad = false;
12300     __ vdupI($dst$$FloatRegister, $src$$Register,
12301              MacroAssembler::VELEM_SIZE_8, quad);
12302   %}
12303   ins_pipe(ialu_reg); // FIXME
12304 %}
12305 
12306 // Replicate scalar to packed byte values in Double register pair
12307 instruct Repl16B_reg(vecX dst, iRegI src) %{
12308   predicate(n->as_Vector()->length_in_bytes() == 16);
12309   match(Set dst (ReplicateB src));
12310   size(4);
12311 
12312   format %{ "VDUP.8 $dst.Q,$src\t" %}
12313   ins_encode %{
12314     bool quad = true;
12315     __ vdupI($dst$$FloatRegister, $src$$Register,
12316              MacroAssembler::VELEM_SIZE_8, quad);
12317   %}
12318   ins_pipe(ialu_reg); // FIXME
12319 %}
12320 
12321 #ifndef AARCH64
12322 // Replicate scalar constant to packed byte values in Double register
12323 instruct Repl8B_immI(vecD dst, immI src, iRegI tmp) %{
12324   predicate(n->as_Vector()->length() == 8);
12325   match(Set dst (ReplicateB src));
12326   ins_cost(DEFAULT_COST*2);
12327   effect(TEMP tmp);
12328   size(12);
12329 
12330   format %{ "MOV      $tmp, Repl4($src))\n\t"
12331             "FMDRR    $dst,$tmp,$tmp\t" %}
12332   ins_encode( LdReplImmI(src, dst, tmp, (4), (1)) );
12333   ins_pipe(loadConFD); // FIXME
12334 %}
12335 #endif /* !AARCH64 */
12336 
12337 // Replicate scalar constant to packed byte values in Double register
12338 // TODO: support negative constants with MVNI?
12339 instruct Repl8B_immU8(vecD dst, immU8 src) %{
12340   predicate(n->as_Vector()->length_in_bytes() == 8 && VM_Version::has_simd());
12341   match(Set dst (ReplicateB src));
12342   size(4);
12343 
12344   format %{ "VMOV.U8  $dst,$src" %}
12345   ins_encode %{
12346     bool quad = false;
12347     __ vmovI($dst$$FloatRegister, $src$$constant,
12348              MacroAssembler::VELEM_SIZE_8, quad);
12349   %}
12350   ins_pipe(loadConFD); // FIXME
12351 %}
12352 
12353 // Replicate scalar constant to packed byte values in Double register pair
12354 instruct Repl16B_immU8(vecX dst, immU8 src) %{
12355   predicate(n->as_Vector()->length_in_bytes() == 16 && VM_Version::has_simd());
12356   match(Set dst (ReplicateB src));
12357   size(4);
12358 
12359   format %{ "VMOV.U8  $dst.Q,$src" %}
12360   ins_encode %{
12361     bool quad = true;
12362     __ vmovI($dst$$FloatRegister, $src$$constant,
12363              MacroAssembler::VELEM_SIZE_8, quad);
12364   %}
12365   ins_pipe(loadConFD); // FIXME
12366 %}
12367 
12368 #ifndef AARCH64
12369 // Replicate scalar to packed short/char values into Double register
12370 instruct Repl4S_reg(vecD dst, iRegI src, iRegI tmp) %{
12371   predicate(n->as_Vector()->length() == 4);
12372   match(Set dst (ReplicateS src));
12373   ins_cost(DEFAULT_COST*3);
12374   effect(TEMP tmp);
12375   size(12);
12376 
12377   // FIXME: could use PKH instruction instead?
12378   format %{ "LSL      $tmp, $src, 16 \n\t"
12379             "OR       $tmp, $tmp, ($tmp >> 16) \n\t"
12380             "FMDRR    $dst,$tmp,$tmp\t" %}
12381   ins_encode %{
12382     __ mov($tmp$$Register, AsmOperand($src$$Register, lsl, 16));
12383     __ orr($tmp$$Register, $tmp$$Register, AsmOperand($tmp$$Register, lsr, 16));
12384     __ fmdrr($dst$$FloatRegister, $tmp$$Register, $tmp$$Register);
12385   %}
12386   ins_pipe(ialu_reg); // FIXME
12387 %}
12388 #endif /* !AARCH64 */
12389 
12390 // Replicate scalar to packed byte values in Double register
12391 instruct Repl4S_reg_simd(vecD dst, iRegI src) %{
12392   predicate(n->as_Vector()->length_in_bytes() == 8 && VM_Version::has_simd());
12393   match(Set dst (ReplicateS src));
12394   size(4);
12395 
12396   format %{ "VDUP.16 $dst,$src\t" %}
12397   ins_encode %{
12398     bool quad = false;
12399     __ vdupI($dst$$FloatRegister, $src$$Register,
12400              MacroAssembler::VELEM_SIZE_16, quad);
12401   %}
12402   ins_pipe(ialu_reg); // FIXME
12403 %}
12404 
12405 // Replicate scalar to packed byte values in Double register pair
12406 instruct Repl8S_reg(vecX dst, iRegI src) %{
12407   predicate(n->as_Vector()->length_in_bytes() == 16 && VM_Version::has_simd());
12408   match(Set dst (ReplicateS src));
12409   size(4);
12410 
12411   format %{ "VDUP.16 $dst.Q,$src\t" %}
12412   ins_encode %{
12413     bool quad = true;
12414     __ vdupI($dst$$FloatRegister, $src$$Register,
12415              MacroAssembler::VELEM_SIZE_16, quad);
12416   %}
12417   ins_pipe(ialu_reg); // FIXME
12418 %}
12419 
12420 
12421 #ifndef AARCH64
12422 // Replicate scalar constant to packed short/char values in Double register
12423 instruct Repl4S_immI(vecD dst, immI src, iRegP tmp) %{
12424   predicate(n->as_Vector()->length() == 4);
12425   match(Set dst (ReplicateS src));
12426   effect(TEMP tmp);
12427   size(12);
12428   ins_cost(DEFAULT_COST*4); // FIXME
12429 
12430   format %{ "MOV      $tmp, Repl2($src))\n\t"
12431             "FMDRR    $dst,$tmp,$tmp\t" %}
12432   ins_encode( LdReplImmI(src, dst, tmp, (2), (2)) );
12433   ins_pipe(loadConFD); // FIXME
12434 %}
12435 #endif /* !AARCH64 */
12436 
12437 // Replicate scalar constant to packed byte values in Double register
12438 instruct Repl4S_immU8(vecD dst, immU8 src) %{
12439   predicate(n->as_Vector()->length_in_bytes() == 8 && VM_Version::has_simd());
12440   match(Set dst (ReplicateS src));
12441   size(4);
12442 
12443   format %{ "VMOV.U16  $dst,$src" %}
12444   ins_encode %{
12445     bool quad = false;
12446     __ vmovI($dst$$FloatRegister, $src$$constant,
12447              MacroAssembler::VELEM_SIZE_16, quad);
12448   %}
12449   ins_pipe(loadConFD); // FIXME
12450 %}
12451 
12452 // Replicate scalar constant to packed byte values in Double register pair
12453 instruct Repl8S_immU8(vecX dst, immU8 src) %{
12454   predicate(n->as_Vector()->length_in_bytes() == 16 && VM_Version::has_simd());
12455   match(Set dst (ReplicateS src));
12456   size(4);
12457 
12458   format %{ "VMOV.U16  $dst.Q,$src" %}
12459   ins_encode %{
12460     bool quad = true;
12461     __ vmovI($dst$$FloatRegister, $src$$constant,
12462              MacroAssembler::VELEM_SIZE_16, quad);
12463   %}
12464   ins_pipe(loadConFD); // FIXME
12465 %}
12466 
12467 #ifndef AARCH64
12468 // Replicate scalar to packed int values in Double register
12469 instruct Repl2I_reg(vecD dst, iRegI src) %{
12470   predicate(n->as_Vector()->length() == 2);
12471   match(Set dst (ReplicateI src));
12472   size(4);
12473 
12474   format %{ "FMDRR    $dst,$src,$src\t" %}
12475   ins_encode %{
12476     __ fmdrr($dst$$FloatRegister, $src$$Register, $src$$Register);
12477   %}
12478   ins_pipe(ialu_reg); // FIXME
12479 %}
12480 
12481 // Replicate scalar to packed int values in Double register pair
12482 instruct Repl4I_reg(vecX dst, iRegI src) %{
12483   predicate(n->as_Vector()->length() == 4);
12484   match(Set dst (ReplicateI src));
12485   ins_cost(DEFAULT_COST*2);
12486   size(8);
12487 
12488   format %{ "FMDRR    $dst.lo,$src,$src\n\t"
12489             "FMDRR    $dst.hi,$src,$src" %}
12490 
12491   ins_encode %{
12492     __ fmdrr($dst$$FloatRegister, $src$$Register, $src$$Register);
12493     __ fmdrr($dst$$FloatRegister->successor()->successor(),
12494              $src$$Register, $src$$Register);
12495   %}
12496   ins_pipe(ialu_reg); // FIXME
12497 %}
12498 #endif /* !AARCH64 */
12499 
12500 // Replicate scalar to packed int values in Double register
12501 instruct Repl2I_reg_simd(vecD dst, iRegI src) %{
12502   predicate(n->as_Vector()->length_in_bytes() == 8 && VM_Version::has_simd());
12503   match(Set dst (ReplicateI src));
12504   size(4);
12505 
12506   format %{ "VDUP.32 $dst.D,$src\t" %}
12507   ins_encode %{
12508     bool quad = false;
12509     __ vdupI($dst$$FloatRegister, $src$$Register,
12510              MacroAssembler::VELEM_SIZE_32, quad);
12511   %}
12512   ins_pipe(ialu_reg); // FIXME
12513 %}
12514 
12515 // Replicate scalar to packed int values in Double register pair
12516 instruct Repl4I_reg_simd(vecX dst, iRegI src) %{
12517   predicate(n->as_Vector()->length_in_bytes() == 16 && VM_Version::has_simd());
12518   match(Set dst (ReplicateI src));
12519   size(4);
12520 
12521   format %{ "VDUP.32 $dst.Q,$src\t" %}
12522   ins_encode %{
12523     bool quad = true;
12524     __ vdupI($dst$$FloatRegister, $src$$Register,
12525              MacroAssembler::VELEM_SIZE_32, quad);
12526   %}
12527   ins_pipe(ialu_reg); // FIXME
12528 %}
12529 
12530 
12531 #ifndef AARCH64
12532 // Replicate scalar zero constant to packed int values in Double register
12533 instruct Repl2I_immI(vecD dst, immI src, iRegI tmp) %{
12534   predicate(n->as_Vector()->length() == 2);
12535   match(Set dst (ReplicateI src));
12536   effect(TEMP tmp);
12537   size(12);
12538   ins_cost(DEFAULT_COST*4); // FIXME
12539 
12540   format %{ "MOV      $tmp, Repl1($src))\n\t"
12541             "FMDRR    $dst,$tmp,$tmp\t" %}
12542   ins_encode( LdReplImmI(src, dst, tmp, (1), (4)) );
12543   ins_pipe(loadConFD); // FIXME
12544 %}
12545 #endif /* !AARCH64 */
12546 
12547 // Replicate scalar constant to packed byte values in Double register
12548 instruct Repl2I_immU8(vecD dst, immU8 src) %{
12549   predicate(n->as_Vector()->length_in_bytes() == 8 && VM_Version::has_simd());
12550   match(Set dst (ReplicateI src));
12551   size(4);
12552 
12553   format %{ "VMOV.I32  $dst.D,$src" %}
12554   ins_encode %{
12555     bool quad = false;
12556     __ vmovI($dst$$FloatRegister, $src$$constant,
12557              MacroAssembler::VELEM_SIZE_32, quad);
12558   %}
12559   ins_pipe(loadConFD); // FIXME
12560 %}
12561 
12562 // Replicate scalar constant to packed byte values in Double register pair
12563 instruct Repl4I_immU8(vecX dst, immU8 src) %{
12564   predicate(n->as_Vector()->length_in_bytes() == 16 && VM_Version::has_simd());
12565   match(Set dst (ReplicateI src));
12566   size(4);
12567 
12568   format %{ "VMOV.I32  $dst.Q,$src" %}
12569   ins_encode %{
12570     bool quad = true;
12571     __ vmovI($dst$$FloatRegister, $src$$constant,
12572              MacroAssembler::VELEM_SIZE_32, quad);
12573   %}
12574   ins_pipe(loadConFD); // FIXME
12575 %}
12576 
12577 #ifdef AARCH64
12578 // Replicate scalar to packed byte values in Double register pair
12579 instruct Repl2L_reg(vecX dst, iRegL src) %{
12580   predicate(n->as_Vector()->length() == 2);
12581   match(Set dst (ReplicateL src));
12582   size(4*1);
12583   ins_cost(DEFAULT_COST*1); // FIXME
12584 
12585   format %{ "VDUP.2D $dst.Q,$src\t" %}
12586   ins_encode %{
12587     bool quad = true;
12588     __ vdupI($dst$$FloatRegister, $src$$Register,
12589              MacroAssembler::VELEM_SIZE_64, quad);
12590   %}
12591   ins_pipe(ialu_reg); // FIXME
12592 %}
12593 #else /* !AARCH64 */
12594 // Replicate scalar to packed byte values in Double register pair
12595 instruct Repl2L_reg(vecX dst, iRegL src) %{
12596   predicate(n->as_Vector()->length() == 2);
12597   match(Set dst (ReplicateL src));
12598   size(8);
12599   ins_cost(DEFAULT_COST*2); // FIXME
12600 
12601   format %{ "FMDRR $dst.D,$src.lo,$src.hi\t\n"
12602             "FMDRR $dst.D.next,$src.lo,$src.hi" %}
12603   ins_encode %{
12604     __ fmdrr($dst$$FloatRegister, $src$$Register, $src$$Register->successor());
12605     __ fmdrr($dst$$FloatRegister->successor()->successor(),
12606              $src$$Register, $src$$Register->successor());
12607   %}
12608   ins_pipe(ialu_reg); // FIXME
12609 %}
12610 
12611 
12612 // Replicate scalar to packed float values in Double register
12613 instruct Repl2F_regI(vecD dst, iRegI src) %{
12614   predicate(n->as_Vector()->length() == 2);
12615   match(Set dst (ReplicateF src));
12616   size(4);
12617 
12618   format %{ "FMDRR    $dst.D,$src,$src\t" %}
12619   ins_encode %{
12620     __ fmdrr($dst$$FloatRegister, $src$$Register, $src$$Register);
12621   %}
12622   ins_pipe(ialu_reg); // FIXME
12623 %}
12624 
12625 // Replicate scalar to packed float values in Double register
12626 instruct Repl2F_reg_vfp(vecD dst, regF src) %{
12627   predicate(n->as_Vector()->length() == 2);
12628   match(Set dst (ReplicateF src));
12629   size(4*2);
12630   ins_cost(DEFAULT_COST*2); // FIXME
12631 
12632   expand %{
12633     iRegI tmp;
12634     MoveF2I_reg_reg(tmp, src);
12635     Repl2F_regI(dst,tmp);
12636   %}
12637 %}
12638 #endif /* !AARCH64 */
12639 
12640 // Replicate scalar to packed float values in Double register
12641 instruct Repl2F_reg_simd(vecD dst, regF src) %{
12642   predicate(n->as_Vector()->length_in_bytes() == 8 && VM_Version::has_simd());
12643   match(Set dst (ReplicateF src));
12644   size(4);
12645   ins_cost(DEFAULT_COST); // FIXME
12646 
12647   format %{ "VDUP.32  $dst.D,$src.D\t" %}
12648   ins_encode %{
12649     bool quad = false;
12650     __ vdupF($dst$$FloatRegister, $src$$FloatRegister, quad);
12651   %}
12652   ins_pipe(ialu_reg); // FIXME
12653 %}
12654 
12655 #ifndef AARCH64
12656 // Replicate scalar to packed float values in Double register pair
12657 instruct Repl4F_reg(vecX dst, regF src, iRegI tmp) %{
12658   predicate(n->as_Vector()->length() == 4);
12659   match(Set dst (ReplicateF src));
12660   effect(TEMP tmp);
12661   size(4*3);
12662   ins_cost(DEFAULT_COST*3); // FIXME
12663 
12664   format %{ "FMRS     $tmp,$src\n\t"
12665             "FMDRR    $dst.D,$tmp,$tmp\n\t"
12666             "FMDRR    $dst.D.next,$tmp,$tmp\t" %}
12667   ins_encode %{
12668     __ fmrs($tmp$$Register, $src$$FloatRegister);
12669     __ fmdrr($dst$$FloatRegister, $tmp$$Register, $tmp$$Register);
12670     __ fmdrr($dst$$FloatRegister->successor()->successor(),
12671              $tmp$$Register, $tmp$$Register);
12672   %}
12673   ins_pipe(ialu_reg); // FIXME
12674 %}
12675 #endif /* !AARCH64 */
12676 
12677 // Replicate scalar to packed float values in Double register pair
12678 instruct Repl4F_reg_simd(vecX dst, regF src) %{
12679   predicate(n->as_Vector()->length_in_bytes() == 16 && VM_Version::has_simd());
12680   match(Set dst (ReplicateF src));
12681   size(4);
12682   ins_cost(DEFAULT_COST); // FIXME
12683 
12684   format %{ "VDUP.32  $dst.Q,$src.D\t" %}
12685   ins_encode %{
12686     bool quad = true;
12687     __ vdupF($dst$$FloatRegister, $src$$FloatRegister, quad);
12688   %}
12689   ins_pipe(ialu_reg); // FIXME
12690 %}
12691 
12692 #ifndef AARCH64
12693 // Replicate scalar zero constant to packed float values in Double register
12694 instruct Repl2F_immI(vecD dst, immF src, iRegI tmp) %{
12695   predicate(n->as_Vector()->length() == 2);
12696   match(Set dst (ReplicateF src));
12697   effect(TEMP tmp);
12698   size(12);
12699   ins_cost(DEFAULT_COST*4); // FIXME
12700 
12701   format %{ "MOV      $tmp, Repl1($src))\n\t"
12702             "FMDRR    $dst,$tmp,$tmp\t" %}
12703   ins_encode( LdReplImmF(src, dst, tmp) );
12704   ins_pipe(loadConFD); // FIXME
12705 %}
12706 #endif /* !AAARCH64 */
12707 
12708 // Replicate scalar to packed double float values in Double register pair
12709 instruct Repl2D_reg(vecX dst, regD src) %{
12710 #ifdef AARCH64
12711   predicate(n->as_Vector()->length() == 2 && VM_Version::has_simd());
12712   match(Set dst (ReplicateD src));
12713   size(4*1);
12714   ins_cost(DEFAULT_COST*1); // FIXME
12715 
12716   format %{ "VDUP     $dst.2D,$src\t" %}
12717   ins_encode %{
12718     bool quad = true;
12719     __ vdupD($dst$$FloatRegister, $src$$FloatRegister, quad);
12720   %}
12721 #else
12722   predicate(n->as_Vector()->length() == 2);
12723   match(Set dst (ReplicateD src));
12724   size(4*2);
12725   ins_cost(DEFAULT_COST*2); // FIXME
12726 
12727   format %{ "FCPYD    $dst.D.a,$src\n\t"
12728             "FCPYD    $dst.D.b,$src\t" %}
12729   ins_encode %{
12730     FloatRegister dsta = $dst$$FloatRegister;
12731     FloatRegister src = $src$$FloatRegister;
12732     __ fcpyd(dsta, src);
12733     FloatRegister dstb = dsta->successor()->successor();
12734     __ fcpyd(dstb, src);
12735   %}
12736 #endif
12737   ins_pipe(ialu_reg); // FIXME
12738 %}
12739 
12740 // ====================VECTOR ARITHMETIC=======================================
12741 
12742 // --------------------------------- ADD --------------------------------------
12743 
12744 // Bytes vector add
12745 instruct vadd8B_reg(vecD dst, vecD src1, vecD src2) %{
12746   predicate(n->as_Vector()->length() == 8);
12747   match(Set dst (AddVB src1 src2));
12748   format %{ "VADD.I8 $dst,$src1,$src2\t! add packed8B" %}
12749   size(4);
12750   ins_encode %{
12751     bool quad = false;
12752     __ vaddI($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister,
12753              MacroAssembler::VELEM_SIZE_8, quad);
12754   %}
12755   ins_pipe( ialu_reg_reg ); // FIXME
12756 %}
12757 
12758 instruct vadd16B_reg(vecX dst, vecX src1, vecX src2) %{
12759   predicate(n->as_Vector()->length() == 16);
12760   match(Set dst (AddVB src1 src2));
12761   size(4);
12762   format %{ "VADD.I8 $dst.Q,$src1.Q,$src2.Q\t! add packed16B" %}
12763   ins_encode %{
12764     bool quad = true;
12765     __ vaddI($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister,
12766              MacroAssembler::VELEM_SIZE_8, quad);
12767   %}
12768   ins_pipe( ialu_reg_reg ); // FIXME
12769 %}
12770 
12771 // Shorts/Chars vector add
12772 instruct vadd4S_reg(vecD dst, vecD src1, vecD src2) %{
12773   predicate(n->as_Vector()->length() == 4);
12774   match(Set dst (AddVS src1 src2));
12775   size(4);
12776   format %{ "VADD.I16 $dst,$src1,$src2\t! add packed4S" %}
12777   ins_encode %{
12778     bool quad = false;
12779     __ vaddI($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister,
12780              MacroAssembler::VELEM_SIZE_16, quad);
12781   %}
12782   ins_pipe( ialu_reg_reg ); // FIXME
12783 %}
12784 
12785 instruct vadd8S_reg(vecX dst, vecX src1, vecX src2) %{
12786   predicate(n->as_Vector()->length() == 8);
12787   match(Set dst (AddVS src1 src2));
12788   size(4);
12789   format %{ "VADD.I16 $dst.Q,$src1.Q,$src2.Q\t! add packed8S" %}
12790   ins_encode %{
12791     bool quad = true;
12792     __ vaddI($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister,
12793              MacroAssembler::VELEM_SIZE_16, quad);
12794   %}
12795   ins_pipe( ialu_reg_reg ); // FIXME
12796 %}
12797 
12798 // Integers vector add
12799 instruct vadd2I_reg(vecD dst, vecD src1, vecD src2) %{
12800   predicate(n->as_Vector()->length() == 2);
12801   match(Set dst (AddVI src1 src2));
12802   size(4);
12803   format %{ "VADD.I32 $dst.D,$src1.D,$src2.D\t! add packed2I" %}
12804   ins_encode %{
12805     bool quad = false;
12806     __ vaddI($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister,
12807              MacroAssembler::VELEM_SIZE_32, quad);
12808   %}
12809   ins_pipe( ialu_reg_reg ); // FIXME
12810 %}
12811 
12812 instruct vadd4I_reg(vecX dst, vecX src1, vecX src2) %{
12813   predicate(n->as_Vector()->length() == 4);
12814   match(Set dst (AddVI src1 src2));
12815   size(4);
12816   format %{ "VADD.I32 $dst.Q,$src1.Q,$src2.Q\t! add packed4I" %}
12817   ins_encode %{
12818     bool quad = true;
12819     __ vaddI($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister,
12820              MacroAssembler::VELEM_SIZE_32, quad);
12821   %}
12822   ins_pipe( ialu_reg_reg ); // FIXME
12823 %}
12824 
12825 // Longs vector add
12826 instruct vadd2L_reg(vecX dst, vecX src1, vecX src2) %{
12827   predicate(n->as_Vector()->length() == 2);
12828   match(Set dst (AddVL src1 src2));
12829   size(4);
12830   format %{ "VADD.I64 $dst.Q,$src1.Q,$src2.Q\t! add packed2L" %}
12831   ins_encode %{
12832     bool quad = true;
12833     __ vaddI($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister,
12834              MacroAssembler::VELEM_SIZE_64, quad);
12835   %}
12836   ins_pipe( ialu_reg_reg ); // FIXME
12837 %}
12838 
12839 // Floats vector add
12840 instruct vadd2F_reg(vecD dst, vecD src1, vecD src2) %{
12841   predicate(n->as_Vector()->length() == 2 && VM_Version::simd_math_is_compliant());
12842   match(Set dst (AddVF src1 src2));
12843   size(4);
12844   format %{ "VADD.F32 $dst,$src1,$src2\t! add packed2F" %}
12845   ins_encode %{
12846     bool quad = false;
12847     __ vaddF($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister,
12848              MacroAssembler::VFA_SIZE_F32, quad);
12849   %}
12850   ins_pipe( faddD_reg_reg ); // FIXME
12851 %}
12852 
12853 #ifndef AARCH64
12854 instruct vadd2F_reg_vfp(vecD dst, vecD src1, vecD src2) %{
12855   predicate(n->as_Vector()->length() == 2 && !VM_Version::simd_math_is_compliant());
12856   match(Set dst (AddVF src1 src2));
12857   ins_cost(DEFAULT_COST*2); // FIXME
12858 
12859   size(4*2);
12860   format %{ "FADDS  $dst.a,$src1.a,$src2.a\n\t"
12861             "FADDS  $dst.b,$src1.b,$src2.b" %}
12862   ins_encode %{
12863     __ add_float($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister);
12864     __ add_float($dst$$FloatRegister->successor(),
12865              $src1$$FloatRegister->successor(),
12866              $src2$$FloatRegister->successor());
12867   %}
12868 
12869   ins_pipe(faddF_reg_reg); // FIXME
12870 %}
12871 #endif
12872 
12873 instruct vadd4F_reg_simd(vecX dst, vecX src1, vecX src2) %{
12874   predicate(n->as_Vector()->length() == 4 && VM_Version::simd_math_is_compliant());
12875   match(Set dst (AddVF src1 src2));
12876   size(4);
12877   format %{ "VADD.F32 $dst.Q,$src1.Q,$src2.Q\t! add packed4F" %}
12878   ins_encode %{
12879     bool quad = true;
12880     __ vaddF($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister,
12881              MacroAssembler::VFA_SIZE_F32, quad);
12882   %}
12883   ins_pipe( faddD_reg_reg ); // FIXME
12884 %}
12885 
12886 #ifdef AARCH64
12887 instruct vadd2D_reg_simd(vecX dst, vecX src1, vecX src2) %{
12888   predicate(n->as_Vector()->length() == 2 && VM_Version::simd_math_is_compliant());
12889   match(Set dst (AddVD src1 src2));
12890   size(4);
12891   format %{ "VADD.F64 $dst.Q,$src1.Q,$src2.Q\t! add packed2D" %}
12892   ins_encode %{
12893     bool quad = true;
12894     __ vaddF($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister,
12895              MacroAssembler::VFA_SIZE_F64, quad);
12896   %}
12897   ins_pipe( faddD_reg_reg ); // FIXME
12898 %}
12899 #else
12900 instruct vadd4F_reg_vfp(vecX dst, vecX src1, vecX src2) %{
12901   predicate(n->as_Vector()->length() == 4 && !VM_Version::simd_math_is_compliant());
12902   match(Set dst (AddVF src1 src2));
12903   size(4*4);
12904   ins_cost(DEFAULT_COST*4); // FIXME
12905 
12906   format %{ "FADDS  $dst.a,$src1.a,$src2.a\n\t"
12907             "FADDS  $dst.b,$src1.b,$src2.b\n\t"
12908             "FADDS  $dst.c,$src1.c,$src2.c\n\t"
12909             "FADDS  $dst.d,$src1.d,$src2.d" %}
12910 
12911   ins_encode %{
12912     FloatRegister dsta = $dst$$FloatRegister;
12913     FloatRegister src1a = $src1$$FloatRegister;
12914     FloatRegister src2a = $src2$$FloatRegister;
12915     __ add_float(dsta, src1a, src2a);
12916     FloatRegister dstb = dsta->successor();
12917     FloatRegister src1b = src1a->successor();
12918     FloatRegister src2b = src2a->successor();
12919     __ add_float(dstb, src1b, src2b);
12920     FloatRegister dstc = dstb->successor();
12921     FloatRegister src1c = src1b->successor();
12922     FloatRegister src2c = src2b->successor();
12923     __ add_float(dstc, src1c, src2c);
12924     FloatRegister dstd = dstc->successor();
12925     FloatRegister src1d = src1c->successor();
12926     FloatRegister src2d = src2c->successor();
12927     __ add_float(dstd, src1d, src2d);
12928   %}
12929 
12930   ins_pipe(faddF_reg_reg); // FIXME
12931 %}
12932 
12933 instruct vadd2D_reg_vfp(vecX dst, vecX src1, vecX src2) %{
12934   predicate(n->as_Vector()->length() == 2);
12935   match(Set dst (AddVD src1 src2));
12936   size(4*2);
12937   ins_cost(DEFAULT_COST*2); // FIXME
12938 
12939   format %{ "FADDD  $dst.a,$src1.a,$src2.a\n\t"
12940             "FADDD  $dst.b,$src1.b,$src2.b" %}
12941 
12942   ins_encode %{
12943     FloatRegister dsta = $dst$$FloatRegister;
12944     FloatRegister src1a = $src1$$FloatRegister;
12945     FloatRegister src2a = $src2$$FloatRegister;
12946     __ add_double(dsta, src1a, src2a);
12947     FloatRegister dstb = dsta->successor()->successor();
12948     FloatRegister src1b = src1a->successor()->successor();
12949     FloatRegister src2b = src2a->successor()->successor();
12950     __ add_double(dstb, src1b, src2b);
12951   %}
12952 
12953   ins_pipe(faddF_reg_reg); // FIXME
12954 %}
12955 #endif
12956 
12957 
12958 // Bytes vector sub
12959 instruct vsub8B_reg(vecD dst, vecD src1, vecD src2) %{
12960   predicate(n->as_Vector()->length() == 8);
12961   match(Set dst (SubVB src1 src2));
12962   size(4);
12963   format %{ "VSUB.I8 $dst,$src1,$src2\t! sub packed8B" %}
12964   ins_encode %{
12965     bool quad = false;
12966     __ vsubI($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister,
12967              MacroAssembler::VELEM_SIZE_8, quad);
12968   %}
12969   ins_pipe( ialu_reg_reg ); // FIXME
12970 %}
12971 
12972 instruct vsub16B_reg(vecX dst, vecX src1, vecX src2) %{
12973   predicate(n->as_Vector()->length() == 16);
12974   match(Set dst (SubVB src1 src2));
12975   size(4);
12976   format %{ "VSUB.I8 $dst.Q,$src1.Q,$src2.Q\t! sub packed16B" %}
12977   ins_encode %{
12978     bool quad = true;
12979     __ vsubI($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister,
12980              MacroAssembler::VELEM_SIZE_8, quad);
12981   %}
12982   ins_pipe( ialu_reg_reg ); // FIXME
12983 %}
12984 
12985 // Shorts/Chars vector sub
12986 instruct vsub4S_reg(vecD dst, vecD src1, vecD src2) %{
12987   predicate(n->as_Vector()->length() == 4);
12988   match(Set dst (SubVS src1 src2));
12989   size(4);
12990   format %{ "VSUB.I16 $dst,$src1,$src2\t! sub packed4S" %}
12991   ins_encode %{
12992     bool quad = false;
12993     __ vsubI($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister,
12994              MacroAssembler::VELEM_SIZE_16, quad);
12995   %}
12996   ins_pipe( ialu_reg_reg ); // FIXME
12997 %}
12998 
12999 instruct vsub16S_reg(vecX dst, vecX src1, vecX src2) %{
13000   predicate(n->as_Vector()->length() == 8);
13001   match(Set dst (SubVS src1 src2));
13002   size(4);
13003   format %{ "VSUB.I16 $dst.Q,$src1.Q,$src2.Q\t! sub packed8S" %}
13004   ins_encode %{
13005     bool quad = true;
13006     __ vsubI($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister,
13007              MacroAssembler::VELEM_SIZE_16, quad);
13008   %}
13009   ins_pipe( ialu_reg_reg ); // FIXME
13010 %}
13011 
13012 // Integers vector sub
13013 instruct vsub2I_reg(vecD dst, vecD src1, vecD src2) %{
13014   predicate(n->as_Vector()->length() == 2);
13015   match(Set dst (SubVI src1 src2));
13016   size(4);
13017   format %{ "VSUB.I32 $dst,$src1,$src2\t! sub packed2I" %}
13018   ins_encode %{
13019     bool quad = false;
13020     __ vsubI($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister,
13021              MacroAssembler::VELEM_SIZE_32, quad);
13022   %}
13023   ins_pipe( ialu_reg_reg ); // FIXME
13024 %}
13025 
13026 instruct vsub4I_reg(vecX dst, vecX src1, vecX src2) %{
13027   predicate(n->as_Vector()->length() == 4);
13028   match(Set dst (SubVI src1 src2));
13029   size(4);
13030   format %{ "VSUB.I32 $dst.Q,$src1.Q,$src2.Q\t! sub packed4I" %}
13031   ins_encode %{
13032     bool quad = true;
13033     __ vsubI($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister,
13034              MacroAssembler::VELEM_SIZE_32, quad);
13035   %}
13036   ins_pipe( ialu_reg_reg ); // FIXME
13037 %}
13038 
13039 // Longs vector sub
13040 instruct vsub2L_reg(vecX dst, vecX src1, vecX src2) %{
13041   predicate(n->as_Vector()->length() == 2);
13042   match(Set dst (SubVL src1 src2));
13043   size(4);
13044   format %{ "VSUB.I64 $dst.Q,$src1.Q,$src2.Q\t! sub packed2L" %}
13045   ins_encode %{
13046     bool quad = true;
13047     __ vsubI($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister,
13048              MacroAssembler::VELEM_SIZE_64, quad);
13049   %}
13050   ins_pipe( ialu_reg_reg ); // FIXME
13051 %}
13052 
13053 // Floats vector sub
13054 instruct vsub2F_reg(vecD dst, vecD src1, vecD src2) %{
13055   predicate(n->as_Vector()->length() == 2 && VM_Version::simd_math_is_compliant());
13056   match(Set dst (SubVF src1 src2));
13057   size(4);
13058   format %{ "VSUB.F32 $dst,$src1,$src2\t! sub packed2F" %}
13059   ins_encode %{
13060     bool quad = false;
13061     __ vsubF($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister,
13062              MacroAssembler::VFA_SIZE_F32, quad);
13063   %}
13064   ins_pipe( faddF_reg_reg ); // FIXME
13065 %}
13066 
13067 #ifndef AARCH64
13068 instruct vsub2F_reg_vfp(vecD dst, vecD src1, vecD src2) %{
13069   predicate(n->as_Vector()->length() == 2 && !VM_Version::simd_math_is_compliant());
13070   match(Set dst (SubVF src1 src2));
13071   size(4*2);
13072   ins_cost(DEFAULT_COST*2); // FIXME
13073 
13074   format %{ "FSUBS  $dst.a,$src1.a,$src2.a\n\t"
13075             "FSUBS  $dst.b,$src1.b,$src2.b" %}
13076 
13077   ins_encode %{
13078     FloatRegister dsta = $dst$$FloatRegister;
13079     FloatRegister src1a = $src1$$FloatRegister;
13080     FloatRegister src2a = $src2$$FloatRegister;
13081     __ sub_float(dsta, src1a, src2a);
13082     FloatRegister dstb = dsta->successor();
13083     FloatRegister src1b = src1a->successor();
13084     FloatRegister src2b = src2a->successor();
13085     __ sub_float(dstb, src1b, src2b);
13086   %}
13087 
13088   ins_pipe(faddF_reg_reg); // FIXME
13089 %}
13090 #endif
13091 
13092 
13093 instruct vsub4F_reg(vecX dst, vecX src1, vecX src2) %{
13094   predicate(n->as_Vector()->length() == 4 && VM_Version::simd_math_is_compliant());
13095   match(Set dst (SubVF src1 src2));
13096   size(4);
13097   format %{ "VSUB.F32 $dst.Q,$src1.Q,$src2.Q\t! sub packed4F" %}
13098   ins_encode %{
13099     bool quad = true;
13100     __ vsubF($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister,
13101              MacroAssembler::VFA_SIZE_F32, quad);
13102   %}
13103   ins_pipe( faddF_reg_reg ); // FIXME
13104 %}
13105 
13106 #ifdef AARCH64
13107 instruct vsub2D_reg_simd(vecX dst, vecX src1, vecX src2) %{
13108   predicate(n->as_Vector()->length() == 2 && VM_Version::simd_math_is_compliant());
13109   match(Set dst (SubVD src1 src2));
13110   size(4);
13111   format %{ "VSUB.F64 $dst.Q,$src1.Q,$src2.Q\t! add packed2D" %}
13112   ins_encode %{
13113     bool quad = true;
13114     __ vsubF($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister,
13115              MacroAssembler::VFA_SIZE_F64, quad);
13116   %}
13117   ins_pipe( faddD_reg_reg ); // FIXME
13118 %}
13119 #else
13120 instruct vsub4F_reg_vfp(vecX dst, vecX src1, vecX src2) %{
13121   predicate(n->as_Vector()->length() == 4 && !VM_Version::simd_math_is_compliant());
13122   match(Set dst (SubVF src1 src2));
13123   size(4*4);
13124   ins_cost(DEFAULT_COST*4); // FIXME
13125 
13126   format %{ "FSUBS  $dst.a,$src1.a,$src2.a\n\t"
13127             "FSUBS  $dst.b,$src1.b,$src2.b\n\t"
13128             "FSUBS  $dst.c,$src1.c,$src2.c\n\t"
13129             "FSUBS  $dst.d,$src1.d,$src2.d" %}
13130 
13131   ins_encode %{
13132     FloatRegister dsta = $dst$$FloatRegister;
13133     FloatRegister src1a = $src1$$FloatRegister;
13134     FloatRegister src2a = $src2$$FloatRegister;
13135     __ sub_float(dsta, src1a, src2a);
13136     FloatRegister dstb = dsta->successor();
13137     FloatRegister src1b = src1a->successor();
13138     FloatRegister src2b = src2a->successor();
13139     __ sub_float(dstb, src1b, src2b);
13140     FloatRegister dstc = dstb->successor();
13141     FloatRegister src1c = src1b->successor();
13142     FloatRegister src2c = src2b->successor();
13143     __ sub_float(dstc, src1c, src2c);
13144     FloatRegister dstd = dstc->successor();
13145     FloatRegister src1d = src1c->successor();
13146     FloatRegister src2d = src2c->successor();
13147     __ sub_float(dstd, src1d, src2d);
13148   %}
13149 
13150   ins_pipe(faddF_reg_reg); // FIXME
13151 %}
13152 
13153 instruct vsub2D_reg_vfp(vecX dst, vecX src1, vecX src2) %{
13154   predicate(n->as_Vector()->length() == 2);
13155   match(Set dst (SubVD src1 src2));
13156   size(4*2);
13157   ins_cost(DEFAULT_COST*2); // FIXME
13158 
13159   format %{ "FSUBD  $dst.a,$src1.a,$src2.a\n\t"
13160             "FSUBD  $dst.b,$src1.b,$src2.b" %}
13161 
13162   ins_encode %{
13163     FloatRegister dsta = $dst$$FloatRegister;
13164     FloatRegister src1a = $src1$$FloatRegister;
13165     FloatRegister src2a = $src2$$FloatRegister;
13166     __ sub_double(dsta, src1a, src2a);
13167     FloatRegister dstb = dsta->successor()->successor();
13168     FloatRegister src1b = src1a->successor()->successor();
13169     FloatRegister src2b = src2a->successor()->successor();
13170     __ sub_double(dstb, src1b, src2b);
13171   %}
13172 
13173   ins_pipe(faddF_reg_reg); // FIXME
13174 %}
13175 #endif
13176 
13177 // Shorts/Chars vector mul
13178 instruct vmul4S_reg(vecD dst, vecD src1, vecD src2) %{
13179   predicate(n->as_Vector()->length() == 4);
13180   match(Set dst (MulVS src1 src2));
13181   size(4);
13182   format %{ "VMUL.I16 $dst,$src1,$src2\t! mul packed4S" %}
13183   ins_encode %{
13184     __ vmulI($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister,
13185              MacroAssembler::VELEM_SIZE_16, 0);
13186   %}
13187   ins_pipe( ialu_reg_reg ); // FIXME
13188 %}
13189 
13190 instruct vmul8S_reg(vecX dst, vecX src1, vecX src2) %{
13191   predicate(n->as_Vector()->length() == 8);
13192   match(Set dst (MulVS src1 src2));
13193   size(4);
13194   format %{ "VMUL.I16 $dst.Q,$src1.Q,$src2.Q\t! mul packed8S" %}
13195   ins_encode %{
13196     __ vmulI($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister,
13197              MacroAssembler::VELEM_SIZE_16, 1);
13198   %}
13199   ins_pipe( ialu_reg_reg ); // FIXME
13200 %}
13201 
13202 // Integers vector mul
13203 instruct vmul2I_reg(vecD dst, vecD src1, vecD src2) %{
13204   predicate(n->as_Vector()->length() == 2);
13205   match(Set dst (MulVI src1 src2));
13206   size(4);
13207   format %{ "VMUL.I32 $dst,$src1,$src2\t! mul packed2I" %}
13208   ins_encode %{
13209     __ vmulI($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister,
13210              MacroAssembler::VELEM_SIZE_32, 0);
13211   %}
13212   ins_pipe( ialu_reg_reg ); // FIXME
13213 %}
13214 
13215 instruct vmul4I_reg(vecX dst, vecX src1, vecX src2) %{
13216   predicate(n->as_Vector()->length() == 4);
13217   match(Set dst (MulVI src1 src2));
13218   size(4);
13219   format %{ "VMUL.I32 $dst.Q,$src1.Q,$src2.Q\t! mul packed4I" %}
13220   ins_encode %{
13221     __ vmulI($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister,
13222              MacroAssembler::VELEM_SIZE_32, 1);
13223   %}
13224   ins_pipe( ialu_reg_reg ); // FIXME
13225 %}
13226 
13227 // Floats vector mul
13228 instruct vmul2F_reg(vecD dst, vecD src1, vecD src2) %{
13229   predicate(n->as_Vector()->length() == 2 && VM_Version::simd_math_is_compliant());
13230   match(Set dst (MulVF src1 src2));
13231   size(4);
13232   format %{ "VMUL.F32 $dst,$src1,$src2\t! mul packed2F" %}
13233   ins_encode %{
13234     __ vmulF($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister,
13235              MacroAssembler::VFA_SIZE_F32, 0);
13236   %}
13237   ins_pipe( fmulF_reg_reg ); // FIXME
13238 %}
13239 
13240 #ifndef AARCH64
13241 instruct vmul2F_reg_vfp(vecD dst, vecD src1, vecD src2) %{
13242   predicate(n->as_Vector()->length() == 2 && !VM_Version::simd_math_is_compliant());
13243   match(Set dst (MulVF src1 src2));
13244   size(4*2);
13245   ins_cost(DEFAULT_COST*2); // FIXME
13246 
13247   format %{ "FMULS  $dst.a,$src1.a,$src2.a\n\t"
13248             "FMULS  $dst.b,$src1.b,$src2.b" %}
13249   ins_encode %{
13250     __ mul_float($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister);
13251     __ mul_float($dst$$FloatRegister->successor(),
13252              $src1$$FloatRegister->successor(),
13253              $src2$$FloatRegister->successor());
13254   %}
13255 
13256   ins_pipe(fmulF_reg_reg); // FIXME
13257 %}
13258 #endif
13259 
13260 instruct vmul4F_reg(vecX dst, vecX src1, vecX src2) %{
13261   predicate(n->as_Vector()->length() == 4 && VM_Version::simd_math_is_compliant());
13262   match(Set dst (MulVF src1 src2));
13263   size(4);
13264   format %{ "VMUL.F32 $dst.Q,$src1.Q,$src2.Q\t! mul packed4F" %}
13265   ins_encode %{
13266     __ vmulF($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister,
13267              MacroAssembler::VFA_SIZE_F32, 1);
13268   %}
13269   ins_pipe( fmulF_reg_reg ); // FIXME
13270 %}
13271 
13272 #ifndef AARCH64
13273 instruct vmul4F_reg_vfp(vecX dst, vecX src1, vecX src2) %{
13274   predicate(n->as_Vector()->length() == 4 && !VM_Version::simd_math_is_compliant());
13275   match(Set dst (MulVF src1 src2));
13276   size(4*4);
13277   ins_cost(DEFAULT_COST*4); // FIXME
13278 
13279   format %{ "FMULS  $dst.a,$src1.a,$src2.a\n\t"
13280             "FMULS  $dst.b,$src1.b,$src2.b\n\t"
13281             "FMULS  $dst.c,$src1.c,$src2.c\n\t"
13282             "FMULS  $dst.d,$src1.d,$src2.d" %}
13283 
13284   ins_encode %{
13285     FloatRegister dsta = $dst$$FloatRegister;
13286     FloatRegister src1a = $src1$$FloatRegister;
13287     FloatRegister src2a = $src2$$FloatRegister;
13288     __ mul_float(dsta, src1a, src2a);
13289     FloatRegister dstb = dsta->successor();
13290     FloatRegister src1b = src1a->successor();
13291     FloatRegister src2b = src2a->successor();
13292     __ mul_float(dstb, src1b, src2b);
13293     FloatRegister dstc = dstb->successor();
13294     FloatRegister src1c = src1b->successor();
13295     FloatRegister src2c = src2b->successor();
13296     __ mul_float(dstc, src1c, src2c);
13297     FloatRegister dstd = dstc->successor();
13298     FloatRegister src1d = src1c->successor();
13299     FloatRegister src2d = src2c->successor();
13300     __ mul_float(dstd, src1d, src2d);
13301   %}
13302 
13303   ins_pipe(fmulF_reg_reg); // FIXME
13304 %}
13305 #endif
13306 
13307 #ifdef AARCH64
13308 instruct vmul2D_reg(vecX dst, vecX src1, vecX src2) %{
13309   predicate(n->as_Vector()->length() == 2 && VM_Version::has_simd());
13310   match(Set dst (MulVD src1 src2));
13311   size(4*1);
13312   ins_cost(DEFAULT_COST*1); // FIXME
13313 
13314   format %{ "FMUL.2D $dst,$src1,$src2\t! double[2]" %}
13315   ins_encode %{
13316     int quad = 1;
13317     __ vmulF($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister,
13318              MacroAssembler::VFA_SIZE_F64, quad);
13319   %}
13320 
13321   ins_pipe(fdivF_reg_reg); // FIXME
13322 %}
13323 #else
13324 instruct vmul2D_reg_vfp(vecX dst, vecX src1, vecX src2) %{
13325   predicate(n->as_Vector()->length() == 2);
13326   match(Set dst (MulVD src1 src2));
13327   size(4*2);
13328   ins_cost(DEFAULT_COST*2); // FIXME
13329 
13330   format %{ "FMULD  $dst.D.a,$src1.D.a,$src2.D.a\n\t"
13331             "FMULD  $dst.D.b,$src1.D.b,$src2.D.b" %}
13332   ins_encode %{
13333     FloatRegister dsta = $dst$$FloatRegister;
13334     FloatRegister src1a = $src1$$FloatRegister;
13335     FloatRegister src2a = $src2$$FloatRegister;
13336     __ mul_double(dsta, src1a, src2a);
13337     FloatRegister dstb = dsta->successor()->successor();
13338     FloatRegister src1b = src1a->successor()->successor();
13339     FloatRegister src2b = src2a->successor()->successor();
13340     __ mul_double(dstb, src1b, src2b);
13341   %}
13342 
13343   ins_pipe(fmulD_reg_reg); // FIXME
13344 %}
13345 #endif
13346 
13347 
13348 // Floats vector div
13349 instruct vdiv2F_reg_vfp(vecD dst, vecD src1, vecD src2) %{
13350   predicate(n->as_Vector()->length() == 2);
13351   match(Set dst (DivVF src1 src2));
13352 #ifdef AARCH64
13353   size(4*1);
13354   ins_cost(DEFAULT_COST*1); // FIXME
13355 
13356   format %{ "FDIV.2S $dst,$src1,$src2\t! float[2]" %}
13357   ins_encode %{
13358     int quad = 0;
13359     __ vdivF($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister,
13360              MacroAssembler::VFA_SIZE_F32, quad);
13361   %}
13362 
13363   ins_pipe(fdivF_reg_reg); // FIXME
13364 #else
13365   size(4*2);
13366   ins_cost(DEFAULT_COST*2); // FIXME
13367 
13368   format %{ "FDIVS  $dst.a,$src1.a,$src2.a\n\t"
13369             "FDIVS  $dst.b,$src1.b,$src2.b" %}
13370   ins_encode %{
13371     __ div_float($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister);
13372     __ div_float($dst$$FloatRegister->successor(),
13373              $src1$$FloatRegister->successor(),
13374              $src2$$FloatRegister->successor());
13375   %}
13376 
13377   ins_pipe(fdivF_reg_reg); // FIXME
13378 #endif
13379 %}
13380 
13381 instruct vdiv4F_reg_vfp(vecX dst, vecX src1, vecX src2) %{
13382   predicate(n->as_Vector()->length() == 4);
13383   match(Set dst (DivVF src1 src2));
13384 #ifdef AARCH64
13385   size(4*1);
13386   ins_cost(DEFAULT_COST*1); // FIXME
13387 
13388   format %{ "FDIV.4S $dst,$src1,$src2\t! float[4]" %}
13389   ins_encode %{
13390     int quad = 1;
13391     __ vdivF($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister,
13392              MacroAssembler::VFA_SIZE_F32, quad);
13393   %}
13394 
13395   ins_pipe(fdivF_reg_reg); // FIXME
13396 #else
13397   size(4*4);
13398   ins_cost(DEFAULT_COST*4); // FIXME
13399 
13400   format %{ "FDIVS  $dst.a,$src1.a,$src2.a\n\t"
13401             "FDIVS  $dst.b,$src1.b,$src2.b\n\t"
13402             "FDIVS  $dst.c,$src1.c,$src2.c\n\t"
13403             "FDIVS  $dst.d,$src1.d,$src2.d" %}
13404 
13405   ins_encode %{
13406     FloatRegister dsta = $dst$$FloatRegister;
13407     FloatRegister src1a = $src1$$FloatRegister;
13408     FloatRegister src2a = $src2$$FloatRegister;
13409     __ div_float(dsta, src1a, src2a);
13410     FloatRegister dstb = dsta->successor();
13411     FloatRegister src1b = src1a->successor();
13412     FloatRegister src2b = src2a->successor();
13413     __ div_float(dstb, src1b, src2b);
13414     FloatRegister dstc = dstb->successor();
13415     FloatRegister src1c = src1b->successor();
13416     FloatRegister src2c = src2b->successor();
13417     __ div_float(dstc, src1c, src2c);
13418     FloatRegister dstd = dstc->successor();
13419     FloatRegister src1d = src1c->successor();
13420     FloatRegister src2d = src2c->successor();
13421     __ div_float(dstd, src1d, src2d);
13422   %}
13423 
13424   ins_pipe(fdivF_reg_reg); // FIXME
13425 #endif
13426 %}
13427 
13428 #ifdef AARCH64
13429 instruct vdiv2D_reg(vecX dst, vecX src1, vecX src2) %{
13430   predicate(n->as_Vector()->length() == 2 && VM_Version::has_simd());
13431   match(Set dst (DivVD src1 src2));
13432   size(4*1);
13433   ins_cost(DEFAULT_COST*1); // FIXME
13434 
13435   format %{ "FDIV.2D $dst,$src1,$src2\t! double[2]" %}
13436   ins_encode %{
13437     int quad = 1;
13438     __ vdivF($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister,
13439              MacroAssembler::VFA_SIZE_F64, quad);
13440   %}
13441 
13442   ins_pipe(fdivF_reg_reg); // FIXME
13443 %}
13444 #else
13445 instruct vdiv2D_reg_vfp(vecX dst, vecX src1, vecX src2) %{
13446   predicate(n->as_Vector()->length() == 2);
13447   match(Set dst (DivVD src1 src2));
13448   size(4*2);
13449   ins_cost(DEFAULT_COST*2); // FIXME
13450 
13451   format %{ "FDIVD  $dst.D.a,$src1.D.a,$src2.D.a\n\t"
13452             "FDIVD  $dst.D.b,$src1.D.b,$src2.D.b" %}
13453   ins_encode %{
13454     FloatRegister dsta = $dst$$FloatRegister;
13455     FloatRegister src1a = $src1$$FloatRegister;
13456     FloatRegister src2a = $src2$$FloatRegister;
13457     __ div_double(dsta, src1a, src2a);
13458     FloatRegister dstb = dsta->successor()->successor();
13459     FloatRegister src1b = src1a->successor()->successor();
13460     FloatRegister src2b = src2a->successor()->successor();
13461     __ div_double(dstb, src1b, src2b);
13462   %}
13463 
13464   ins_pipe(fdivD_reg_reg); // FIXME
13465 %}
13466 #endif
13467 
13468 // --------------------------------- NEG --------------------------------------
13469 
13470 instruct vneg8B_reg(vecD dst, vecD src) %{
13471   predicate(n->as_Vector()->length_in_bytes() == 8);
13472   effect(DEF dst, USE src);
13473   size(4);
13474   ins_cost(DEFAULT_COST); // FIXME
13475   format %{ "VNEG.S8 $dst.D,$src.D\t! neg packed8B" %}
13476   ins_encode %{
13477     bool quad = false;
13478     __ vnegI($dst$$FloatRegister, $src$$FloatRegister,
13479               MacroAssembler::VELEM_SIZE_8, quad);
13480   %}
13481   ins_pipe( ialu_reg_reg ); // FIXME
13482 %}
13483 
13484 instruct vneg16B_reg(vecX dst, vecX src) %{
13485   predicate(n->as_Vector()->length_in_bytes() == 16);
13486   effect(DEF dst, USE src);
13487   size(4);
13488   ins_cost(DEFAULT_COST); // FIXME
13489   format %{ "VNEG.S8 $dst.Q,$src.Q\t! neg0 packed16B" %}
13490   ins_encode %{
13491     bool _float = false;
13492     bool quad = true;
13493     __ vnegI($dst$$FloatRegister, $src$$FloatRegister,
13494               MacroAssembler::VELEM_SIZE_8, quad);
13495   %}
13496   ins_pipe( ialu_reg_reg ); // FIXME
13497 %}
13498 
13499 // ------------------------------ Shift ---------------------------------------
13500 
13501 instruct vslcntD(vecD dst, iRegI cnt) %{
13502   predicate(n->as_Vector()->length_in_bytes() == 8 && VM_Version::has_simd());
13503   match(Set dst (LShiftCntV cnt));
13504   size(4);
13505   ins_cost(DEFAULT_COST); // FIXME
13506   expand %{
13507     Repl8B_reg_simd(dst, cnt);
13508   %}
13509 %}
13510 
13511 instruct vslcntX(vecX dst, iRegI cnt) %{
13512   predicate(n->as_Vector()->length_in_bytes() == 16 && VM_Version::has_simd());
13513   match(Set dst (LShiftCntV cnt));
13514   size(4);
13515   ins_cost(DEFAULT_COST); // FIXME
13516   expand %{
13517     Repl16B_reg(dst, cnt);
13518   %}
13519 %}
13520 
13521 // Low bits of vector "shift" elements are used, so it
13522 // doesn't matter if we treat it as ints or bytes here.
13523 instruct vsrcntD(vecD dst, iRegI cnt) %{
13524   predicate(n->as_Vector()->length_in_bytes() == 8 && VM_Version::has_simd());
13525   match(Set dst (RShiftCntV cnt));
13526   size(4*2);
13527   ins_cost(DEFAULT_COST*2); // FIXME
13528 
13529   format %{ "VDUP.8 $dst.D,$cnt\n\t"
13530             "VNEG.S8 $dst.D,$dst.D\t! neg packed8B" %}
13531   ins_encode %{
13532     bool quad = false;
13533     __ vdupI($dst$$FloatRegister, $cnt$$Register,
13534              MacroAssembler::VELEM_SIZE_8, quad);
13535     __ vnegI($dst$$FloatRegister, $dst$$FloatRegister,
13536               MacroAssembler::VELEM_SIZE_8, quad);
13537   %}
13538   ins_pipe( ialu_reg_reg ); // FIXME
13539 %}
13540 
13541 instruct vsrcntX(vecX dst, iRegI cnt) %{
13542   predicate(n->as_Vector()->length_in_bytes() == 16 && VM_Version::has_simd());
13543   match(Set dst (RShiftCntV cnt));
13544   size(4*2);
13545   ins_cost(DEFAULT_COST*2); // FIXME
13546   format %{ "VDUP.8 $dst.Q,$cnt\n\t"
13547             "VNEG.S8 $dst.Q,$dst.Q\t! neg packed16B" %}
13548   ins_encode %{
13549     bool quad = true;
13550     __ vdupI($dst$$FloatRegister, $cnt$$Register,
13551              MacroAssembler::VELEM_SIZE_8, quad);
13552     __ vnegI($dst$$FloatRegister, $dst$$FloatRegister,
13553               MacroAssembler::VELEM_SIZE_8, quad);
13554   %}
13555   ins_pipe( ialu_reg_reg ); // FIXME
13556 %}
13557 
13558 // Byte vector logical left/right shift based on sign
13559 instruct vsh8B_reg(vecD dst, vecD src, vecD shift) %{
13560   predicate(n->as_Vector()->length() == 8);
13561   effect(DEF dst, USE src, USE shift);
13562   size(4);
13563   ins_cost(DEFAULT_COST); // FIXME
13564   format %{
13565     "VSHL.U8 $dst.D,$src.D,$shift.D\t! logical left/right shift packed8B"
13566   %}
13567   ins_encode %{
13568     bool quad = false;
13569     __ vshlUI($dst$$FloatRegister, $shift$$FloatRegister, $src$$FloatRegister,
13570               MacroAssembler::VELEM_SIZE_8, quad);
13571   %}
13572   ins_pipe( ialu_reg_reg ); // FIXME
13573 %}
13574 
13575 instruct vsh16B_reg(vecX dst, vecX src, vecX shift) %{
13576   predicate(n->as_Vector()->length() == 16);
13577   effect(DEF dst, USE src, USE shift);
13578   size(4);
13579   ins_cost(DEFAULT_COST); // FIXME
13580   format %{
13581     "VSHL.U8 $dst.Q,$src.Q,$shift.Q\t! logical left/right shift packed16B"
13582   %}
13583   ins_encode %{
13584     bool quad = true;
13585     __ vshlUI($dst$$FloatRegister, $shift$$FloatRegister, $src$$FloatRegister,
13586               MacroAssembler::VELEM_SIZE_8, quad);
13587   %}
13588   ins_pipe( ialu_reg_reg ); // FIXME
13589 %}
13590 
13591 // Shorts/Char vector logical left/right shift based on sign
13592 instruct vsh4S_reg(vecD dst, vecD src, vecD shift) %{
13593   predicate(n->as_Vector()->length() == 4);
13594   effect(DEF dst, USE src, USE shift);
13595   size(4);
13596   ins_cost(DEFAULT_COST); // FIXME
13597   format %{
13598     "VSHL.U16 $dst.D,$src.D,$shift.D\t! logical left/right shift packed4S"
13599   %}
13600   ins_encode %{
13601     bool quad = false;
13602     __ vshlUI($dst$$FloatRegister, $shift$$FloatRegister, $src$$FloatRegister,
13603               MacroAssembler::VELEM_SIZE_16, quad);
13604   %}
13605   ins_pipe( ialu_reg_reg ); // FIXME
13606 %}
13607 
13608 instruct vsh8S_reg(vecX dst, vecX src, vecX shift) %{
13609   predicate(n->as_Vector()->length() == 8);
13610   effect(DEF dst, USE src, USE shift);
13611   size(4);
13612   ins_cost(DEFAULT_COST); // FIXME
13613   format %{
13614     "VSHL.U16 $dst.Q,$src.Q,$shift.Q\t! logical left/right shift packed8S"
13615   %}
13616   ins_encode %{
13617     bool quad = true;
13618     __ vshlUI($dst$$FloatRegister, $shift$$FloatRegister, $src$$FloatRegister,
13619               MacroAssembler::VELEM_SIZE_16, quad);
13620   %}
13621   ins_pipe( ialu_reg_reg ); // FIXME
13622 %}
13623 
13624 // Integers vector logical left/right shift based on sign
13625 instruct vsh2I_reg(vecD dst, vecD src, vecD shift) %{
13626   predicate(n->as_Vector()->length() == 2);
13627   effect(DEF dst, USE src, USE shift);
13628   size(4);
13629   ins_cost(DEFAULT_COST); // FIXME
13630   format %{
13631     "VSHL.U32 $dst.D,$src.D,$shift.D\t! logical left/right shift packed2I"
13632   %}
13633   ins_encode %{
13634     bool quad = false;
13635     __ vshlUI($dst$$FloatRegister, $shift$$FloatRegister, $src$$FloatRegister,
13636               MacroAssembler::VELEM_SIZE_32, quad);
13637   %}
13638   ins_pipe( ialu_reg_reg ); // FIXME
13639 %}
13640 
13641 instruct vsh4I_reg(vecX dst, vecX src, vecX shift) %{
13642   predicate(n->as_Vector()->length() == 4);
13643   effect(DEF dst, USE src, USE shift);
13644   size(4);
13645   ins_cost(DEFAULT_COST); // FIXME
13646   format %{
13647     "VSHL.U32 $dst.Q,$src.Q,$shift.Q\t! logical left/right shift packed4I"
13648   %}
13649   ins_encode %{
13650     bool quad = true;
13651     __ vshlUI($dst$$FloatRegister, $shift$$FloatRegister, $src$$FloatRegister,
13652               MacroAssembler::VELEM_SIZE_32, quad);
13653   %}
13654   ins_pipe( ialu_reg_reg ); // FIXME
13655 %}
13656 
13657 // Longs vector logical left/right shift based on sign
13658 instruct vsh2L_reg(vecX dst, vecX src, vecX shift) %{
13659   predicate(n->as_Vector()->length() == 2);
13660   effect(DEF dst, USE src, USE shift);
13661   size(4);
13662   ins_cost(DEFAULT_COST); // FIXME
13663   format %{
13664     "VSHL.U64 $dst.Q,$src.Q,$shift.Q\t! logical left/right shift packed2L"
13665   %}
13666   ins_encode %{
13667     bool quad = true;
13668     __ vshlUI($dst$$FloatRegister, $shift$$FloatRegister, $src$$FloatRegister,
13669               MacroAssembler::VELEM_SIZE_64, quad);
13670   %}
13671   ins_pipe( ialu_reg_reg ); // FIXME
13672 %}
13673 
13674 // ------------------------------ LeftShift -----------------------------------
13675 
13676 // Byte vector left shift
13677 instruct vsl8B_reg(vecD dst, vecD src, vecD shift) %{
13678   predicate(n->as_Vector()->length() == 8);
13679   match(Set dst (LShiftVB src shift));
13680   size(4*1);
13681   ins_cost(DEFAULT_COST*1); // FIXME
13682   expand %{
13683     vsh8B_reg(dst, src, shift);
13684   %}
13685 %}
13686 
13687 instruct vsl16B_reg(vecX dst, vecX src, vecX shift) %{
13688   predicate(n->as_Vector()->length() == 16);
13689   match(Set dst (LShiftVB src shift));
13690   size(4*1);
13691   ins_cost(DEFAULT_COST*1); // FIXME
13692   expand %{
13693     vsh16B_reg(dst, src, shift);
13694   %}
13695 %}
13696 
13697 instruct vsl8B_immI(vecD dst, vecD src, immI shift) %{
13698   predicate(n->as_Vector()->length() == 8);
13699   match(Set dst (LShiftVB src shift));
13700   size(4);
13701   ins_cost(DEFAULT_COST); // FIXME
13702   format %{
13703     "VSHL.I8 $dst.D,$src.D,$shift\t! logical left shift packed8B"
13704   %}
13705   ins_encode %{
13706     bool quad = false;
13707     __ vshli($dst$$FloatRegister, $src$$FloatRegister, 8, $shift$$constant,
13708              quad);
13709   %}
13710   ins_pipe( ialu_reg_reg ); // FIXME
13711 %}
13712 
13713 instruct vsl16B_immI(vecX dst, vecX src, immI shift) %{
13714   predicate(n->as_Vector()->length() == 16);
13715   match(Set dst (LShiftVB src shift));
13716   size(4);
13717   ins_cost(DEFAULT_COST); // FIXME
13718   format %{
13719     "VSHL.I8 $dst.Q,$src.Q,$shift\t! logical left shift packed16B"
13720   %}
13721   ins_encode %{
13722     bool quad = true;
13723     __ vshli($dst$$FloatRegister, $src$$FloatRegister, 8, $shift$$constant,
13724              quad);
13725   %}
13726   ins_pipe( ialu_reg_reg ); // FIXME
13727 %}
13728 
13729 // Shorts/Chars vector logical left/right shift
13730 instruct vsl4S_reg(vecD dst, vecD src, vecD shift) %{
13731   predicate(n->as_Vector()->length() == 4);
13732   match(Set dst (LShiftVS src shift));
13733   match(Set dst (URShiftVS src shift));
13734   size(4*1);
13735   ins_cost(DEFAULT_COST*1); // FIXME
13736   expand %{
13737     vsh4S_reg(dst, src, shift);
13738   %}
13739 %}
13740 
13741 instruct vsl8S_reg(vecX dst, vecX src, vecX shift) %{
13742   predicate(n->as_Vector()->length() == 8);
13743   match(Set dst (LShiftVS src shift));
13744   match(Set dst (URShiftVS src shift));
13745   size(4*1);
13746   ins_cost(DEFAULT_COST*1); // FIXME
13747   expand %{
13748     vsh8S_reg(dst, src, shift);
13749   %}
13750 %}
13751 
13752 instruct vsl4S_immI(vecD dst, vecD src, immI shift) %{
13753   predicate(n->as_Vector()->length() == 4);
13754   match(Set dst (LShiftVS src shift));
13755   size(4);
13756   ins_cost(DEFAULT_COST); // FIXME
13757   format %{
13758     "VSHL.I16 $dst.D,$src.D,$shift\t! logical left shift packed4S"
13759   %}
13760   ins_encode %{
13761     bool quad = false;
13762     __ vshli($dst$$FloatRegister, $src$$FloatRegister, 16, $shift$$constant,
13763              quad);
13764   %}
13765   ins_pipe( ialu_reg_reg ); // FIXME
13766 %}
13767 
13768 instruct vsl8S_immI(vecX dst, vecX src, immI shift) %{
13769   predicate(n->as_Vector()->length() == 8);
13770   match(Set dst (LShiftVS src shift));
13771   size(4);
13772   ins_cost(DEFAULT_COST); // FIXME
13773   format %{
13774     "VSHL.I16 $dst.Q,$src.Q,$shift\t! logical left shift packed8S"
13775   %}
13776   ins_encode %{
13777     bool quad = true;
13778     __ vshli($dst$$FloatRegister, $src$$FloatRegister, 16, $shift$$constant,
13779              quad);
13780   %}
13781   ins_pipe( ialu_reg_reg ); // FIXME
13782 %}
13783 
13784 // Integers vector logical left/right shift
13785 instruct vsl2I_reg(vecD dst, vecD src, vecD shift) %{
13786   predicate(n->as_Vector()->length() == 2 && VM_Version::has_simd());
13787   match(Set dst (LShiftVI src shift));
13788   match(Set dst (URShiftVI src shift));
13789   size(4*1);
13790   ins_cost(DEFAULT_COST*1); // FIXME
13791   expand %{
13792     vsh2I_reg(dst, src, shift);
13793   %}
13794 %}
13795 
13796 instruct vsl4I_reg(vecX dst, vecX src, vecX shift) %{
13797   predicate(n->as_Vector()->length() == 4 && VM_Version::has_simd());
13798   match(Set dst (LShiftVI src shift));
13799   match(Set dst (URShiftVI src shift));
13800   size(4*1);
13801   ins_cost(DEFAULT_COST*1); // FIXME
13802   expand %{
13803     vsh4I_reg(dst, src, shift);
13804   %}
13805 %}
13806 
13807 instruct vsl2I_immI(vecD dst, vecD src, immI shift) %{
13808   predicate(n->as_Vector()->length() == 2 && VM_Version::has_simd());
13809   match(Set dst (LShiftVI src shift));
13810   size(4);
13811   ins_cost(DEFAULT_COST); // FIXME
13812   format %{
13813     "VSHL.I32 $dst.D,$src.D,$shift\t! logical left shift packed2I"
13814   %}
13815   ins_encode %{
13816     bool quad = false;
13817     __ vshli($dst$$FloatRegister, $src$$FloatRegister, 32, $shift$$constant,
13818              quad);
13819   %}
13820   ins_pipe( ialu_reg_reg ); // FIXME
13821 %}
13822 
13823 instruct vsl4I_immI(vecX dst, vecX src, immI shift) %{
13824   predicate(n->as_Vector()->length() == 4 && VM_Version::has_simd());
13825   match(Set dst (LShiftVI src shift));
13826   size(4);
13827   ins_cost(DEFAULT_COST); // FIXME
13828   format %{
13829     "VSHL.I32 $dst.Q,$src.Q,$shift\t! logical left shift packed4I"
13830   %}
13831   ins_encode %{
13832     bool quad = true;
13833     __ vshli($dst$$FloatRegister, $src$$FloatRegister, 32, $shift$$constant,
13834              quad);
13835   %}
13836   ins_pipe( ialu_reg_reg ); // FIXME
13837 %}
13838 
13839 // Longs vector logical left/right shift
13840 instruct vsl2L_reg(vecX dst, vecX src, vecX shift) %{
13841   predicate(n->as_Vector()->length() == 2);
13842   match(Set dst (LShiftVL src shift));
13843   match(Set dst (URShiftVL src shift));
13844   size(4*1);
13845   ins_cost(DEFAULT_COST*1); // FIXME
13846   expand %{
13847     vsh2L_reg(dst, src, shift);
13848   %}
13849 %}
13850 
13851 instruct vsl2L_immI(vecX dst, vecX src, immI shift) %{
13852   predicate(n->as_Vector()->length() == 2);
13853   match(Set dst (LShiftVL src shift));
13854   size(4);
13855   ins_cost(DEFAULT_COST); // FIXME
13856   format %{
13857     "VSHL.I64 $dst.Q,$src.Q,$shift\t! logical left shift packed2L"
13858   %}
13859   ins_encode %{
13860     bool quad = true;
13861     __ vshli($dst$$FloatRegister, $src$$FloatRegister, 64, $shift$$constant,
13862              quad);
13863   %}
13864   ins_pipe( ialu_reg_reg ); // FIXME
13865 %}
13866 
13867 // ----------------------- LogicalRightShift -----------------------------------
13868 
13869 // Bytes/Shorts vector logical right shift produces incorrect Java result
13870 // for negative data because java code convert short value into int with
13871 // sign extension before a shift.
13872 
13873 // Chars vector logical right shift
13874 instruct vsrl4S_immI(vecD dst, vecD src, immI shift) %{
13875   predicate(n->as_Vector()->length() == 4);
13876   match(Set dst (URShiftVS src shift));
13877   size(4);
13878   ins_cost(DEFAULT_COST); // FIXME
13879   format %{
13880     "VSHR.U16 $dst.D,$src.D,$shift\t! logical right shift packed4S"
13881   %}
13882   ins_encode %{
13883     bool quad = false;
13884     __ vshrUI($dst$$FloatRegister, $src$$FloatRegister, 16, $shift$$constant,
13885              quad);
13886   %}
13887   ins_pipe( ialu_reg_reg ); // FIXME
13888 %}
13889 
13890 instruct vsrl8S_immI(vecX dst, vecX src, immI shift) %{
13891   predicate(n->as_Vector()->length() == 8);
13892   match(Set dst (URShiftVS src shift));
13893   size(4);
13894   ins_cost(DEFAULT_COST); // FIXME
13895   format %{
13896     "VSHR.U16 $dst.Q,$src.Q,$shift\t! logical right shift packed8S"
13897   %}
13898   ins_encode %{
13899     bool quad = true;
13900     __ vshrUI($dst$$FloatRegister, $src$$FloatRegister, 16, $shift$$constant,
13901              quad);
13902   %}
13903   ins_pipe( ialu_reg_reg ); // FIXME
13904 %}
13905 
13906 // Integers vector logical right shift
13907 instruct vsrl2I_immI(vecD dst, vecD src, immI shift) %{
13908   predicate(n->as_Vector()->length() == 2 && VM_Version::has_simd());
13909   match(Set dst (URShiftVI src shift));
13910   size(4);
13911   ins_cost(DEFAULT_COST); // FIXME
13912   format %{
13913     "VSHR.U32 $dst.D,$src.D,$shift\t! logical right shift packed2I"
13914   %}
13915   ins_encode %{
13916     bool quad = false;
13917     __ vshrUI($dst$$FloatRegister, $src$$FloatRegister, 32, $shift$$constant,
13918              quad);
13919   %}
13920   ins_pipe( ialu_reg_reg ); // FIXME
13921 %}
13922 
13923 instruct vsrl4I_immI(vecX dst, vecX src, immI shift) %{
13924   predicate(n->as_Vector()->length() == 4 && VM_Version::has_simd());
13925   match(Set dst (URShiftVI src shift));
13926   size(4);
13927   ins_cost(DEFAULT_COST); // FIXME
13928   format %{
13929     "VSHR.U32 $dst.Q,$src.Q,$shift\t! logical right shift packed4I"
13930   %}
13931   ins_encode %{
13932     bool quad = true;
13933     __ vshrUI($dst$$FloatRegister, $src$$FloatRegister, 32, $shift$$constant,
13934              quad);
13935   %}
13936   ins_pipe( ialu_reg_reg ); // FIXME
13937 %}
13938 
13939 // Longs vector logical right shift
13940 instruct vsrl2L_immI(vecX dst, vecX src, immI shift) %{
13941   predicate(n->as_Vector()->length() == 2);
13942   match(Set dst (URShiftVL src shift));
13943   size(4);
13944   ins_cost(DEFAULT_COST); // FIXME
13945   format %{
13946     "VSHR.U64 $dst.Q,$src.Q,$shift\t! logical right shift packed2L"
13947   %}
13948   ins_encode %{
13949     bool quad = true;
13950     __ vshrUI($dst$$FloatRegister, $src$$FloatRegister, 64, $shift$$constant,
13951              quad);
13952   %}
13953   ins_pipe( ialu_reg_reg ); // FIXME
13954 %}
13955 
13956 // ------------------- ArithmeticRightShift -----------------------------------
13957 
13958 // Bytes vector arithmetic left/right shift based on sign
13959 instruct vsha8B_reg(vecD dst, vecD src, vecD shift) %{
13960   predicate(n->as_Vector()->length() == 8);
13961   effect(DEF dst, USE src, USE shift);
13962   size(4);
13963   ins_cost(DEFAULT_COST); // FIXME
13964   format %{
13965     "VSHL.S8 $dst.D,$src.D,$shift.D\t! arithmetic right shift packed8B"
13966   %}
13967   ins_encode %{
13968     bool quad = false;
13969     __ vshlSI($dst$$FloatRegister, $shift$$FloatRegister, $src$$FloatRegister,
13970               MacroAssembler::VELEM_SIZE_8, quad);
13971   %}
13972   ins_pipe( ialu_reg_reg ); // FIXME
13973 %}
13974 
13975 instruct vsha16B_reg(vecX dst, vecX src, vecX shift) %{
13976   predicate(n->as_Vector()->length() == 16);
13977   effect(DEF dst, USE src, USE shift);
13978   size(4);
13979   ins_cost(DEFAULT_COST); // FIXME
13980   format %{
13981     "VSHL.S8 $dst.Q,$src.Q,$shift.Q\t! arithmetic right shift packed16B"
13982   %}
13983   ins_encode %{
13984     bool quad = true;
13985     __ vshlSI($dst$$FloatRegister, $shift$$FloatRegister, $src$$FloatRegister,
13986               MacroAssembler::VELEM_SIZE_8, quad);
13987   %}
13988   ins_pipe( ialu_reg_reg ); // FIXME
13989 %}
13990 
13991 // Shorts vector arithmetic left/right shift based on sign
13992 instruct vsha4S_reg(vecD dst, vecD src, vecD shift) %{
13993   predicate(n->as_Vector()->length() == 4);
13994   effect(DEF dst, USE src, USE shift);
13995   size(4);
13996   ins_cost(DEFAULT_COST); // FIXME
13997   format %{
13998     "VSHL.S16 $dst.D,$src.D,$shift.D\t! arithmetic right shift packed4S"
13999   %}
14000   ins_encode %{
14001     bool quad = false;
14002     __ vshlSI($dst$$FloatRegister, $shift$$FloatRegister, $src$$FloatRegister,
14003               MacroAssembler::VELEM_SIZE_16, quad);
14004   %}
14005   ins_pipe( ialu_reg_reg ); // FIXME
14006 %}
14007 
14008 instruct vsha8S_reg(vecX dst, vecX src, vecX shift) %{
14009   predicate(n->as_Vector()->length() == 8);
14010   effect(DEF dst, USE src, USE shift);
14011   size(4);
14012   ins_cost(DEFAULT_COST); // FIXME
14013   format %{
14014     "VSHL.S16 $dst.Q,$src.Q,$shift.Q\t! arithmetic right shift packed8S"
14015   %}
14016   ins_encode %{
14017     bool quad = true;
14018     __ vshlSI($dst$$FloatRegister, $shift$$FloatRegister, $src$$FloatRegister,
14019               MacroAssembler::VELEM_SIZE_16, quad);
14020   %}
14021   ins_pipe( ialu_reg_reg ); // FIXME
14022 %}
14023 
14024 // Integers vector arithmetic left/right shift based on sign
14025 instruct vsha2I_reg(vecD dst, vecD src, vecD shift) %{
14026   predicate(n->as_Vector()->length() == 2);
14027   effect(DEF dst, USE src, USE shift);
14028   size(4);
14029   ins_cost(DEFAULT_COST); // FIXME
14030   format %{
14031     "VSHL.S32 $dst.D,$src.D,$shift.D\t! arithmetic right shift packed2I"
14032   %}
14033   ins_encode %{
14034     bool quad = false;
14035     __ vshlSI($dst$$FloatRegister, $shift$$FloatRegister, $src$$FloatRegister,
14036               MacroAssembler::VELEM_SIZE_32, quad);
14037   %}
14038   ins_pipe( ialu_reg_reg ); // FIXME
14039 %}
14040 
14041 instruct vsha4I_reg(vecX dst, vecX src, vecX shift) %{
14042   predicate(n->as_Vector()->length() == 4);
14043   effect(DEF dst, USE src, USE shift);
14044   size(4);
14045   ins_cost(DEFAULT_COST); // FIXME
14046   format %{
14047     "VSHL.S32 $dst.Q,$src.Q,$shift.Q\t! arithmetic right shift packed4I"
14048   %}
14049   ins_encode %{
14050     bool quad = true;
14051     __ vshlSI($dst$$FloatRegister, $shift$$FloatRegister, $src$$FloatRegister,
14052               MacroAssembler::VELEM_SIZE_32, quad);
14053   %}
14054   ins_pipe( ialu_reg_reg ); // FIXME
14055 %}
14056 
14057 // Longs vector arithmetic left/right shift based on sign
14058 instruct vsha2L_reg(vecX dst, vecX src, vecX shift) %{
14059   predicate(n->as_Vector()->length() == 2);
14060   effect(DEF dst, USE src, USE shift);
14061   size(4);
14062   ins_cost(DEFAULT_COST); // FIXME
14063   format %{
14064     "VSHL.S64 $dst.Q,$src.Q,$shift.Q\t! arithmetic right shift packed2L"
14065   %}
14066   ins_encode %{
14067     bool quad = true;
14068     __ vshlSI($dst$$FloatRegister, $shift$$FloatRegister, $src$$FloatRegister,
14069               MacroAssembler::VELEM_SIZE_64, quad);
14070   %}
14071   ins_pipe( ialu_reg_reg ); // FIXME
14072 %}
14073 
14074 // Byte vector arithmetic right shift
14075 
14076 instruct vsra8B_reg(vecD dst, vecD src, vecD shift) %{
14077   predicate(n->as_Vector()->length() == 8);
14078   match(Set dst (RShiftVB src shift));
14079   size(4);
14080   ins_cost(DEFAULT_COST); // FIXME
14081   expand %{
14082     vsha8B_reg(dst, src, shift);
14083   %}
14084 %}
14085 
14086 instruct vsrl16B_reg(vecX dst, vecX src, vecX shift) %{
14087   predicate(n->as_Vector()->length() == 16);
14088   match(Set dst (RShiftVB src shift));
14089   size(4);
14090   ins_cost(DEFAULT_COST); // FIXME
14091   expand %{
14092     vsha16B_reg(dst, src, shift);
14093   %}
14094 %}
14095 
14096 instruct vsrl8B_immI(vecD dst, vecD src, immI shift) %{
14097   predicate(n->as_Vector()->length() == 8);
14098   match(Set dst (RShiftVB src shift));
14099   size(4);
14100   ins_cost(DEFAULT_COST); // FIXME
14101   format %{
14102     "VSHR.S8 $dst.D,$src.D,$shift\t! logical right shift packed8B"
14103   %}
14104   ins_encode %{
14105     bool quad = false;
14106     __ vshrSI($dst$$FloatRegister, $src$$FloatRegister, 8, $shift$$constant,
14107              quad);
14108   %}
14109   ins_pipe( ialu_reg_reg ); // FIXME
14110 %}
14111 
14112 instruct vsrl16B_immI(vecX dst, vecX src, immI shift) %{
14113   predicate(n->as_Vector()->length() == 16);
14114   match(Set dst (RShiftVB src shift));
14115   size(4);
14116   ins_cost(DEFAULT_COST); // FIXME
14117   format %{
14118     "VSHR.S8 $dst.Q,$src.Q,$shift\t! logical right shift packed16B"
14119   %}
14120   ins_encode %{
14121     bool quad = true;
14122     __ vshrSI($dst$$FloatRegister, $src$$FloatRegister, 8, $shift$$constant,
14123              quad);
14124   %}
14125   ins_pipe( ialu_reg_reg ); // FIXME
14126 %}
14127 
14128 // Shorts vector arithmetic right shift
14129 instruct vsra4S_reg(vecD dst, vecD src, vecD shift) %{
14130   predicate(n->as_Vector()->length() == 4);
14131   match(Set dst (RShiftVS src shift));
14132   size(4);
14133   ins_cost(DEFAULT_COST); // FIXME
14134   expand %{
14135     vsha4S_reg(dst, src, shift);
14136   %}
14137 %}
14138 
14139 instruct vsra8S_reg(vecX dst, vecX src, vecX shift) %{
14140   predicate(n->as_Vector()->length() == 8);
14141   match(Set dst (RShiftVS src shift));
14142   size(4);
14143   ins_cost(DEFAULT_COST); // FIXME
14144   expand %{
14145     vsha8S_reg(dst, src, shift);
14146   %}
14147 %}
14148 
14149 instruct vsra4S_immI(vecD dst, vecD src, immI shift) %{
14150   predicate(n->as_Vector()->length() == 4);
14151   match(Set dst (RShiftVS src shift));
14152   size(4);
14153   ins_cost(DEFAULT_COST); // FIXME
14154   format %{
14155     "VSHR.S16 $dst.D,$src.D,$shift\t! logical right shift packed4S"
14156   %}
14157   ins_encode %{
14158     bool quad = false;
14159     __ vshrSI($dst$$FloatRegister, $src$$FloatRegister, 16, $shift$$constant,
14160              quad);
14161   %}
14162   ins_pipe( ialu_reg_reg ); // FIXME
14163 %}
14164 
14165 instruct vsra8S_immI(vecX dst, vecX src, immI shift) %{
14166   predicate(n->as_Vector()->length() == 8);
14167   match(Set dst (RShiftVS src shift));
14168   size(4);
14169   ins_cost(DEFAULT_COST); // FIXME
14170   format %{
14171     "VSHR.S16 $dst.Q,$src.Q,$shift\t! logical right shift packed8S"
14172   %}
14173   ins_encode %{
14174     bool quad = true;
14175     __ vshrSI($dst$$FloatRegister, $src$$FloatRegister, 16, $shift$$constant,
14176              quad);
14177   %}
14178   ins_pipe( ialu_reg_reg ); // FIXME
14179 %}
14180 
14181 // Integers vector arithmetic right shift
14182 instruct vsra2I_reg(vecD dst, vecD src, vecD shift) %{
14183   predicate(n->as_Vector()->length() == 2);
14184   match(Set dst (RShiftVI src shift));
14185   size(4);
14186   ins_cost(DEFAULT_COST); // FIXME
14187   expand %{
14188     vsha2I_reg(dst, src, shift);
14189   %}
14190 %}
14191 
14192 instruct vsra4I_reg(vecX dst, vecX src, vecX shift) %{
14193   predicate(n->as_Vector()->length() == 4);
14194   match(Set dst (RShiftVI src shift));
14195   size(4);
14196   ins_cost(DEFAULT_COST); // FIXME
14197   expand %{
14198     vsha4I_reg(dst, src, shift);
14199   %}
14200 %}
14201 
14202 instruct vsra2I_immI(vecD dst, vecD src, immI shift) %{
14203   predicate(n->as_Vector()->length() == 2);
14204   match(Set dst (RShiftVI src shift));
14205   size(4);
14206   ins_cost(DEFAULT_COST); // FIXME
14207   format %{
14208     "VSHR.S32 $dst.D,$src.D,$shift\t! logical right shift packed2I"
14209   %}
14210   ins_encode %{
14211     bool quad = false;
14212     __ vshrSI($dst$$FloatRegister, $src$$FloatRegister, 32, $shift$$constant,
14213              quad);
14214   %}
14215   ins_pipe( ialu_reg_reg ); // FIXME
14216 %}
14217 
14218 instruct vsra4I_immI(vecX dst, vecX src, immI shift) %{
14219   predicate(n->as_Vector()->length() == 4);
14220   match(Set dst (RShiftVI src shift));
14221   size(4);
14222   ins_cost(DEFAULT_COST); // FIXME
14223   format %{
14224     "VSHR.S32 $dst.Q,$src.Q,$shift\t! logical right shift packed4I"
14225   %}
14226   ins_encode %{
14227     bool quad = true;
14228     __ vshrSI($dst$$FloatRegister, $src$$FloatRegister, 32, $shift$$constant,
14229              quad);
14230   %}
14231   ins_pipe( ialu_reg_reg ); // FIXME
14232 %}
14233 
14234 // Longs vector arithmetic right shift
14235 instruct vsra2L_reg(vecX dst, vecX src, vecX shift) %{
14236   predicate(n->as_Vector()->length() == 2);
14237   match(Set dst (RShiftVL src shift));
14238   size(4);
14239   ins_cost(DEFAULT_COST); // FIXME
14240   expand %{
14241     vsha2L_reg(dst, src, shift);
14242   %}
14243 %}
14244 
14245 instruct vsra2L_immI(vecX dst, vecX src, immI shift) %{
14246   predicate(n->as_Vector()->length() == 2);
14247   match(Set dst (RShiftVL src shift));
14248   size(4);
14249   ins_cost(DEFAULT_COST); // FIXME
14250   format %{
14251     "VSHR.S64 $dst.Q,$src.Q,$shift\t! logical right shift packed2L"
14252   %}
14253   ins_encode %{
14254     bool quad = true;
14255     __ vshrSI($dst$$FloatRegister, $src$$FloatRegister, 64, $shift$$constant,
14256              quad);
14257   %}
14258   ins_pipe( ialu_reg_reg ); // FIXME
14259 %}
14260 
14261 // --------------------------------- AND --------------------------------------
14262 
14263 instruct vandD(vecD dst, vecD src1, vecD src2) %{
14264   predicate(n->as_Vector()->length_in_bytes() == 8);
14265   match(Set dst (AndV src1 src2));
14266   format %{ "VAND    $dst.D,$src1.D,$src2.D\t! and vectors (8 bytes)" %}
14267   ins_encode %{
14268     bool quad = false;
14269     __ vandI($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister,
14270              quad);
14271   %}
14272   ins_pipe( ialu_reg_reg ); // FIXME
14273 %}
14274 
14275 instruct vandX(vecX dst, vecX src1, vecX src2) %{
14276   predicate(n->as_Vector()->length_in_bytes() == 16);
14277   match(Set dst (AndV src1 src2));
14278   format %{ "VAND    $dst.Q,$src1.Q,$src2.Q\t! and vectors (16 bytes)" %}
14279   ins_encode %{
14280     bool quad = true;
14281     __ vandI($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister,
14282              quad);
14283   %}
14284   ins_pipe( ialu_reg_reg ); // FIXME
14285 %}
14286 
14287 // --------------------------------- OR ---------------------------------------
14288 
14289 instruct vorD(vecD dst, vecD src1, vecD src2) %{
14290   predicate(n->as_Vector()->length_in_bytes() == 8);
14291   match(Set dst (OrV src1 src2));
14292   format %{ "VOR     $dst.D,$src1.D,$src2.D\t! and vectors (8 bytes)" %}
14293   ins_encode %{
14294     bool quad = false;
14295     __ vorI($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister,
14296             quad);
14297   %}
14298   ins_pipe( ialu_reg_reg ); // FIXME
14299 %}
14300 
14301 instruct vorX(vecX dst, vecX src1, vecX src2) %{
14302   predicate(n->as_Vector()->length_in_bytes() == 16);
14303   match(Set dst (OrV src1 src2));
14304   format %{ "VOR     $dst.Q,$src1.Q,$src2.Q\t! and vectors (16 bytes)" %}
14305   ins_encode %{
14306     bool quad = true;
14307     __ vorI($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister,
14308             quad);
14309   %}
14310   ins_pipe( ialu_reg_reg ); // FIXME
14311 %}
14312 
14313 // --------------------------------- XOR --------------------------------------
14314 
14315 instruct vxorD(vecD dst, vecD src1, vecD src2) %{
14316   predicate(n->as_Vector()->length_in_bytes() == 8);
14317   match(Set dst (XorV src1 src2));
14318   format %{ "VXOR    $dst.D,$src1.D,$src2.D\t! and vectors (8 bytes)" %}
14319   ins_encode %{
14320     bool quad = false;
14321     __ vxorI($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister,
14322              quad);
14323   %}
14324   ins_pipe( ialu_reg_reg ); // FIXME
14325 %}
14326 
14327 instruct vxorX(vecX dst, vecX src1, vecX src2) %{
14328   predicate(n->as_Vector()->length_in_bytes() == 16);
14329   match(Set dst (XorV src1 src2));
14330   format %{ "VXOR    $dst.Q,$src1.Q,$src2.Q\t! and vectors (16 bytes)" %}
14331   ins_encode %{
14332     bool quad = true;
14333     __ vxorI($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister,
14334              quad);
14335   %}
14336   ins_pipe( ialu_reg_reg ); // FIXME
14337 %}
14338 
14339 
14340 //----------PEEPHOLE RULES-----------------------------------------------------
14341 // These must follow all instruction definitions as they use the names
14342 // defined in the instructions definitions.
14343 //
14344 // peepmatch ( root_instr_name [preceding_instruction]* );
14345 //
14346 // peepconstraint %{
14347 // (instruction_number.operand_name relational_op instruction_number.operand_name
14348 //  [, ...] );
14349 // // instruction numbers are zero-based using left to right order in peepmatch
14350 //
14351 // peepreplace ( instr_name  ( [instruction_number.operand_name]* ) );
14352 // // provide an instruction_number.operand_name for each operand that appears
14353 // // in the replacement instruction's match rule
14354 //
14355 // ---------VM FLAGS---------------------------------------------------------
14356 //
14357 // All peephole optimizations can be turned off using -XX:-OptoPeephole
14358 //
14359 // Each peephole rule is given an identifying number starting with zero and
14360 // increasing by one in the order seen by the parser.  An individual peephole
14361 // can be enabled, and all others disabled, by using -XX:OptoPeepholeAt=#
14362 // on the command-line.
14363 //
14364 // ---------CURRENT LIMITATIONS----------------------------------------------
14365 //
14366 // Only match adjacent instructions in same basic block
14367 // Only equality constraints
14368 // Only constraints between operands, not (0.dest_reg == EAX_enc)
14369 // Only one replacement instruction
14370 //
14371 // ---------EXAMPLE----------------------------------------------------------
14372 //
14373 // // pertinent parts of existing instructions in architecture description
14374 // instruct movI(eRegI dst, eRegI src) %{
14375 //   match(Set dst (CopyI src));
14376 // %}
14377 //
14378 // instruct incI_eReg(eRegI dst, immI1 src, eFlagsReg cr) %{
14379 //   match(Set dst (AddI dst src));
14380 //   effect(KILL cr);
14381 // %}
14382 //
14383 // // Change (inc mov) to lea
14384 // peephole %{
14385 //   // increment preceeded by register-register move
14386 //   peepmatch ( incI_eReg movI );
14387 //   // require that the destination register of the increment
14388 //   // match the destination register of the move
14389 //   peepconstraint ( 0.dst == 1.dst );
14390 //   // construct a replacement instruction that sets
14391 //   // the destination to ( move's source register + one )
14392 //   peepreplace ( incI_eReg_immI1( 0.dst 1.src 0.src ) );
14393 // %}
14394 //
14395 
14396 // // Change load of spilled value to only a spill
14397 // instruct storeI(memory mem, eRegI src) %{
14398 //   match(Set mem (StoreI mem src));
14399 // %}
14400 //
14401 // instruct loadI(eRegI dst, memory mem) %{
14402 //   match(Set dst (LoadI mem));
14403 // %}
14404 //
14405 // peephole %{
14406 //   peepmatch ( loadI storeI );
14407 //   peepconstraint ( 1.src == 0.dst, 1.mem == 0.mem );
14408 //   peepreplace ( storeI( 1.mem 1.mem 1.src ) );
14409 // %}
14410 
14411 //----------SMARTSPILL RULES---------------------------------------------------
14412 // These must follow all instruction definitions as they use the names
14413 // defined in the instructions definitions.
14414 //
14415 // ARM will probably not have any of these rules due to RISC instruction set.
14416 
14417 //----------PIPELINE-----------------------------------------------------------
14418 // Rules which define the behavior of the target architectures pipeline.