1 //
   2 // Copyright (c) 2008, 2018, Oracle and/or its affiliates. All rights reserved.
   3 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4 //
   5 // This code is free software; you can redistribute it and/or modify it
   6 // under the terms of the GNU General Public License version 2 only, as
   7 // published by the Free Software Foundation.
   8 //
   9 // This code is distributed in the hope that it will be useful, but WITHOUT
  10 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11 // FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12 // version 2 for more details (a copy is included in the LICENSE file that
  13 // accompanied this code).
  14 //
  15 // You should have received a copy of the GNU General Public License version
  16 // 2 along with this work; if not, write to the Free Software Foundation,
  17 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18 //
  19 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20 // or visit www.oracle.com if you need additional information or have any
  21 // questions.
  22 //
  23 
  24 // ARM Architecture Description File
  25 
  26 //----------DEFINITION BLOCK---------------------------------------------------
  27 // Define name --> value mappings to inform the ADLC of an integer valued name
  28 // Current support includes integer values in the range [0, 0x7FFFFFFF]
  29 // Format:
  30 //        int_def  <name>         ( <int_value>, <expression>);
  31 // Generated Code in ad_<arch>.hpp
  32 //        #define  <name>   (<expression>)
  33 //        // value == <int_value>
  34 // Generated code in ad_<arch>.cpp adlc_verification()
  35 //        assert( <name> == <int_value>, "Expect (<expression>) to equal <int_value>");
  36 //
  37 definitions %{
  38 // The default cost (of an ALU instruction).
  39   int_def DEFAULT_COST      (    100,     100);
  40   int_def HUGE_COST         (1000000, 1000000);
  41 
  42 // Memory refs are twice as expensive as run-of-the-mill.
  43   int_def MEMORY_REF_COST   (    200, DEFAULT_COST * 2);
  44 
  45 // Branches are even more expensive.
  46   int_def BRANCH_COST       (    300, DEFAULT_COST * 3);
  47   int_def CALL_COST         (    300, DEFAULT_COST * 3);
  48 %}
  49 
  50 
  51 //----------SOURCE BLOCK-------------------------------------------------------
  52 // This is a block of C++ code which provides values, functions, and
  53 // definitions necessary in the rest of the architecture description
  54 source_hpp %{
  55 // Header information of the source block.
  56 // Method declarations/definitions which are used outside
  57 // the ad-scope can conveniently be defined here.
  58 //
  59 // To keep related declarations/definitions/uses close together,
  60 // we switch between source %{ }% and source_hpp %{ }% freely as needed.
  61 
  62 // Does destination need to be loaded in a register then passed to a
  63 // branch instruction?
  64 extern bool maybe_far_call(const CallNode *n);
  65 extern bool maybe_far_call(const MachCallNode *n);
  66 static inline bool cache_reachable() {
  67   return MacroAssembler::_cache_fully_reachable();
  68 }
  69 
  70 #ifdef AARCH64
  71 #define ldr_32 ldr_w
  72 #define str_32 str_w
  73 #else
  74 #define ldr_32 ldr
  75 #define str_32 str
  76 #define tst_32 tst
  77 #define teq_32 teq
  78 #endif
  79 #if 1
  80 extern bool PrintOptoAssembly;
  81 #endif
  82 
  83 class c2 {
  84 public:
  85   static OptoRegPair return_value(int ideal_reg);
  86 };
  87 
  88 class CallStubImpl {
  89 
  90   //--------------------------------------------------------------
  91   //---<  Used for optimization in Compile::Shorten_branches  >---
  92   //--------------------------------------------------------------
  93 
  94  public:
  95   // Size of call trampoline stub.
  96   static uint size_call_trampoline() {
  97     return 0; // no call trampolines on this platform
  98   }
  99 
 100   // number of relocations needed by a call trampoline stub
 101   static uint reloc_call_trampoline() {
 102     return 0; // no call trampolines on this platform
 103   }
 104 };
 105 
 106 class HandlerImpl {
 107 
 108  public:
 109 
 110   static int emit_exception_handler(CodeBuffer &cbuf);
 111   static int emit_deopt_handler(CodeBuffer& cbuf);
 112 
 113   static uint size_exception_handler() {
 114 #ifdef AARCH64
 115     // ldr_literal; br; (pad); <literal>
 116     return 3 * Assembler::InstructionSize + wordSize;
 117 #else
 118     return ( 3 * 4 );
 119 #endif
 120   }
 121 
 122 
 123   static uint size_deopt_handler() {
 124     return ( 9 * 4 );
 125   }
 126 
 127 };
 128 
 129 %}
 130 
 131 source %{
 132 #define __ _masm.
 133 
 134 static FloatRegister reg_to_FloatRegister_object(int register_encoding);
 135 static Register reg_to_register_object(int register_encoding);
 136 
 137 
 138 // ****************************************************************************
 139 
 140 // REQUIRED FUNCTIONALITY
 141 
 142 // Indicate if the safepoint node needs the polling page as an input.
 143 // Since ARM does not have absolute addressing, it does.
 144 bool SafePointNode::needs_polling_address_input() {
 145   return true;
 146 }
 147 
 148 // emit an interrupt that is caught by the debugger (for debugging compiler)
 149 void emit_break(CodeBuffer &cbuf) {
 150   MacroAssembler _masm(&cbuf);
 151   __ breakpoint();
 152 }
 153 
 154 #ifndef PRODUCT
 155 void MachBreakpointNode::format( PhaseRegAlloc *, outputStream *st ) const {
 156   st->print("TA");
 157 }
 158 #endif
 159 
 160 void MachBreakpointNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 161   emit_break(cbuf);
 162 }
 163 
 164 uint MachBreakpointNode::size(PhaseRegAlloc *ra_) const {
 165   return MachNode::size(ra_);
 166 }
 167 
 168 
 169 void emit_nop(CodeBuffer &cbuf) {
 170   MacroAssembler _masm(&cbuf);
 171   __ nop();
 172 }
 173 
 174 
 175 void emit_call_reloc(CodeBuffer &cbuf, const MachCallNode *n, MachOper *m, RelocationHolder const& rspec) {
 176   int ret_addr_offset0 = n->as_MachCall()->ret_addr_offset();
 177   int call_site_offset = cbuf.insts()->mark_off();
 178   MacroAssembler _masm(&cbuf);
 179   __ set_inst_mark(); // needed in emit_to_interp_stub() to locate the call
 180   address target = (address)m->method();
 181   assert(n->as_MachCall()->entry_point() == target, "sanity");
 182   assert(maybe_far_call(n) == !__ reachable_from_cache(target), "sanity");
 183   assert(cache_reachable() == __ cache_fully_reachable(), "sanity");
 184 
 185   assert(target != NULL, "need real address");
 186 
 187   int ret_addr_offset = -1;
 188   if (rspec.type() == relocInfo::runtime_call_type) {
 189     __ call(target, rspec);
 190     ret_addr_offset = __ offset();
 191   } else {
 192     // scratches Rtemp
 193     ret_addr_offset = __ patchable_call(target, rspec, true);
 194   }
 195   assert(ret_addr_offset - call_site_offset == ret_addr_offset0, "fix ret_addr_offset()");
 196 }
 197 
 198 //=============================================================================
 199 // REQUIRED FUNCTIONALITY for encoding
 200 void emit_lo(CodeBuffer &cbuf, int val) {  }
 201 void emit_hi(CodeBuffer &cbuf, int val) {  }
 202 
 203 
 204 //=============================================================================
 205 const RegMask& MachConstantBaseNode::_out_RegMask = PTR_REG_mask();
 206 
 207 int Compile::ConstantTable::calculate_table_base_offset() const {
 208 #ifdef AARCH64
 209   return 0;
 210 #else
 211   int offset = -(size() / 2);
 212   // flds, fldd: 8-bit  offset multiplied by 4: +/- 1024
 213   // ldr, ldrb : 12-bit offset:                 +/- 4096
 214   if (!Assembler::is_simm10(offset)) {
 215     offset = Assembler::min_simm10();
 216   }
 217   return offset;
 218 #endif
 219 }
 220 
 221 bool MachConstantBaseNode::requires_postalloc_expand() const { return false; }
 222 void MachConstantBaseNode::postalloc_expand(GrowableArray <Node *> *nodes, PhaseRegAlloc *ra_) {
 223   ShouldNotReachHere();
 224 }
 225 
 226 void MachConstantBaseNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const {
 227   Compile* C = ra_->C;
 228   Compile::ConstantTable& constant_table = C->constant_table();
 229   MacroAssembler _masm(&cbuf);
 230 
 231   Register r = as_Register(ra_->get_encode(this));
 232   CodeSection* consts_section = __ code()->consts();
 233   int consts_size = consts_section->align_at_start(consts_section->size());
 234   assert(constant_table.size() == consts_size, "must be: %d == %d", constant_table.size(), consts_size);
 235 
 236   // Materialize the constant table base.
 237   address baseaddr = consts_section->start() + -(constant_table.table_base_offset());
 238   RelocationHolder rspec = internal_word_Relocation::spec(baseaddr);
 239   __ mov_address(r, baseaddr, rspec);
 240 }
 241 
 242 uint MachConstantBaseNode::size(PhaseRegAlloc*) const {
 243 #ifdef AARCH64
 244   return 5 * Assembler::InstructionSize;
 245 #else
 246   return 8;
 247 #endif
 248 }
 249 
 250 #ifndef PRODUCT
 251 void MachConstantBaseNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
 252   char reg[128];
 253   ra_->dump_register(this, reg);
 254   st->print("MOV_SLOW    &constanttable,%s\t! constant table base", reg);
 255 }
 256 #endif
 257 
 258 #ifndef PRODUCT
 259 void MachPrologNode::format( PhaseRegAlloc *ra_, outputStream *st ) const {
 260   Compile* C = ra_->C;
 261 
 262   for (int i = 0; i < OptoPrologueNops; i++) {
 263     st->print_cr("NOP"); st->print("\t");
 264   }
 265 #ifdef AARCH64
 266   if (OptoPrologueNops <= 0) {
 267     st->print_cr("NOP\t! required for safe patching");
 268     st->print("\t");
 269   }
 270 #endif
 271 
 272   size_t framesize = C->frame_size_in_bytes();
 273   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
 274   int bangsize = C->bang_size_in_bytes();
 275   // Remove two words for return addr and rbp,
 276   framesize -= 2*wordSize;
 277   bangsize -= 2*wordSize;
 278 
 279   // Calls to C2R adapters often do not accept exceptional returns.
 280   // We require that their callers must bang for them.  But be careful, because
 281   // some VM calls (such as call site linkage) can use several kilobytes of
 282   // stack.  But the stack safety zone should account for that.
 283   // See bugs 4446381, 4468289, 4497237.
 284   if (C->need_stack_bang(bangsize)) {
 285     st->print_cr("! stack bang (%d bytes)", bangsize); st->print("\t");
 286   }
 287   st->print_cr("PUSH   R_FP|R_LR_LR"); st->print("\t");
 288   if (framesize != 0) {
 289     st->print   ("SUB    R_SP, R_SP, " SIZE_FORMAT,framesize);
 290   }
 291 }
 292 #endif
 293 
 294 void MachPrologNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 295   Compile* C = ra_->C;
 296   MacroAssembler _masm(&cbuf);
 297 
 298   for (int i = 0; i < OptoPrologueNops; i++) {
 299     __ nop();
 300   }
 301 #ifdef AARCH64
 302   if (OptoPrologueNops <= 0) {
 303     __ nop(); // required for safe patching by patch_verified_entry()
 304   }
 305 #endif
 306 
 307   size_t framesize = C->frame_size_in_bytes();
 308   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
 309   int bangsize = C->bang_size_in_bytes();
 310   // Remove two words for return addr and fp,
 311   framesize -= 2*wordSize;
 312   bangsize -= 2*wordSize;
 313 
 314   // Calls to C2R adapters often do not accept exceptional returns.
 315   // We require that their callers must bang for them.  But be careful, because
 316   // some VM calls (such as call site linkage) can use several kilobytes of
 317   // stack.  But the stack safety zone should account for that.
 318   // See bugs 4446381, 4468289, 4497237.
 319   if (C->need_stack_bang(bangsize)) {
 320     __ arm_stack_overflow_check(bangsize, Rtemp);
 321   }
 322 
 323   __ raw_push(FP, LR);
 324   if (framesize != 0) {
 325     __ sub_slow(SP, SP, framesize);
 326   }
 327 
 328   // offset from scratch buffer is not valid
 329   if (strcmp(cbuf.name(), "Compile::Fill_buffer") == 0) {
 330     C->set_frame_complete( __ offset() );
 331   }
 332 
 333   if (C->has_mach_constant_base_node()) {
 334     // NOTE: We set the table base offset here because users might be
 335     // emitted before MachConstantBaseNode.
 336     Compile::ConstantTable& constant_table = C->constant_table();
 337     constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
 338   }
 339 }
 340 
 341 uint MachPrologNode::size(PhaseRegAlloc *ra_) const {
 342   return MachNode::size(ra_);
 343 }
 344 
 345 int MachPrologNode::reloc() const {
 346   return 10; // a large enough number
 347 }
 348 
 349 //=============================================================================
 350 #ifndef PRODUCT
 351 void MachEpilogNode::format( PhaseRegAlloc *ra_, outputStream *st ) const {
 352   Compile* C = ra_->C;
 353 
 354   size_t framesize = C->frame_size_in_bytes();
 355   framesize -= 2*wordSize;
 356 
 357   if (framesize != 0) {
 358     st->print("ADD    R_SP, R_SP, " SIZE_FORMAT "\n\t",framesize);
 359   }
 360   st->print("POP    R_FP|R_LR_LR");
 361 
 362   if (do_polling() && ra_->C->is_method_compilation()) {
 363     st->print("\n\t");
 364 #ifdef AARCH64
 365     if (MacroAssembler::page_reachable_from_cache(os::get_polling_page())) {
 366       st->print("ADRP     Rtemp, #PollAddr\t! Load Polling address\n\t");
 367       st->print("LDR      ZR,[Rtemp + #PollAddr & 0xfff]\t!Poll for Safepointing");
 368     } else {
 369       st->print("mov_slow Rtemp, #PollAddr\t! Load Polling address\n\t");
 370       st->print("LDR      ZR,[Rtemp]\t!Poll for Safepointing");
 371     }
 372 #else
 373     st->print("MOV    Rtemp, #PollAddr\t! Load Polling address\n\t");
 374     st->print("LDR    Rtemp,[Rtemp]\t!Poll for Safepointing");
 375 #endif
 376   }
 377 }
 378 #endif
 379 
 380 void MachEpilogNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 381   MacroAssembler _masm(&cbuf);
 382   Compile* C = ra_->C;
 383 
 384   size_t framesize = C->frame_size_in_bytes();
 385   framesize -= 2*wordSize;
 386   if (framesize != 0) {
 387     __ add_slow(SP, SP, framesize);
 388   }
 389   __ raw_pop(FP, LR);
 390 
 391   // If this does safepoint polling, then do it here
 392   if (do_polling() && ra_->C->is_method_compilation()) {
 393 #ifdef AARCH64
 394     if (false && MacroAssembler::page_reachable_from_cache(os::get_polling_page())) {
 395 /* FIXME: TODO
 396       __ relocate(relocInfo::xxx);
 397       __ adrp(Rtemp, (intptr_t)os::get_polling_page());
 398       __ relocate(relocInfo::poll_return_type);
 399       int offset = os::get_polling_page() & 0xfff;
 400       __ ldr(ZR, Address(Rtemp + offset));
 401 */
 402     } else {
 403       __ mov_address(Rtemp, (address)os::get_polling_page(), symbolic_Relocation::polling_page_reference);
 404       __ relocate(relocInfo::poll_return_type);
 405       __ ldr(ZR, Address(Rtemp));
 406     }
 407 #else
 408     // mov_slow here is usually one or two instruction
 409     __ mov_address(Rtemp, (address)os::get_polling_page(), symbolic_Relocation::polling_page_reference);
 410     __ relocate(relocInfo::poll_return_type);
 411     __ ldr(Rtemp, Address(Rtemp));
 412 #endif
 413   }
 414 }
 415 
 416 uint MachEpilogNode::size(PhaseRegAlloc *ra_) const {
 417 #ifdef AARCH64
 418   // allow for added alignment nop from mov_address bind_literal
 419   return MachNode::size(ra_) + 1 * Assembler::InstructionSize;
 420 #else
 421   return MachNode::size(ra_);
 422 #endif
 423 }
 424 
 425 int MachEpilogNode::reloc() const {
 426   return 16; // a large enough number
 427 }
 428 
 429 const Pipeline * MachEpilogNode::pipeline() const {
 430   return MachNode::pipeline_class();
 431 }
 432 
 433 int MachEpilogNode::safepoint_offset() const {
 434   assert( do_polling(), "no return for this epilog node");
 435   //  return MacroAssembler::size_of_sethi(os::get_polling_page());
 436   Unimplemented();
 437   return 0;
 438 }
 439 
 440 //=============================================================================
 441 
 442 // Figure out which register class each belongs in: rc_int, rc_float, rc_stack
 443 enum RC { rc_bad, rc_int, rc_float, rc_stack };
 444 static enum RC rc_class( OptoReg::Name reg ) {
 445   if (!OptoReg::is_valid(reg)) return rc_bad;
 446   if (OptoReg::is_stack(reg)) return rc_stack;
 447   VMReg r = OptoReg::as_VMReg(reg);
 448   if (r->is_Register()) return rc_int;
 449   assert(r->is_FloatRegister(), "must be");
 450   return rc_float;
 451 }
 452 
 453 static inline bool is_iRegLd_memhd(OptoReg::Name src_first, OptoReg::Name src_second, int offset) {
 454 #ifdef AARCH64
 455   return is_memoryHD(offset);
 456 #else
 457   int rlo = Matcher::_regEncode[src_first];
 458   int rhi = Matcher::_regEncode[src_second];
 459   if (!((rlo&1)==0 && (rlo+1 == rhi))) {
 460     tty->print_cr("CAUGHT BAD LDRD/STRD");
 461   }
 462   return (rlo&1)==0 && (rlo+1 == rhi) && is_memoryHD(offset);
 463 #endif
 464 }
 465 
 466 uint MachSpillCopyNode::implementation( CodeBuffer *cbuf,
 467                                         PhaseRegAlloc *ra_,
 468                                         bool do_size,
 469                                         outputStream* st ) const {
 470   // Get registers to move
 471   OptoReg::Name src_second = ra_->get_reg_second(in(1));
 472   OptoReg::Name src_first = ra_->get_reg_first(in(1));
 473   OptoReg::Name dst_second = ra_->get_reg_second(this );
 474   OptoReg::Name dst_first = ra_->get_reg_first(this );
 475 
 476   enum RC src_second_rc = rc_class(src_second);
 477   enum RC src_first_rc = rc_class(src_first);
 478   enum RC dst_second_rc = rc_class(dst_second);
 479   enum RC dst_first_rc = rc_class(dst_first);
 480 
 481   assert( OptoReg::is_valid(src_first) && OptoReg::is_valid(dst_first), "must move at least 1 register" );
 482 
 483   // Generate spill code!
 484   int size = 0;
 485 
 486   if (src_first == dst_first && src_second == dst_second)
 487     return size;            // Self copy, no move
 488 
 489 #ifdef TODO
 490   if (bottom_type()->isa_vect() != NULL) {
 491   }
 492 #endif
 493 
 494   // Shared code does not expect instruction set capability based bailouts here.
 495   // Handle offset unreachable bailout with minimal change in shared code.
 496   // Bailout only for real instruction emit.
 497   // This requires a single comment change in shared code. ( see output.cpp "Normal" instruction case )
 498 
 499   MacroAssembler _masm(cbuf);
 500 
 501   // --------------------------------------
 502   // Check for mem-mem move.  Load into unused float registers and fall into
 503   // the float-store case.
 504   if (src_first_rc == rc_stack && dst_first_rc == rc_stack) {
 505     int offset = ra_->reg2offset(src_first);
 506     if (cbuf && !is_memoryfp(offset)) {
 507       ra_->C->record_method_not_compilable("unable to handle large constant offsets");
 508       return 0;
 509     } else {
 510       if (src_second_rc != rc_bad) {
 511         assert((src_first&1)==0 && src_first+1 == src_second, "pair of registers must be aligned/contiguous");
 512         src_first     = OptoReg::Name(R_mem_copy_lo_num);
 513         src_second    = OptoReg::Name(R_mem_copy_hi_num);
 514         src_first_rc  = rc_float;
 515         src_second_rc = rc_float;
 516         if (cbuf) {
 517           __ ldr_double(Rmemcopy, Address(SP, offset));
 518         } else if (!do_size) {
 519           st->print(LDR_DOUBLE "   R_%s,[R_SP + #%d]\t! spill",OptoReg::regname(src_first),offset);
 520         }
 521       } else {
 522         src_first     = OptoReg::Name(R_mem_copy_lo_num);
 523         src_first_rc  = rc_float;
 524         if (cbuf) {
 525           __ ldr_float(Rmemcopy, Address(SP, offset));
 526         } else if (!do_size) {
 527           st->print(LDR_FLOAT "   R_%s,[R_SP + #%d]\t! spill",OptoReg::regname(src_first),offset);
 528         }
 529       }
 530       size += 4;
 531     }
 532   }
 533 
 534   if (src_second_rc == rc_stack && dst_second_rc == rc_stack) {
 535     Unimplemented();
 536   }
 537 
 538   // --------------------------------------
 539   // Check for integer reg-reg copy
 540   if (src_first_rc == rc_int && dst_first_rc == rc_int) {
 541     // Else normal reg-reg copy
 542     assert( src_second != dst_first, "smashed second before evacuating it" );
 543     if (cbuf) {
 544       __ mov(reg_to_register_object(Matcher::_regEncode[dst_first]), reg_to_register_object(Matcher::_regEncode[src_first]));
 545 #ifndef PRODUCT
 546     } else if (!do_size) {
 547       st->print("MOV    R_%s, R_%s\t# spill",
 548                 Matcher::regName[dst_first],
 549                 Matcher::regName[src_first]);
 550 #endif
 551     }
 552 #ifdef AARCH64
 553     if (src_first+1 == src_second && dst_first+1 == dst_second) {
 554       return size + 4;
 555     }
 556 #endif
 557     size += 4;
 558   }
 559 
 560   // Check for integer store
 561   if (src_first_rc == rc_int && dst_first_rc == rc_stack) {
 562     int offset = ra_->reg2offset(dst_first);
 563     if (cbuf && !is_memoryI(offset)) {
 564       ra_->C->record_method_not_compilable("unable to handle large constant offsets");
 565       return 0;
 566     } else {
 567       if (src_second_rc != rc_bad && is_iRegLd_memhd(src_first, src_second, offset)) {
 568         assert((src_first&1)==0 && src_first+1 == src_second, "pair of registers must be aligned/contiguous");
 569         if (cbuf) {
 570           __ str_64(reg_to_register_object(Matcher::_regEncode[src_first]), Address(SP, offset));
 571 #ifndef PRODUCT
 572         } else if (!do_size) {
 573           if (size != 0) st->print("\n\t");
 574           st->print(STR_64 "   R_%s,[R_SP + #%d]\t! spill",OptoReg::regname(src_first), offset);
 575 #endif
 576         }
 577         return size + 4;
 578       } else {
 579         if (cbuf) {
 580           __ str_32(reg_to_register_object(Matcher::_regEncode[src_first]), Address(SP, offset));
 581 #ifndef PRODUCT
 582         } else if (!do_size) {
 583           if (size != 0) st->print("\n\t");
 584           st->print(STR_32 "   R_%s,[R_SP + #%d]\t! spill",OptoReg::regname(src_first), offset);
 585 #endif
 586         }
 587       }
 588     }
 589     size += 4;
 590   }
 591 
 592   // Check for integer load
 593   if (dst_first_rc == rc_int && src_first_rc == rc_stack) {
 594     int offset = ra_->reg2offset(src_first);
 595     if (cbuf && !is_memoryI(offset)) {
 596       ra_->C->record_method_not_compilable("unable to handle large constant offsets");
 597       return 0;
 598     } else {
 599       if (src_second_rc != rc_bad && is_iRegLd_memhd(dst_first, dst_second, offset)) {
 600         assert((src_first&1)==0 && src_first+1 == src_second, "pair of registers must be aligned/contiguous");
 601         if (cbuf) {
 602           __ ldr_64(reg_to_register_object(Matcher::_regEncode[dst_first]), Address(SP, offset));
 603 #ifndef PRODUCT
 604         } else if (!do_size) {
 605           if (size != 0) st->print("\n\t");
 606           st->print(LDR_64 "   R_%s,[R_SP + #%d]\t! spill",OptoReg::regname(dst_first), offset);
 607 #endif
 608         }
 609         return size + 4;
 610       } else {
 611         if (cbuf) {
 612           __ ldr_32(reg_to_register_object(Matcher::_regEncode[dst_first]), Address(SP, offset));
 613 #ifndef PRODUCT
 614         } else if (!do_size) {
 615           if (size != 0) st->print("\n\t");
 616           st->print(LDR_32 "   R_%s,[R_SP + #%d]\t! spill",OptoReg::regname(dst_first), offset);
 617 #endif
 618         }
 619       }
 620     }
 621     size += 4;
 622   }
 623 
 624   // Check for float reg-reg copy
 625   if (src_first_rc == rc_float && dst_first_rc == rc_float) {
 626     if (src_second_rc != rc_bad) {
 627       assert((src_first&1)==0 && src_first+1 == src_second && (dst_first&1)==0 && dst_first+1 == dst_second, "pairs of registers must be aligned/contiguous");
 628       if (cbuf) {
 629       __ mov_double(reg_to_FloatRegister_object(Matcher::_regEncode[dst_first]), reg_to_FloatRegister_object(Matcher::_regEncode[src_first]));
 630 #ifndef PRODUCT
 631       } else if (!do_size) {
 632         st->print(MOV_DOUBLE "    R_%s, R_%s\t# spill",
 633                   Matcher::regName[dst_first],
 634                   Matcher::regName[src_first]);
 635 #endif
 636       }
 637       return 4;
 638     }
 639     if (cbuf) {
 640       __ mov_float(reg_to_FloatRegister_object(Matcher::_regEncode[dst_first]), reg_to_FloatRegister_object(Matcher::_regEncode[src_first]));
 641 #ifndef PRODUCT
 642     } else if (!do_size) {
 643       st->print(MOV_FLOAT "    R_%s, R_%s\t# spill",
 644                 Matcher::regName[dst_first],
 645                 Matcher::regName[src_first]);
 646 #endif
 647     }
 648     size = 4;
 649   }
 650 
 651   // Check for float store
 652   if (src_first_rc == rc_float && dst_first_rc == rc_stack) {
 653     int offset = ra_->reg2offset(dst_first);
 654     if (cbuf && !is_memoryfp(offset)) {
 655       ra_->C->record_method_not_compilable("unable to handle large constant offsets");
 656       return 0;
 657     } else {
 658       // Further check for aligned-adjacent pair, so we can use a double store
 659       if (src_second_rc != rc_bad) {
 660         assert((src_first&1)==0 && src_first+1 == src_second && (dst_first&1)==0 && dst_first+1 == dst_second, "pairs of registers and stack slots must be aligned/contiguous");
 661         if (cbuf) {
 662           __ str_double(reg_to_FloatRegister_object(Matcher::_regEncode[src_first]), Address(SP, offset));
 663 #ifndef PRODUCT
 664         } else if (!do_size) {
 665           if (size != 0) st->print("\n\t");
 666           st->print(STR_DOUBLE "   R_%s,[R_SP + #%d]\t! spill",OptoReg::regname(src_first),offset);
 667 #endif
 668         }
 669         return size + 4;
 670       } else {
 671         if (cbuf) {
 672           __ str_float(reg_to_FloatRegister_object(Matcher::_regEncode[src_first]), Address(SP, offset));
 673 #ifndef PRODUCT
 674         } else if (!do_size) {
 675           if (size != 0) st->print("\n\t");
 676           st->print(STR_FLOAT "   R_%s,[R_SP + #%d]\t! spill",OptoReg::regname(src_first),offset);
 677 #endif
 678         }
 679       }
 680     }
 681     size += 4;
 682   }
 683 
 684   // Check for float load
 685   if (dst_first_rc == rc_float && src_first_rc == rc_stack) {
 686     int offset = ra_->reg2offset(src_first);
 687     if (cbuf && !is_memoryfp(offset)) {
 688       ra_->C->record_method_not_compilable("unable to handle large constant offsets");
 689       return 0;
 690     } else {
 691       // Further check for aligned-adjacent pair, so we can use a double store
 692       if (src_second_rc != rc_bad) {
 693         assert((src_first&1)==0 && src_first+1 == src_second && (dst_first&1)==0 && dst_first+1 == dst_second, "pairs of registers and stack slots must be aligned/contiguous");
 694         if (cbuf) {
 695           __ ldr_double(reg_to_FloatRegister_object(Matcher::_regEncode[dst_first]), Address(SP, offset));
 696 #ifndef PRODUCT
 697         } else if (!do_size) {
 698           if (size != 0) st->print("\n\t");
 699           st->print(LDR_DOUBLE "   R_%s,[R_SP + #%d]\t! spill",OptoReg::regname(dst_first),offset);
 700 #endif
 701         }
 702         return size + 4;
 703       } else {
 704         if (cbuf) {
 705           __ ldr_float(reg_to_FloatRegister_object(Matcher::_regEncode[dst_first]), Address(SP, offset));
 706 #ifndef PRODUCT
 707         } else if (!do_size) {
 708           if (size != 0) st->print("\n\t");
 709           st->print(LDR_FLOAT "   R_%s,[R_SP + #%d]\t! spill",OptoReg::regname(dst_first),offset);
 710 #endif
 711         }
 712       }
 713     }
 714     size += 4;
 715   }
 716 
 717   // check for int reg -> float reg move
 718   if (src_first_rc == rc_int && dst_first_rc == rc_float) {
 719     // Further check for aligned-adjacent pair, so we can use a single instruction
 720     if (src_second_rc != rc_bad) {
 721       assert((dst_first&1)==0 && dst_first+1 == dst_second, "pairs of registers must be aligned/contiguous");
 722       assert((src_first&1)==0 && src_first+1 == src_second, "pairs of registers must be aligned/contiguous");
 723       assert(src_second_rc == rc_int && dst_second_rc == rc_float, "unsupported");
 724       if (cbuf) {
 725 #ifdef AARCH64
 726         __ fmov_dx(reg_to_FloatRegister_object(Matcher::_regEncode[dst_first]), reg_to_register_object(Matcher::_regEncode[src_first]));
 727 #else
 728         __ fmdrr(reg_to_FloatRegister_object(Matcher::_regEncode[dst_first]), reg_to_register_object(Matcher::_regEncode[src_first]), reg_to_register_object(Matcher::_regEncode[src_second]));
 729 #endif
 730 #ifndef PRODUCT
 731       } else if (!do_size) {
 732         if (size != 0) st->print("\n\t");
 733 #ifdef AARCH64
 734         st->print("FMOV_DX   R_%s, R_%s\t! spill",OptoReg::regname(dst_first), OptoReg::regname(src_first));
 735 #else
 736         st->print("FMDRR   R_%s, R_%s, R_%s\t! spill",OptoReg::regname(dst_first), OptoReg::regname(src_first), OptoReg::regname(src_second));
 737 #endif
 738 #endif
 739       }
 740       return size + 4;
 741     } else {
 742       if (cbuf) {
 743         __ fmsr(reg_to_FloatRegister_object(Matcher::_regEncode[dst_first]), reg_to_register_object(Matcher::_regEncode[src_first]));
 744 #ifndef PRODUCT
 745       } else if (!do_size) {
 746         if (size != 0) st->print("\n\t");
 747         st->print(FMSR "   R_%s, R_%s\t! spill",OptoReg::regname(dst_first), OptoReg::regname(src_first));
 748 #endif
 749       }
 750       size += 4;
 751     }
 752   }
 753 
 754   // check for float reg -> int reg move
 755   if (src_first_rc == rc_float && dst_first_rc == rc_int) {
 756     // Further check for aligned-adjacent pair, so we can use a single instruction
 757     if (src_second_rc != rc_bad) {
 758       assert((src_first&1)==0 && src_first+1 == src_second, "pairs of registers must be aligned/contiguous");
 759       assert((dst_first&1)==0 && dst_first+1 == dst_second, "pairs of registers must be aligned/contiguous");
 760       assert(src_second_rc == rc_float && dst_second_rc == rc_int, "unsupported");
 761       if (cbuf) {
 762 #ifdef AARCH64
 763         __ fmov_xd(reg_to_register_object(Matcher::_regEncode[dst_first]), reg_to_FloatRegister_object(Matcher::_regEncode[src_first]));
 764 #else
 765         __ fmrrd(reg_to_register_object(Matcher::_regEncode[dst_first]), reg_to_register_object(Matcher::_regEncode[dst_second]), reg_to_FloatRegister_object(Matcher::_regEncode[src_first]));
 766 #endif
 767 #ifndef PRODUCT
 768       } else if (!do_size) {
 769         if (size != 0) st->print("\n\t");
 770 #ifdef AARCH64
 771         st->print("FMOV_XD R_%s, R_%s\t! spill",OptoReg::regname(dst_first), OptoReg::regname(src_first));
 772 #else
 773         st->print("FMRRD   R_%s, R_%s, R_%s\t! spill",OptoReg::regname(dst_first), OptoReg::regname(dst_second), OptoReg::regname(src_first));
 774 #endif
 775 #endif
 776       }
 777       return size + 4;
 778     } else {
 779       if (cbuf) {
 780         __ fmrs(reg_to_register_object(Matcher::_regEncode[dst_first]), reg_to_FloatRegister_object(Matcher::_regEncode[src_first]));
 781 #ifndef PRODUCT
 782       } else if (!do_size) {
 783         if (size != 0) st->print("\n\t");
 784         st->print(FMRS "   R_%s, R_%s\t! spill",OptoReg::regname(dst_first), OptoReg::regname(src_first));
 785 #endif
 786       }
 787       size += 4;
 788     }
 789   }
 790 
 791   // --------------------------------------------------------------------
 792   // Check for hi bits still needing moving.  Only happens for misaligned
 793   // arguments to native calls.
 794   if (src_second == dst_second)
 795     return size;               // Self copy; no move
 796   assert( src_second_rc != rc_bad && dst_second_rc != rc_bad, "src_second & dst_second cannot be Bad" );
 797 
 798 #ifndef AARCH64
 799   // Check for integer reg-reg copy.  Hi bits are stuck up in the top
 800   // 32-bits of a 64-bit register, but are needed in low bits of another
 801   // register (else it's a hi-bits-to-hi-bits copy which should have
 802   // happened already as part of a 64-bit move)
 803   if (src_second_rc == rc_int && dst_second_rc == rc_int) {
 804     if (cbuf) {
 805       __ mov(reg_to_register_object(Matcher::_regEncode[dst_second]), reg_to_register_object(Matcher::_regEncode[src_second]));
 806 #ifndef PRODUCT
 807     } else if (!do_size) {
 808       if (size != 0) st->print("\n\t");
 809       st->print("MOV    R_%s, R_%s\t# spill high",
 810                 Matcher::regName[dst_second],
 811                 Matcher::regName[src_second]);
 812 #endif
 813     }
 814     return size+4;
 815   }
 816 
 817   // Check for high word integer store
 818   if (src_second_rc == rc_int && dst_second_rc == rc_stack) {
 819     int offset = ra_->reg2offset(dst_second);
 820 
 821     if (cbuf && !is_memoryP(offset)) {
 822       ra_->C->record_method_not_compilable("unable to handle large constant offsets");
 823       return 0;
 824     } else {
 825       if (cbuf) {
 826         __ str(reg_to_register_object(Matcher::_regEncode[src_second]), Address(SP, offset));
 827 #ifndef PRODUCT
 828       } else if (!do_size) {
 829         if (size != 0) st->print("\n\t");
 830         st->print("STR   R_%s,[R_SP + #%d]\t! spill",OptoReg::regname(src_second), offset);
 831 #endif
 832       }
 833     }
 834     return size + 4;
 835   }
 836 
 837   // Check for high word integer load
 838   if (dst_second_rc == rc_int && src_second_rc == rc_stack) {
 839     int offset = ra_->reg2offset(src_second);
 840     if (cbuf && !is_memoryP(offset)) {
 841       ra_->C->record_method_not_compilable("unable to handle large constant offsets");
 842       return 0;
 843     } else {
 844       if (cbuf) {
 845         __ ldr(reg_to_register_object(Matcher::_regEncode[dst_second]), Address(SP, offset));
 846 #ifndef PRODUCT
 847       } else if (!do_size) {
 848         if (size != 0) st->print("\n\t");
 849         st->print("LDR   R_%s,[R_SP + #%d]\t! spill",OptoReg::regname(dst_second), offset);
 850 #endif
 851       }
 852     }
 853     return size + 4;
 854   }
 855 #endif
 856 
 857   Unimplemented();
 858   return 0; // Mute compiler
 859 }
 860 
 861 #ifndef PRODUCT
 862 void MachSpillCopyNode::format( PhaseRegAlloc *ra_, outputStream *st ) const {
 863   implementation( NULL, ra_, false, st );
 864 }
 865 #endif
 866 
 867 void MachSpillCopyNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 868   implementation( &cbuf, ra_, false, NULL );
 869 }
 870 
 871 uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
 872   return implementation( NULL, ra_, true, NULL );
 873 }
 874 
 875 //=============================================================================
 876 #ifndef PRODUCT
 877 void MachNopNode::format( PhaseRegAlloc *, outputStream *st ) const {
 878   st->print("NOP \t# %d bytes pad for loops and calls", 4 * _count);
 879 }
 880 #endif
 881 
 882 void MachNopNode::emit(CodeBuffer &cbuf, PhaseRegAlloc * ) const {
 883   MacroAssembler _masm(&cbuf);
 884   for(int i = 0; i < _count; i += 1) {
 885     __ nop();
 886   }
 887 }
 888 
 889 uint MachNopNode::size(PhaseRegAlloc *ra_) const {
 890   return 4 * _count;
 891 }
 892 
 893 
 894 //=============================================================================
 895 #ifndef PRODUCT
 896 void BoxLockNode::format( PhaseRegAlloc *ra_, outputStream *st ) const {
 897   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
 898   int reg = ra_->get_reg_first(this);
 899   st->print("ADD    %s,R_SP+#%d",Matcher::regName[reg], offset);
 900 }
 901 #endif
 902 
 903 void BoxLockNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 904   MacroAssembler _masm(&cbuf);
 905   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
 906   int reg = ra_->get_encode(this);
 907   Register dst = reg_to_register_object(reg);
 908 
 909   if (is_aimm(offset)) {
 910     __ add(dst, SP, offset);
 911   } else {
 912     __ mov_slow(dst, offset);
 913 #ifdef AARCH64
 914     __ add(dst, SP, dst, ex_lsl);
 915 #else
 916     __ add(dst, SP, dst);
 917 #endif
 918   }
 919 }
 920 
 921 uint BoxLockNode::size(PhaseRegAlloc *ra_) const {
 922   // BoxLockNode is not a MachNode, so we can't just call MachNode::size(ra_)
 923   assert(ra_ == ra_->C->regalloc(), "sanity");
 924   return ra_->C->scratch_emit_size(this);
 925 }
 926 
 927 //=============================================================================
 928 #ifndef PRODUCT
 929 #ifdef AARCH64
 930 #define R_RTEMP "R_R16"
 931 #else
 932 #define R_RTEMP "R_R12"
 933 #endif
 934 void MachUEPNode::format( PhaseRegAlloc *ra_, outputStream *st ) const {
 935   st->print_cr("\nUEP:");
 936   if (UseCompressedClassPointers) {
 937     st->print_cr("\tLDR_w " R_RTEMP ",[R_R0 + oopDesc::klass_offset_in_bytes]\t! Inline cache check");
 938     st->print_cr("\tdecode_klass " R_RTEMP);
 939   } else {
 940     st->print_cr("\tLDR   " R_RTEMP ",[R_R0 + oopDesc::klass_offset_in_bytes]\t! Inline cache check");
 941   }
 942   st->print_cr("\tCMP   " R_RTEMP ",R_R8" );
 943   st->print   ("\tB.NE  SharedRuntime::handle_ic_miss_stub");
 944 }
 945 #endif
 946 
 947 void MachUEPNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 948   MacroAssembler _masm(&cbuf);
 949   Register iCache  = reg_to_register_object(Matcher::inline_cache_reg_encode());
 950   assert(iCache == Ricklass, "should be");
 951   Register receiver = R0;
 952 
 953   __ load_klass(Rtemp, receiver);
 954   __ cmp(Rtemp, iCache);
 955 #ifdef AARCH64
 956   Label match;
 957   __ b(match, eq);
 958   __ jump(SharedRuntime::get_ic_miss_stub(), relocInfo::runtime_call_type, Rtemp);
 959   __ bind(match);
 960 #else
 961   __ jump(SharedRuntime::get_ic_miss_stub(), relocInfo::runtime_call_type, noreg, ne);
 962 #endif
 963 }
 964 
 965 uint MachUEPNode::size(PhaseRegAlloc *ra_) const {
 966   return MachNode::size(ra_);
 967 }
 968 
 969 
 970 //=============================================================================
 971 
 972 // Emit exception handler code.
 973 int HandlerImpl::emit_exception_handler(CodeBuffer& cbuf) {
 974   MacroAssembler _masm(&cbuf);
 975 
 976   address base = __ start_a_stub(size_exception_handler());
 977   if (base == NULL) {
 978     ciEnv::current()->record_failure("CodeCache is full");
 979     return 0;  // CodeBuffer::expand failed
 980   }
 981 
 982   int offset = __ offset();
 983 
 984   // OK to trash LR, because exception blob will kill it
 985   __ jump(OptoRuntime::exception_blob()->entry_point(), relocInfo::runtime_call_type, LR_tmp);
 986 
 987   assert(__ offset() - offset <= (int) size_exception_handler(), "overflow");
 988 
 989   __ end_a_stub();
 990 
 991   return offset;
 992 }
 993 
 994 int HandlerImpl::emit_deopt_handler(CodeBuffer& cbuf) {
 995   // Can't use any of the current frame's registers as we may have deopted
 996   // at a poll and everything can be live.
 997   MacroAssembler _masm(&cbuf);
 998 
 999   address base = __ start_a_stub(size_deopt_handler());
1000   if (base == NULL) {
1001     ciEnv::current()->record_failure("CodeCache is full");
1002     return 0;  // CodeBuffer::expand failed
1003   }
1004 
1005   int offset = __ offset();
1006   address deopt_pc = __ pc();
1007 
1008 #ifdef AARCH64
1009   // See LR saved by caller in sharedRuntime_arm.cpp
1010   // see also hse1 ws
1011   // see also LIR_Assembler::emit_deopt_handler
1012 
1013   __ raw_push(LR, LR); // preserve LR in both slots
1014   __ mov_relative_address(LR, deopt_pc);
1015   __ str(LR, Address(SP, 1 * wordSize)); // save deopt PC
1016   // OK to kill LR, because deopt blob will restore it from SP[0]
1017   __ jump(SharedRuntime::deopt_blob()->unpack(), relocInfo::runtime_call_type, LR_tmp);
1018 #else
1019   __ sub(SP, SP, wordSize); // make room for saved PC
1020   __ push(LR); // save LR that may be live when we get here
1021   __ mov_relative_address(LR, deopt_pc);
1022   __ str(LR, Address(SP, wordSize)); // save deopt PC
1023   __ pop(LR); // restore LR
1024   __ jump(SharedRuntime::deopt_blob()->unpack(), relocInfo::runtime_call_type, noreg);
1025 #endif
1026 
1027   assert(__ offset() - offset <= (int) size_deopt_handler(), "overflow");
1028 
1029   __ end_a_stub();
1030   return offset;
1031 }
1032 
1033 const bool Matcher::match_rule_supported(int opcode) {
1034   if (!has_match_rule(opcode))
1035     return false;
1036 
1037   switch (opcode) {
1038   case Op_PopCountI:
1039   case Op_PopCountL:
1040     if (!UsePopCountInstruction)
1041       return false;
1042     break;
1043   case Op_LShiftCntV:
1044   case Op_RShiftCntV:
1045   case Op_AddVB:
1046   case Op_AddVS:
1047   case Op_AddVI:
1048   case Op_AddVL:
1049   case Op_SubVB:
1050   case Op_SubVS:
1051   case Op_SubVI:
1052   case Op_SubVL:
1053   case Op_MulVS:
1054   case Op_MulVI:
1055   case Op_LShiftVB:
1056   case Op_LShiftVS:
1057   case Op_LShiftVI:
1058   case Op_LShiftVL:
1059   case Op_RShiftVB:
1060   case Op_RShiftVS:
1061   case Op_RShiftVI:
1062   case Op_RShiftVL:
1063   case Op_URShiftVB:
1064   case Op_URShiftVS:
1065   case Op_URShiftVI:
1066   case Op_URShiftVL:
1067   case Op_AndV:
1068   case Op_OrV:
1069   case Op_XorV:
1070     return VM_Version::has_simd();
1071   case Op_LoadVector:
1072   case Op_StoreVector:
1073   case Op_AddVF:
1074   case Op_SubVF:
1075   case Op_MulVF:
1076 #ifdef AARCH64
1077     return VM_Version::has_simd();
1078 #else
1079     return VM_Version::has_vfp() || VM_Version::has_simd();
1080 #endif
1081   case Op_AddVD:
1082   case Op_SubVD:
1083   case Op_MulVD:
1084   case Op_DivVF:
1085   case Op_DivVD:
1086 #ifdef AARCH64
1087     return VM_Version::has_simd();
1088 #else
1089     return VM_Version::has_vfp();
1090 #endif
1091   }
1092 
1093   return true;  // Per default match rules are supported.
1094 }
1095 
1096 const bool Matcher::match_rule_supported_vector(int opcode, int vlen) {
1097 
1098   // TODO
1099   // identify extra cases that we might want to provide match rules for
1100   // e.g. Op_ vector nodes and other intrinsics while guarding with vlen
1101   bool ret_value = match_rule_supported(opcode);
1102   // Add rules here.
1103 
1104   return ret_value;  // Per default match rules are supported.
1105 }
1106 
1107 const bool Matcher::has_predicated_vectors(void) {
1108   return false;
1109 }
1110 
1111 const int Matcher::float_pressure(int default_pressure_threshold) {
1112   return default_pressure_threshold;
1113 }
1114 
1115 int Matcher::regnum_to_fpu_offset(int regnum) {
1116   return regnum - 32; // The FP registers are in the second chunk
1117 }
1118 
1119 // Vector width in bytes
1120 const int Matcher::vector_width_in_bytes(BasicType bt) {
1121   return MaxVectorSize;
1122 }
1123 
1124 // Vector ideal reg corresponding to specified size in bytes
1125 const uint Matcher::vector_ideal_reg(int size) {
1126   assert(MaxVectorSize >= size, "");
1127   switch(size) {
1128     case  8: return Op_VecD;
1129     case 16: return Op_VecX;
1130   }
1131   ShouldNotReachHere();
1132   return 0;
1133 }
1134 
1135 const uint Matcher::vector_shift_count_ideal_reg(int size) {
1136   return vector_ideal_reg(size);
1137 }
1138 
1139 // Limits on vector size (number of elements) loaded into vector.
1140 const int Matcher::max_vector_size(const BasicType bt) {
1141   assert(is_java_primitive(bt), "only primitive type vectors");
1142   return vector_width_in_bytes(bt)/type2aelembytes(bt);
1143 }
1144 
1145 const int Matcher::min_vector_size(const BasicType bt) {
1146   assert(is_java_primitive(bt), "only primitive type vectors");
1147   return 8/type2aelembytes(bt);
1148 }
1149 
1150 // ARM doesn't support misaligned vectors store/load.
1151 const bool Matcher::misaligned_vectors_ok() {
1152   return false;
1153 }
1154 
1155 // ARM doesn't support AES intrinsics
1156 const bool Matcher::pass_original_key_for_aes() {
1157   return false;
1158 }
1159 
1160 const bool Matcher::convL2FSupported(void) {
1161 #ifdef AARCH64
1162   return true;
1163 #else
1164   return false;
1165 #endif
1166 }
1167 
1168 // Is this branch offset short enough that a short branch can be used?
1169 //
1170 // NOTE: If the platform does not provide any short branch variants, then
1171 //       this method should return false for offset 0.
1172 bool Matcher::is_short_branch_offset(int rule, int br_size, int offset) {
1173   // The passed offset is relative to address of the branch.
1174   // On ARM a branch displacement is calculated relative to address
1175   // of the branch + 8.
1176   //
1177   // offset -= 8;
1178   // return (Assembler::is_simm24(offset));
1179   return false;
1180 }
1181 
1182 const bool Matcher::isSimpleConstant64(jlong value) {
1183   // Will one (StoreL ConL) be cheaper than two (StoreI ConI)?.
1184 #ifdef AARCH64
1185   return (value == 0);
1186 #else
1187   return false;
1188 #endif
1189 }
1190 
1191 // No scaling for the parameter the ClearArray node.
1192 const bool Matcher::init_array_count_is_in_bytes = true;
1193 
1194 #ifdef AARCH64
1195 const int Matcher::long_cmove_cost() { return 1; }
1196 #else
1197 // Needs 2 CMOV's for longs.
1198 const int Matcher::long_cmove_cost() { return 2; }
1199 #endif
1200 
1201 #ifdef AARCH64
1202 const int Matcher::float_cmove_cost() { return 1; }
1203 #else
1204 // CMOVF/CMOVD are expensive on ARM.
1205 const int Matcher::float_cmove_cost() { return ConditionalMoveLimit; }
1206 #endif
1207 
1208 // Does the CPU require late expand (see block.cpp for description of late expand)?
1209 const bool Matcher::require_postalloc_expand = false;
1210 
1211 // Do we need to mask the count passed to shift instructions or does
1212 // the cpu only look at the lower 5/6 bits anyway?
1213 // FIXME: does this handle vector shifts as well?
1214 #ifdef AARCH64
1215 const bool Matcher::need_masked_shift_count = false;
1216 #else
1217 const bool Matcher::need_masked_shift_count = true;
1218 #endif
1219 
1220 const bool Matcher::convi2l_type_required = true;
1221 
1222 // Should the Matcher clone shifts on addressing modes, expecting them
1223 // to be subsumed into complex addressing expressions or compute them
1224 // into registers?
1225 bool Matcher::clone_address_expressions(AddPNode* m, Matcher::MStack& mstack, VectorSet& address_visited) {
1226   return clone_base_plus_offset_address(m, mstack, address_visited);
1227 }
1228 
1229 void Compile::reshape_address(AddPNode* addp) {
1230 }
1231 
1232 bool Matcher::narrow_oop_use_complex_address() {
1233   NOT_LP64(ShouldNotCallThis());
1234   assert(UseCompressedOops, "only for compressed oops code");
1235   return false;
1236 }
1237 
1238 bool Matcher::narrow_klass_use_complex_address() {
1239   NOT_LP64(ShouldNotCallThis());
1240   assert(UseCompressedClassPointers, "only for compressed klass code");
1241   return false;
1242 }
1243 
1244 bool Matcher::const_oop_prefer_decode() {
1245   NOT_LP64(ShouldNotCallThis());
1246   return true;
1247 }
1248 
1249 bool Matcher::const_klass_prefer_decode() {
1250   NOT_LP64(ShouldNotCallThis());
1251   return true;
1252 }
1253 
1254 // Is it better to copy float constants, or load them directly from memory?
1255 // Intel can load a float constant from a direct address, requiring no
1256 // extra registers.  Most RISCs will have to materialize an address into a
1257 // register first, so they would do better to copy the constant from stack.
1258 const bool Matcher::rematerialize_float_constants = false;
1259 
1260 // If CPU can load and store mis-aligned doubles directly then no fixup is
1261 // needed.  Else we split the double into 2 integer pieces and move it
1262 // piece-by-piece.  Only happens when passing doubles into C code as the
1263 // Java calling convention forces doubles to be aligned.
1264 #ifdef AARCH64
1265 // On stack replacement support:
1266 // We don't need Load[DL]_unaligned support, because interpreter stack
1267 // has correct alignment
1268 const bool Matcher::misaligned_doubles_ok = true;
1269 #else
1270 const bool Matcher::misaligned_doubles_ok = false;
1271 #endif
1272 
1273 // No-op on ARM.
1274 void Matcher::pd_implicit_null_fixup(MachNode *node, uint idx) {
1275 }
1276 
1277 // Advertise here if the CPU requires explicit rounding operations
1278 // to implement the UseStrictFP mode.
1279 const bool Matcher::strict_fp_requires_explicit_rounding = false;
1280 
1281 // Are floats converted to double when stored to stack during deoptimization?
1282 // ARM does not handle callee-save floats.
1283 bool Matcher::float_in_double() {
1284   return false;
1285 }
1286 
1287 // Do ints take an entire long register or just half?
1288 // Note that we if-def off of _LP64.
1289 // The relevant question is how the int is callee-saved.  In _LP64
1290 // the whole long is written but de-opt'ing will have to extract
1291 // the relevant 32 bits, in not-_LP64 only the low 32 bits is written.
1292 #ifdef _LP64
1293 const bool Matcher::int_in_long = true;
1294 #else
1295 const bool Matcher::int_in_long = false;
1296 #endif
1297 
1298 // Return whether or not this register is ever used as an argument.  This
1299 // function is used on startup to build the trampoline stubs in generateOptoStub.
1300 // Registers not mentioned will be killed by the VM call in the trampoline, and
1301 // arguments in those registers not be available to the callee.
1302 bool Matcher::can_be_java_arg( int reg ) {
1303 #ifdef AARCH64
1304   if (reg >= R_R0_num && reg < R_R8_num) return true;
1305   if (reg >= R_V0_num && reg <= R_V7b_num && ((reg & 3) < 2)) return true;
1306 #else
1307   if (reg == R_R0_num ||
1308       reg == R_R1_num ||
1309       reg == R_R2_num ||
1310       reg == R_R3_num) return true;
1311 
1312   if (reg >= R_S0_num &&
1313       reg <= R_S13_num) return true;
1314 #endif
1315   return false;
1316 }
1317 
1318 bool Matcher::is_spillable_arg( int reg ) {
1319   return can_be_java_arg(reg);
1320 }
1321 
1322 bool Matcher::use_asm_for_ldiv_by_con( jlong divisor ) {
1323   return false;
1324 }
1325 
1326 // Register for DIVI projection of divmodI
1327 RegMask Matcher::divI_proj_mask() {
1328   ShouldNotReachHere();
1329   return RegMask();
1330 }
1331 
1332 // Register for MODI projection of divmodI
1333 RegMask Matcher::modI_proj_mask() {
1334   ShouldNotReachHere();
1335   return RegMask();
1336 }
1337 
1338 // Register for DIVL projection of divmodL
1339 RegMask Matcher::divL_proj_mask() {
1340   ShouldNotReachHere();
1341   return RegMask();
1342 }
1343 
1344 // Register for MODL projection of divmodL
1345 RegMask Matcher::modL_proj_mask() {
1346   ShouldNotReachHere();
1347   return RegMask();
1348 }
1349 
1350 const RegMask Matcher::method_handle_invoke_SP_save_mask() {
1351   return FP_REGP_mask();
1352 }
1353 
1354 bool maybe_far_call(const CallNode *n) {
1355   return !MacroAssembler::_reachable_from_cache(n->as_Call()->entry_point());
1356 }
1357 
1358 bool maybe_far_call(const MachCallNode *n) {
1359   return !MacroAssembler::_reachable_from_cache(n->as_MachCall()->entry_point());
1360 }
1361 
1362 %}
1363 
1364 //----------ENCODING BLOCK-----------------------------------------------------
1365 // This block specifies the encoding classes used by the compiler to output
1366 // byte streams.  Encoding classes are parameterized macros used by
1367 // Machine Instruction Nodes in order to generate the bit encoding of the
1368 // instruction.  Operands specify their base encoding interface with the
1369 // interface keyword.  There are currently supported four interfaces,
1370 // REG_INTER, CONST_INTER, MEMORY_INTER, & COND_INTER.  REG_INTER causes an
1371 // operand to generate a function which returns its register number when
1372 // queried.   CONST_INTER causes an operand to generate a function which
1373 // returns the value of the constant when queried.  MEMORY_INTER causes an
1374 // operand to generate four functions which return the Base Register, the
1375 // Index Register, the Scale Value, and the Offset Value of the operand when
1376 // queried.  COND_INTER causes an operand to generate six functions which
1377 // return the encoding code (ie - encoding bits for the instruction)
1378 // associated with each basic boolean condition for a conditional instruction.
1379 //
1380 // Instructions specify two basic values for encoding.  Again, a function
1381 // is available to check if the constant displacement is an oop. They use the
1382 // ins_encode keyword to specify their encoding classes (which must be
1383 // a sequence of enc_class names, and their parameters, specified in
1384 // the encoding block), and they use the
1385 // opcode keyword to specify, in order, their primary, secondary, and
1386 // tertiary opcode.  Only the opcode sections which a particular instruction
1387 // needs for encoding need to be specified.
1388 encode %{
1389   enc_class call_epilog %{
1390     // nothing
1391   %}
1392 
1393   enc_class Java_To_Runtime (method meth) %{
1394     // CALL directly to the runtime
1395     emit_call_reloc(cbuf, as_MachCall(), $meth, runtime_call_Relocation::spec());
1396   %}
1397 
1398   enc_class Java_Static_Call (method meth) %{
1399     // CALL to fixup routine.  Fixup routine uses ScopeDesc info to determine
1400     // who we intended to call.
1401 
1402     if ( !_method) {
1403       emit_call_reloc(cbuf, as_MachCall(), $meth, runtime_call_Relocation::spec());
1404     } else {
1405       int method_index = resolved_method_index(cbuf);
1406       RelocationHolder rspec = _optimized_virtual ? opt_virtual_call_Relocation::spec(method_index)
1407                                                   : static_call_Relocation::spec(method_index);
1408       emit_call_reloc(cbuf, as_MachCall(), $meth, rspec);
1409 
1410       // Emit stubs for static call.
1411       address stub = CompiledStaticCall::emit_to_interp_stub(cbuf);
1412       if (stub == NULL) {
1413         ciEnv::current()->record_failure("CodeCache is full");
1414         return;
1415       }
1416     }
1417   %}
1418 
1419   enc_class save_last_PC %{
1420     // preserve mark
1421     address mark = cbuf.insts()->mark();
1422     debug_only(int off0 = cbuf.insts_size());
1423     MacroAssembler _masm(&cbuf);
1424     int ret_addr_offset = as_MachCall()->ret_addr_offset();
1425     __ adr(LR, mark + ret_addr_offset);
1426     __ str(LR, Address(Rthread, JavaThread::last_Java_pc_offset()));
1427     debug_only(int off1 = cbuf.insts_size());
1428     assert(off1 - off0 == 2 * Assembler::InstructionSize, "correct size prediction");
1429     // restore mark
1430     cbuf.insts()->set_mark(mark);
1431   %}
1432 
1433   enc_class preserve_SP %{
1434     // preserve mark
1435     address mark = cbuf.insts()->mark();
1436     debug_only(int off0 = cbuf.insts_size());
1437     MacroAssembler _masm(&cbuf);
1438     // FP is preserved across all calls, even compiled calls.
1439     // Use it to preserve SP in places where the callee might change the SP.
1440     __ mov(Rmh_SP_save, SP);
1441     debug_only(int off1 = cbuf.insts_size());
1442     assert(off1 - off0 == 4, "correct size prediction");
1443     // restore mark
1444     cbuf.insts()->set_mark(mark);
1445   %}
1446 
1447   enc_class restore_SP %{
1448     MacroAssembler _masm(&cbuf);
1449     __ mov(SP, Rmh_SP_save);
1450   %}
1451 
1452   enc_class Java_Dynamic_Call (method meth) %{
1453     MacroAssembler _masm(&cbuf);
1454     Register R8_ic_reg = reg_to_register_object(Matcher::inline_cache_reg_encode());
1455     assert(R8_ic_reg == Ricklass, "should be");
1456     __ set_inst_mark();
1457 #ifdef AARCH64
1458 // TODO: see C1 LIR_Assembler::ic_call()
1459     InlinedAddress oop_literal((address)Universe::non_oop_word());
1460     int offset = __ offset();
1461     int fixed_size = mov_oop_size * 4;
1462     if (VM_Version::prefer_moves_over_load_literal()) {
1463       uintptr_t val = (uintptr_t)Universe::non_oop_word();
1464       __ movz(R8_ic_reg, (val >>  0) & 0xffff,  0);
1465       __ movk(R8_ic_reg, (val >> 16) & 0xffff, 16);
1466       __ movk(R8_ic_reg, (val >> 32) & 0xffff, 32);
1467       __ movk(R8_ic_reg, (val >> 48) & 0xffff, 48);
1468     } else {
1469       __ ldr_literal(R8_ic_reg, oop_literal);
1470     }
1471     assert(__ offset() - offset == fixed_size, "bad mov_oop size");
1472 #else
1473     __ movw(R8_ic_reg, ((unsigned int)Universe::non_oop_word()) & 0xffff);
1474     __ movt(R8_ic_reg, ((unsigned int)Universe::non_oop_word()) >> 16);
1475 #endif
1476     address  virtual_call_oop_addr = __ inst_mark();
1477     // CALL to fixup routine.  Fixup routine uses ScopeDesc info to determine
1478     // who we intended to call.
1479     int method_index = resolved_method_index(cbuf);
1480     __ relocate(virtual_call_Relocation::spec(virtual_call_oop_addr, method_index));
1481     emit_call_reloc(cbuf, as_MachCall(), $meth, RelocationHolder::none);
1482 #ifdef AARCH64
1483     if (!VM_Version::prefer_moves_over_load_literal()) {
1484       Label skip_literal;
1485       __ b(skip_literal);
1486       int off2 = __ offset();
1487       __ bind_literal(oop_literal);
1488       if (__ offset() - off2 == wordSize) {
1489         // no padding, so insert nop for worst-case sizing
1490         __ nop();
1491       }
1492       __ bind(skip_literal);
1493     }
1494 #endif
1495   %}
1496 
1497   enc_class LdReplImmI(immI src, regD dst, iRegI tmp, int cnt, int wth) %{
1498     // FIXME: load from constant table?
1499     // Load a constant replicated "count" times with width "width"
1500     int count = $cnt$$constant;
1501     int width = $wth$$constant;
1502     assert(count*width == 4, "sanity");
1503     int val = $src$$constant;
1504     if (width < 4) {
1505       int bit_width = width * 8;
1506       val &= (((int)1) << bit_width) - 1; // mask off sign bits
1507       for (int i = 0; i < count - 1; i++) {
1508         val |= (val << bit_width);
1509       }
1510     }
1511     MacroAssembler _masm(&cbuf);
1512 
1513     if (val == -1) {
1514       __ mvn($tmp$$Register, 0);
1515     } else if (val == 0) {
1516       __ mov($tmp$$Register, 0);
1517     } else {
1518       __ movw($tmp$$Register, val & 0xffff);
1519       __ movt($tmp$$Register, (unsigned int)val >> 16);
1520     }
1521     __ fmdrr($dst$$FloatRegister, $tmp$$Register, $tmp$$Register);
1522   %}
1523 
1524   enc_class LdReplImmF(immF src, regD dst, iRegI tmp) %{
1525     // Replicate float con 2 times and pack into vector (8 bytes) in regD.
1526     float fval = $src$$constant;
1527     int val = *((int*)&fval);
1528     MacroAssembler _masm(&cbuf);
1529 
1530     if (val == -1) {
1531       __ mvn($tmp$$Register, 0);
1532     } else if (val == 0) {
1533       __ mov($tmp$$Register, 0);
1534     } else {
1535       __ movw($tmp$$Register, val & 0xffff);
1536       __ movt($tmp$$Register, (unsigned int)val >> 16);
1537     }
1538     __ fmdrr($dst$$FloatRegister, $tmp$$Register, $tmp$$Register);
1539   %}
1540 
1541   enc_class enc_String_Compare(R0RegP str1, R1RegP str2, R2RegI cnt1, R3RegI cnt2, iRegI result, iRegI tmp1, iRegI tmp2) %{
1542     Label Ldone, Lloop;
1543     MacroAssembler _masm(&cbuf);
1544 
1545     Register   str1_reg = $str1$$Register;
1546     Register   str2_reg = $str2$$Register;
1547     Register   cnt1_reg = $cnt1$$Register; // int
1548     Register   cnt2_reg = $cnt2$$Register; // int
1549     Register   tmp1_reg = $tmp1$$Register;
1550     Register   tmp2_reg = $tmp2$$Register;
1551     Register result_reg = $result$$Register;
1552 
1553     assert_different_registers(str1_reg, str2_reg, cnt1_reg, cnt2_reg, tmp1_reg, tmp2_reg);
1554 
1555     // Compute the minimum of the string lengths(str1_reg) and the
1556     // difference of the string lengths (stack)
1557 
1558     // See if the lengths are different, and calculate min in str1_reg.
1559     // Stash diff in tmp2 in case we need it for a tie-breaker.
1560     __ subs_32(tmp2_reg, cnt1_reg, cnt2_reg);
1561 #ifdef AARCH64
1562     Label Lskip;
1563     __ _lsl_w(cnt1_reg, cnt1_reg, exact_log2(sizeof(jchar))); // scale the limit
1564     __ b(Lskip, mi);
1565     __ _lsl_w(cnt1_reg, cnt2_reg, exact_log2(sizeof(jchar))); // scale the limit
1566     __ bind(Lskip);
1567 #else
1568     __ mov(cnt1_reg, AsmOperand(cnt1_reg, lsl, exact_log2(sizeof(jchar)))); // scale the limit
1569     __ mov(cnt1_reg, AsmOperand(cnt2_reg, lsl, exact_log2(sizeof(jchar))), pl); // scale the limit
1570 #endif
1571 
1572     // reallocate cnt1_reg, cnt2_reg, result_reg
1573     // Note:  limit_reg holds the string length pre-scaled by 2
1574     Register limit_reg = cnt1_reg;
1575     Register  chr2_reg = cnt2_reg;
1576     Register  chr1_reg = tmp1_reg;
1577     // str{12} are the base pointers
1578 
1579     // Is the minimum length zero?
1580     __ cmp_32(limit_reg, 0);
1581     if (result_reg != tmp2_reg) {
1582       __ mov(result_reg, tmp2_reg, eq);
1583     }
1584     __ b(Ldone, eq);
1585 
1586     // Load first characters
1587     __ ldrh(chr1_reg, Address(str1_reg, 0));
1588     __ ldrh(chr2_reg, Address(str2_reg, 0));
1589 
1590     // Compare first characters
1591     __ subs(chr1_reg, chr1_reg, chr2_reg);
1592     if (result_reg != chr1_reg) {
1593       __ mov(result_reg, chr1_reg, ne);
1594     }
1595     __ b(Ldone, ne);
1596 
1597     {
1598       // Check after comparing first character to see if strings are equivalent
1599       // Check if the strings start at same location
1600       __ cmp(str1_reg, str2_reg);
1601       // Check if the length difference is zero
1602       __ cond_cmp(tmp2_reg, 0, eq);
1603       __ mov(result_reg, 0, eq); // result is zero
1604       __ b(Ldone, eq);
1605       // Strings might not be equal
1606     }
1607 
1608     __ subs(chr1_reg, limit_reg, 1 * sizeof(jchar));
1609     if (result_reg != tmp2_reg) {
1610       __ mov(result_reg, tmp2_reg, eq);
1611     }
1612     __ b(Ldone, eq);
1613 
1614     // Shift str1_reg and str2_reg to the end of the arrays, negate limit
1615     __ add(str1_reg, str1_reg, limit_reg);
1616     __ add(str2_reg, str2_reg, limit_reg);
1617     __ neg(limit_reg, chr1_reg);  // limit = -(limit-2)
1618 
1619     // Compare the rest of the characters
1620     __ bind(Lloop);
1621     __ ldrh(chr1_reg, Address(str1_reg, limit_reg));
1622     __ ldrh(chr2_reg, Address(str2_reg, limit_reg));
1623     __ subs(chr1_reg, chr1_reg, chr2_reg);
1624     if (result_reg != chr1_reg) {
1625       __ mov(result_reg, chr1_reg, ne);
1626     }
1627     __ b(Ldone, ne);
1628 
1629     __ adds(limit_reg, limit_reg, sizeof(jchar));
1630     __ b(Lloop, ne);
1631 
1632     // If strings are equal up to min length, return the length difference.
1633     if (result_reg != tmp2_reg) {
1634       __ mov(result_reg, tmp2_reg);
1635     }
1636 
1637     // Otherwise, return the difference between the first mismatched chars.
1638     __ bind(Ldone);
1639   %}
1640 
1641   enc_class enc_String_Equals(R0RegP str1, R1RegP str2, R2RegI cnt, iRegI result, iRegI tmp1, iRegI tmp2) %{
1642     Label Lchar, Lchar_loop, Ldone, Lequal;
1643     MacroAssembler _masm(&cbuf);
1644 
1645     Register   str1_reg = $str1$$Register;
1646     Register   str2_reg = $str2$$Register;
1647     Register    cnt_reg = $cnt$$Register; // int
1648     Register   tmp1_reg = $tmp1$$Register;
1649     Register   tmp2_reg = $tmp2$$Register;
1650     Register result_reg = $result$$Register;
1651 
1652     assert_different_registers(str1_reg, str2_reg, cnt_reg, tmp1_reg, tmp2_reg, result_reg);
1653 
1654     __ cmp(str1_reg, str2_reg); //same char[] ?
1655     __ b(Lequal, eq);
1656 
1657     __ cbz_32(cnt_reg, Lequal); // count == 0
1658 
1659     //rename registers
1660     Register limit_reg = cnt_reg;
1661     Register  chr1_reg = tmp1_reg;
1662     Register  chr2_reg = tmp2_reg;
1663 
1664     __ logical_shift_left(limit_reg, limit_reg, exact_log2(sizeof(jchar)));
1665 
1666     //check for alignment and position the pointers to the ends
1667     __ orr(chr1_reg, str1_reg, str2_reg);
1668     __ tst(chr1_reg, 0x3);
1669 
1670     // notZero means at least one not 4-byte aligned.
1671     // We could optimize the case when both arrays are not aligned
1672     // but it is not frequent case and it requires additional checks.
1673     __ b(Lchar, ne);
1674 
1675     // Compare char[] arrays aligned to 4 bytes.
1676     __ char_arrays_equals(str1_reg, str2_reg, limit_reg, result_reg,
1677                           chr1_reg, chr2_reg, Ldone);
1678 
1679     __ b(Lequal); // equal
1680 
1681     // char by char compare
1682     __ bind(Lchar);
1683     __ mov(result_reg, 0);
1684     __ add(str1_reg, limit_reg, str1_reg);
1685     __ add(str2_reg, limit_reg, str2_reg);
1686     __ neg(limit_reg, limit_reg); //negate count
1687 
1688     // Lchar_loop
1689     __ bind(Lchar_loop);
1690     __ ldrh(chr1_reg, Address(str1_reg, limit_reg));
1691     __ ldrh(chr2_reg, Address(str2_reg, limit_reg));
1692     __ cmp(chr1_reg, chr2_reg);
1693     __ b(Ldone, ne);
1694     __ adds(limit_reg, limit_reg, sizeof(jchar));
1695     __ b(Lchar_loop, ne);
1696 
1697     __ bind(Lequal);
1698     __ mov(result_reg, 1);  //equal
1699 
1700     __ bind(Ldone);
1701   %}
1702 
1703   enc_class enc_Array_Equals(R0RegP ary1, R1RegP ary2, iRegI tmp1, iRegI tmp2, iRegI tmp3, iRegI result) %{
1704     Label Ldone, Lloop, Lequal;
1705     MacroAssembler _masm(&cbuf);
1706 
1707     Register   ary1_reg = $ary1$$Register;
1708     Register   ary2_reg = $ary2$$Register;
1709     Register   tmp1_reg = $tmp1$$Register;
1710     Register   tmp2_reg = $tmp2$$Register;
1711     Register   tmp3_reg = $tmp3$$Register;
1712     Register result_reg = $result$$Register;
1713 
1714     assert_different_registers(ary1_reg, ary2_reg, tmp1_reg, tmp2_reg, tmp3_reg, result_reg);
1715 
1716     int length_offset  = arrayOopDesc::length_offset_in_bytes();
1717     int base_offset    = arrayOopDesc::base_offset_in_bytes(T_CHAR);
1718 
1719     // return true if the same array
1720 #ifdef AARCH64
1721     __ cmp(ary1_reg, ary2_reg);
1722     __ b(Lequal, eq);
1723 
1724     __ mov(result_reg, 0);
1725 
1726     __ cbz(ary1_reg, Ldone); // not equal
1727 
1728     __ cbz(ary2_reg, Ldone); // not equal
1729 #else
1730     __ teq(ary1_reg, ary2_reg);
1731     __ mov(result_reg, 1, eq);
1732     __ b(Ldone, eq); // equal
1733 
1734     __ tst(ary1_reg, ary1_reg);
1735     __ mov(result_reg, 0, eq);
1736     __ b(Ldone, eq);    // not equal
1737 
1738     __ tst(ary2_reg, ary2_reg);
1739     __ mov(result_reg, 0, eq);
1740     __ b(Ldone, eq);    // not equal
1741 #endif
1742 
1743     //load the lengths of arrays
1744     __ ldr_s32(tmp1_reg, Address(ary1_reg, length_offset)); // int
1745     __ ldr_s32(tmp2_reg, Address(ary2_reg, length_offset)); // int
1746 
1747     // return false if the two arrays are not equal length
1748 #ifdef AARCH64
1749     __ cmp_w(tmp1_reg, tmp2_reg);
1750     __ b(Ldone, ne);    // not equal
1751 
1752     __ cbz_w(tmp1_reg, Lequal); // zero-length arrays are equal
1753 #else
1754     __ teq_32(tmp1_reg, tmp2_reg);
1755     __ mov(result_reg, 0, ne);
1756     __ b(Ldone, ne);    // not equal
1757 
1758     __ tst(tmp1_reg, tmp1_reg);
1759     __ mov(result_reg, 1, eq);
1760     __ b(Ldone, eq);    // zero-length arrays are equal
1761 #endif
1762 
1763     // load array addresses
1764     __ add(ary1_reg, ary1_reg, base_offset);
1765     __ add(ary2_reg, ary2_reg, base_offset);
1766 
1767     // renaming registers
1768     Register chr1_reg  =  tmp3_reg;   // for characters in ary1
1769     Register chr2_reg  =  tmp2_reg;   // for characters in ary2
1770     Register limit_reg =  tmp1_reg;   // length
1771 
1772     // set byte count
1773     __ logical_shift_left_32(limit_reg, limit_reg, exact_log2(sizeof(jchar)));
1774 
1775     // Compare char[] arrays aligned to 4 bytes.
1776     __ char_arrays_equals(ary1_reg, ary2_reg, limit_reg, result_reg,
1777                           chr1_reg, chr2_reg, Ldone);
1778     __ bind(Lequal);
1779     __ mov(result_reg, 1);  //equal
1780 
1781     __ bind(Ldone);
1782     %}
1783 %}
1784 
1785 //----------FRAME--------------------------------------------------------------
1786 // Definition of frame structure and management information.
1787 //
1788 //  S T A C K   L A Y O U T    Allocators stack-slot number
1789 //                             |   (to get allocators register number
1790 //  G  Owned by    |        |  v    add VMRegImpl::stack0)
1791 //  r   CALLER     |        |
1792 //  o     |        +--------+      pad to even-align allocators stack-slot
1793 //  w     V        |  pad0  |        numbers; owned by CALLER
1794 //  t   -----------+--------+----> Matcher::_in_arg_limit, unaligned
1795 //  h     ^        |   in   |  5
1796 //        |        |  args  |  4   Holes in incoming args owned by SELF
1797 //  |     |        |        |  3
1798 //  |     |        +--------+
1799 //  V     |        | old out|      Empty on Intel, window on Sparc
1800 //        |    old |preserve|      Must be even aligned.
1801 //        |     SP-+--------+----> Matcher::_old_SP, 8 (or 16 in LP64)-byte aligned
1802 //        |        |   in   |  3   area for Intel ret address
1803 //     Owned by    |preserve|      Empty on Sparc.
1804 //       SELF      +--------+
1805 //        |        |  pad2  |  2   pad to align old SP
1806 //        |        +--------+  1
1807 //        |        | locks  |  0
1808 //        |        +--------+----> VMRegImpl::stack0, 8 (or 16 in LP64)-byte aligned
1809 //        |        |  pad1  | 11   pad to align new SP
1810 //        |        +--------+
1811 //        |        |        | 10
1812 //        |        | spills |  9   spills
1813 //        V        |        |  8   (pad0 slot for callee)
1814 //      -----------+--------+----> Matcher::_out_arg_limit, unaligned
1815 //        ^        |  out   |  7
1816 //        |        |  args  |  6   Holes in outgoing args owned by CALLEE
1817 //     Owned by    +--------+
1818 //      CALLEE     | new out|  6   Empty on Intel, window on Sparc
1819 //        |    new |preserve|      Must be even-aligned.
1820 //        |     SP-+--------+----> Matcher::_new_SP, even aligned
1821 //        |        |        |
1822 //
1823 // Note 1: Only region 8-11 is determined by the allocator.  Region 0-5 is
1824 //         known from SELF's arguments and the Java calling convention.
1825 //         Region 6-7 is determined per call site.
1826 // Note 2: If the calling convention leaves holes in the incoming argument
1827 //         area, those holes are owned by SELF.  Holes in the outgoing area
1828 //         are owned by the CALLEE.  Holes should not be nessecary in the
1829 //         incoming area, as the Java calling convention is completely under
1830 //         the control of the AD file.  Doubles can be sorted and packed to
1831 //         avoid holes.  Holes in the outgoing arguments may be nessecary for
1832 //         varargs C calling conventions.
1833 // Note 3: Region 0-3 is even aligned, with pad2 as needed.  Region 3-5 is
1834 //         even aligned with pad0 as needed.
1835 //         Region 6 is even aligned.  Region 6-7 is NOT even aligned;
1836 //         region 6-11 is even aligned; it may be padded out more so that
1837 //         the region from SP to FP meets the minimum stack alignment.
1838 
1839 frame %{
1840   // What direction does stack grow in (assumed to be same for native & Java)
1841   stack_direction(TOWARDS_LOW);
1842 
1843   // These two registers define part of the calling convention
1844   // between compiled code and the interpreter.
1845   inline_cache_reg(R_Ricklass);          // Inline Cache Register or Method* for I2C
1846   interpreter_method_oop_reg(R_Rmethod); // Method Oop Register when calling interpreter
1847 
1848   // Optional: name the operand used by cisc-spilling to access [stack_pointer + offset]
1849   cisc_spilling_operand_name(indOffset);
1850 
1851   // Number of stack slots consumed by a Monitor enter
1852   sync_stack_slots(1 * VMRegImpl::slots_per_word);
1853 
1854   // Compiled code's Frame Pointer
1855 #ifdef AARCH64
1856   frame_pointer(R_SP);
1857 #else
1858   frame_pointer(R_R13);
1859 #endif
1860 
1861   // Stack alignment requirement
1862   stack_alignment(StackAlignmentInBytes);
1863   //  LP64: Alignment size in bytes (128-bit -> 16 bytes)
1864   // !LP64: Alignment size in bytes (64-bit  ->  8 bytes)
1865 
1866   // Number of stack slots between incoming argument block and the start of
1867   // a new frame.  The PROLOG must add this many slots to the stack.  The
1868   // EPILOG must remove this many slots.
1869   // FP + LR
1870   in_preserve_stack_slots(2 * VMRegImpl::slots_per_word);
1871 
1872   // Number of outgoing stack slots killed above the out_preserve_stack_slots
1873   // for calls to C.  Supports the var-args backing area for register parms.
1874   // ADLC doesn't support parsing expressions, so I folded the math by hand.
1875   varargs_C_out_slots_killed( 0);
1876 
1877   // The after-PROLOG location of the return address.  Location of
1878   // return address specifies a type (REG or STACK) and a number
1879   // representing the register number (i.e. - use a register name) or
1880   // stack slot.
1881   // Ret Addr is on stack in slot 0 if no locks or verification or alignment.
1882   // Otherwise, it is above the locks and verification slot and alignment word
1883   return_addr(STACK - 1*VMRegImpl::slots_per_word +
1884               align_up((Compile::current()->in_preserve_stack_slots() +
1885                         Compile::current()->fixed_slots()),
1886                        stack_alignment_in_slots()));
1887 
1888   // Body of function which returns an OptoRegs array locating
1889   // arguments either in registers or in stack slots for calling
1890   // java
1891   calling_convention %{
1892     (void) SharedRuntime::java_calling_convention(sig_bt, regs, length, is_outgoing);
1893 
1894   %}
1895 
1896   // Body of function which returns an OptoRegs array locating
1897   // arguments either in registers or in stack slots for callin
1898   // C.
1899   c_calling_convention %{
1900     // This is obviously always outgoing
1901     (void) SharedRuntime::c_calling_convention(sig_bt, regs, /*regs2=*/NULL, length);
1902   %}
1903 
1904   // Location of compiled Java return values.  Same as C
1905   return_value %{
1906     return c2::return_value(ideal_reg);
1907   %}
1908 
1909 %}
1910 
1911 //----------ATTRIBUTES---------------------------------------------------------
1912 //----------Instruction Attributes---------------------------------------------
1913 ins_attrib ins_cost(DEFAULT_COST); // Required cost attribute
1914 ins_attrib ins_size(32);           // Required size attribute (in bits)
1915 ins_attrib ins_short_branch(0);    // Required flag: is this instruction a
1916                                    // non-matching short branch variant of some
1917                                                             // long branch?
1918 
1919 //----------OPERANDS-----------------------------------------------------------
1920 // Operand definitions must precede instruction definitions for correct parsing
1921 // in the ADLC because operands constitute user defined types which are used in
1922 // instruction definitions.
1923 
1924 //----------Simple Operands----------------------------------------------------
1925 // Immediate Operands
1926 // Integer Immediate: 32-bit
1927 operand immI() %{
1928   match(ConI);
1929 
1930   op_cost(0);
1931   // formats are generated automatically for constants and base registers
1932   format %{ %}
1933   interface(CONST_INTER);
1934 %}
1935 
1936 // Integer Immediate: 8-bit unsigned - for VMOV
1937 operand immU8() %{
1938   predicate(0 <= n->get_int() && (n->get_int() <= 255));
1939   match(ConI);
1940   op_cost(0);
1941 
1942   format %{ %}
1943   interface(CONST_INTER);
1944 %}
1945 
1946 // Integer Immediate: 16-bit
1947 operand immI16() %{
1948   predicate((n->get_int() >> 16) == 0 && VM_Version::supports_movw());
1949   match(ConI);
1950   op_cost(0);
1951 
1952   format %{ %}
1953   interface(CONST_INTER);
1954 %}
1955 
1956 #ifndef AARCH64
1957 // Integer Immediate: offset for half and double word loads and stores
1958 operand immIHD() %{
1959   predicate(is_memoryHD(n->get_int()));
1960   match(ConI);
1961   op_cost(0);
1962   format %{ %}
1963   interface(CONST_INTER);
1964 %}
1965 
1966 // Integer Immediate: offset for fp loads and stores
1967 operand immIFP() %{
1968   predicate(is_memoryfp(n->get_int()) && ((n->get_int() & 3) == 0));
1969   match(ConI);
1970   op_cost(0);
1971 
1972   format %{ %}
1973   interface(CONST_INTER);
1974 %}
1975 #endif
1976 
1977 // Valid scale values for addressing modes and shifts
1978 operand immU5() %{
1979   predicate(0 <= n->get_int() && (n->get_int() <= 31));
1980   match(ConI);
1981   op_cost(0);
1982 
1983   format %{ %}
1984   interface(CONST_INTER);
1985 %}
1986 
1987 // Integer Immediate: 6-bit
1988 operand immU6Big() %{
1989   predicate(n->get_int() >= 32 && n->get_int() <= 63);
1990   match(ConI);
1991   op_cost(0);
1992   format %{ %}
1993   interface(CONST_INTER);
1994 %}
1995 
1996 // Integer Immediate: 0-bit
1997 operand immI0() %{
1998   predicate(n->get_int() == 0);
1999   match(ConI);
2000   op_cost(0);
2001 
2002   format %{ %}
2003   interface(CONST_INTER);
2004 %}
2005 
2006 // Integer Immediate: the value 1
2007 operand immI_1() %{
2008   predicate(n->get_int() == 1);
2009   match(ConI);
2010   op_cost(0);
2011 
2012   format %{ %}
2013   interface(CONST_INTER);
2014 %}
2015 
2016 // Integer Immediate: the value 2
2017 operand immI_2() %{
2018   predicate(n->get_int() == 2);
2019   match(ConI);
2020   op_cost(0);
2021 
2022   format %{ %}
2023   interface(CONST_INTER);
2024 %}
2025 
2026 // Integer Immediate: the value 3
2027 operand immI_3() %{
2028   predicate(n->get_int() == 3);
2029   match(ConI);
2030   op_cost(0);
2031 
2032   format %{ %}
2033   interface(CONST_INTER);
2034 %}
2035 
2036 // Integer Immediate: the value 4
2037 operand immI_4() %{
2038   predicate(n->get_int() == 4);
2039   match(ConI);
2040   op_cost(0);
2041 
2042   format %{ %}
2043   interface(CONST_INTER);
2044 %}
2045 
2046 // Integer Immediate: the value 8
2047 operand immI_8() %{
2048   predicate(n->get_int() == 8);
2049   match(ConI);
2050   op_cost(0);
2051 
2052   format %{ %}
2053   interface(CONST_INTER);
2054 %}
2055 
2056 // Int Immediate non-negative
2057 operand immU31()
2058 %{
2059   predicate(n->get_int() >= 0);
2060   match(ConI);
2061 
2062   op_cost(0);
2063   format %{ %}
2064   interface(CONST_INTER);
2065 %}
2066 
2067 // Integer Immediate: the values 32-63
2068 operand immI_32_63() %{
2069   predicate(n->get_int() >= 32 && n->get_int() <= 63);
2070   match(ConI);
2071   op_cost(0);
2072 
2073   format %{ %}
2074   interface(CONST_INTER);
2075 %}
2076 
2077 // Immediates for special shifts (sign extend)
2078 
2079 // Integer Immediate: the value 16
2080 operand immI_16() %{
2081   predicate(n->get_int() == 16);
2082   match(ConI);
2083   op_cost(0);
2084 
2085   format %{ %}
2086   interface(CONST_INTER);
2087 %}
2088 
2089 // Integer Immediate: the value 24
2090 operand immI_24() %{
2091   predicate(n->get_int() == 24);
2092   match(ConI);
2093   op_cost(0);
2094 
2095   format %{ %}
2096   interface(CONST_INTER);
2097 %}
2098 
2099 // Integer Immediate: the value 255
2100 operand immI_255() %{
2101   predicate( n->get_int() == 255 );
2102   match(ConI);
2103   op_cost(0);
2104 
2105   format %{ %}
2106   interface(CONST_INTER);
2107 %}
2108 
2109 // Integer Immediate: the value 65535
2110 operand immI_65535() %{
2111   predicate(n->get_int() == 65535);
2112   match(ConI);
2113   op_cost(0);
2114 
2115   format %{ %}
2116   interface(CONST_INTER);
2117 %}
2118 
2119 // Integer Immediates for arithmetic instructions
2120 
2121 operand aimmI() %{
2122   predicate(is_aimm(n->get_int()));
2123   match(ConI);
2124   op_cost(0);
2125 
2126   format %{ %}
2127   interface(CONST_INTER);
2128 %}
2129 
2130 operand aimmIneg() %{
2131   predicate(is_aimm(-n->get_int()));
2132   match(ConI);
2133   op_cost(0);
2134 
2135   format %{ %}
2136   interface(CONST_INTER);
2137 %}
2138 
2139 operand aimmU31() %{
2140   predicate((0 <= n->get_int()) && is_aimm(n->get_int()));
2141   match(ConI);
2142   op_cost(0);
2143 
2144   format %{ %}
2145   interface(CONST_INTER);
2146 %}
2147 
2148 // Integer Immediates for logical instructions
2149 
2150 operand limmI() %{
2151   predicate(is_limmI(n->get_int()));
2152   match(ConI);
2153   op_cost(0);
2154 
2155   format %{ %}
2156   interface(CONST_INTER);
2157 %}
2158 
2159 operand limmIlow8() %{
2160   predicate(is_limmI_low(n->get_int(), 8));
2161   match(ConI);
2162   op_cost(0);
2163 
2164   format %{ %}
2165   interface(CONST_INTER);
2166 %}
2167 
2168 operand limmU31() %{
2169   predicate(0 <= n->get_int() && is_limmI(n->get_int()));
2170   match(ConI);
2171   op_cost(0);
2172 
2173   format %{ %}
2174   interface(CONST_INTER);
2175 %}
2176 
2177 operand limmIn() %{
2178   predicate(is_limmI(~n->get_int()));
2179   match(ConI);
2180   op_cost(0);
2181 
2182   format %{ %}
2183   interface(CONST_INTER);
2184 %}
2185 
2186 #ifdef AARCH64
2187 // Long Immediate: for logical instruction
2188 operand limmL() %{
2189   predicate(is_limmL(n->get_long()));
2190   match(ConL);
2191   op_cost(0);
2192 
2193   format %{ %}
2194   interface(CONST_INTER);
2195 %}
2196 
2197 operand limmLn() %{
2198   predicate(is_limmL(~n->get_long()));
2199   match(ConL);
2200   op_cost(0);
2201 
2202   format %{ %}
2203   interface(CONST_INTER);
2204 %}
2205 
2206 // Long Immediate: for arithmetic instruction
2207 operand aimmL() %{
2208   predicate(is_aimm(n->get_long()));
2209   match(ConL);
2210   op_cost(0);
2211 
2212   format %{ %}
2213   interface(CONST_INTER);
2214 %}
2215 
2216 operand aimmLneg() %{
2217   predicate(is_aimm(-n->get_long()));
2218   match(ConL);
2219   op_cost(0);
2220 
2221   format %{ %}
2222   interface(CONST_INTER);
2223 %}
2224 #endif // AARCH64
2225 
2226 // Long Immediate: the value FF
2227 operand immL_FF() %{
2228   predicate( n->get_long() == 0xFFL );
2229   match(ConL);
2230   op_cost(0);
2231 
2232   format %{ %}
2233   interface(CONST_INTER);
2234 %}
2235 
2236 // Long Immediate: the value FFFF
2237 operand immL_FFFF() %{
2238   predicate( n->get_long() == 0xFFFFL );
2239   match(ConL);
2240   op_cost(0);
2241 
2242   format %{ %}
2243   interface(CONST_INTER);
2244 %}
2245 
2246 // Pointer Immediate: 32 or 64-bit
2247 operand immP() %{
2248   match(ConP);
2249 
2250   op_cost(5);
2251   // formats are generated automatically for constants and base registers
2252   format %{ %}
2253   interface(CONST_INTER);
2254 %}
2255 
2256 operand immP0() %{
2257   predicate(n->get_ptr() == 0);
2258   match(ConP);
2259   op_cost(0);
2260 
2261   format %{ %}
2262   interface(CONST_INTER);
2263 %}
2264 
2265 operand immP_poll() %{
2266   predicate(n->get_ptr() != 0 && n->get_ptr() == (intptr_t)os::get_polling_page());
2267   match(ConP);
2268 
2269   // formats are generated automatically for constants and base registers
2270   format %{ %}
2271   interface(CONST_INTER);
2272 %}
2273 
2274 // Pointer Immediate
2275 operand immN()
2276 %{
2277   match(ConN);
2278 
2279   op_cost(10);
2280   format %{ %}
2281   interface(CONST_INTER);
2282 %}
2283 
2284 operand immNKlass()
2285 %{
2286   match(ConNKlass);
2287 
2288   op_cost(10);
2289   format %{ %}
2290   interface(CONST_INTER);
2291 %}
2292 
2293 // NULL Pointer Immediate
2294 operand immN0()
2295 %{
2296   predicate(n->get_narrowcon() == 0);
2297   match(ConN);
2298 
2299   op_cost(0);
2300   format %{ %}
2301   interface(CONST_INTER);
2302 %}
2303 
2304 operand immL() %{
2305   match(ConL);
2306   op_cost(40);
2307   // formats are generated automatically for constants and base registers
2308   format %{ %}
2309   interface(CONST_INTER);
2310 %}
2311 
2312 operand immL0() %{
2313   predicate(n->get_long() == 0L);
2314   match(ConL);
2315   op_cost(0);
2316   // formats are generated automatically for constants and base registers
2317   format %{ %}
2318   interface(CONST_INTER);
2319 %}
2320 
2321 // Long Immediate: 16-bit
2322 operand immL16() %{
2323   predicate(n->get_long() >= 0 && n->get_long() < (1<<16)  && VM_Version::supports_movw());
2324   match(ConL);
2325   op_cost(0);
2326 
2327   format %{ %}
2328   interface(CONST_INTER);
2329 %}
2330 
2331 // Long Immediate: low 32-bit mask
2332 operand immL_32bits() %{
2333   predicate(n->get_long() == 0xFFFFFFFFL);
2334   match(ConL);
2335   op_cost(0);
2336 
2337   format %{ %}
2338   interface(CONST_INTER);
2339 %}
2340 
2341 // Double Immediate
2342 operand immD() %{
2343   match(ConD);
2344 
2345   op_cost(40);
2346   format %{ %}
2347   interface(CONST_INTER);
2348 %}
2349 
2350 // Double Immediate: +0.0d.
2351 operand immD0() %{
2352   predicate(jlong_cast(n->getd()) == 0);
2353 
2354   match(ConD);
2355   op_cost(0);
2356   format %{ %}
2357   interface(CONST_INTER);
2358 %}
2359 
2360 operand imm8D() %{
2361   predicate(Assembler::double_num(n->getd()).can_be_imm8());
2362   match(ConD);
2363 
2364   op_cost(0);
2365   format %{ %}
2366   interface(CONST_INTER);
2367 %}
2368 
2369 // Float Immediate
2370 operand immF() %{
2371   match(ConF);
2372 
2373   op_cost(20);
2374   format %{ %}
2375   interface(CONST_INTER);
2376 %}
2377 
2378 // Float Immediate: +0.0f
2379 operand immF0() %{
2380   predicate(jint_cast(n->getf()) == 0);
2381   match(ConF);
2382 
2383   op_cost(0);
2384   format %{ %}
2385   interface(CONST_INTER);
2386 %}
2387 
2388 // Float Immediate: encoded as 8 bits
2389 operand imm8F() %{
2390   predicate(Assembler::float_num(n->getf()).can_be_imm8());
2391   match(ConF);
2392 
2393   op_cost(0);
2394   format %{ %}
2395   interface(CONST_INTER);
2396 %}
2397 
2398 // Integer Register Operands
2399 // Integer Register
2400 operand iRegI() %{
2401   constraint(ALLOC_IN_RC(int_reg));
2402   match(RegI);
2403   match(R0RegI);
2404   match(R1RegI);
2405   match(R2RegI);
2406   match(R3RegI);
2407 #ifdef AARCH64
2408   match(ZRRegI);
2409 #else
2410   match(R12RegI);
2411 #endif
2412 
2413   format %{ %}
2414   interface(REG_INTER);
2415 %}
2416 
2417 // Pointer Register
2418 operand iRegP() %{
2419   constraint(ALLOC_IN_RC(ptr_reg));
2420   match(RegP);
2421   match(R0RegP);
2422   match(R1RegP);
2423   match(R2RegP);
2424   match(RExceptionRegP);
2425   match(R8RegP);
2426   match(R9RegP);
2427   match(RthreadRegP); // FIXME: move to sp_ptr_RegP?
2428   match(R12RegP);
2429   match(LRRegP);
2430 
2431   match(sp_ptr_RegP);
2432   match(store_ptr_RegP);
2433 
2434   format %{ %}
2435   interface(REG_INTER);
2436 %}
2437 
2438 // GPRs + Rthread + SP
2439 operand sp_ptr_RegP() %{
2440   constraint(ALLOC_IN_RC(sp_ptr_reg));
2441   match(RegP);
2442   match(iRegP);
2443   match(SPRegP); // FIXME: check cost
2444 
2445   format %{ %}
2446   interface(REG_INTER);
2447 %}
2448 
2449 #ifdef AARCH64
2450 // Like sp_ptr_reg, but exclude regs (Aarch64 SP) that can't be
2451 // stored directly.  Includes ZR, so can't be used as a destination.
2452 operand store_ptr_RegP() %{
2453   constraint(ALLOC_IN_RC(store_ptr_reg));
2454   match(RegP);
2455   match(iRegP);
2456   match(ZRRegP);
2457 
2458   format %{ %}
2459   interface(REG_INTER);
2460 %}
2461 
2462 operand store_RegI() %{
2463   constraint(ALLOC_IN_RC(store_reg));
2464   match(RegI);
2465   match(iRegI);
2466   match(ZRRegI);
2467 
2468   format %{ %}
2469   interface(REG_INTER);
2470 %}
2471 
2472 operand store_RegL() %{
2473   constraint(ALLOC_IN_RC(store_ptr_reg));
2474   match(RegL);
2475   match(iRegL);
2476   match(ZRRegL);
2477 
2478   format %{ %}
2479   interface(REG_INTER);
2480 %}
2481 
2482 operand store_RegN() %{
2483   constraint(ALLOC_IN_RC(store_reg));
2484   match(RegN);
2485   match(iRegN);
2486   match(ZRRegN);
2487 
2488   format %{ %}
2489   interface(REG_INTER);
2490 %}
2491 #endif
2492 
2493 operand R0RegP() %{
2494   constraint(ALLOC_IN_RC(R0_regP));
2495   match(iRegP);
2496 
2497   format %{ %}
2498   interface(REG_INTER);
2499 %}
2500 
2501 operand R1RegP() %{
2502   constraint(ALLOC_IN_RC(R1_regP));
2503   match(iRegP);
2504 
2505   format %{ %}
2506   interface(REG_INTER);
2507 %}
2508 
2509 operand R2RegP() %{
2510   constraint(ALLOC_IN_RC(R2_regP));
2511   match(iRegP);
2512 
2513   format %{ %}
2514   interface(REG_INTER);
2515 %}
2516 
2517 operand RExceptionRegP() %{
2518   constraint(ALLOC_IN_RC(Rexception_regP));
2519   match(iRegP);
2520 
2521   format %{ %}
2522   interface(REG_INTER);
2523 %}
2524 
2525 operand RthreadRegP() %{
2526   constraint(ALLOC_IN_RC(Rthread_regP));
2527   match(iRegP);
2528 
2529   format %{ %}
2530   interface(REG_INTER);
2531 %}
2532 
2533 operand IPRegP() %{
2534   constraint(ALLOC_IN_RC(IP_regP));
2535   match(iRegP);
2536 
2537   format %{ %}
2538   interface(REG_INTER);
2539 %}
2540 
2541 operand LRRegP() %{
2542   constraint(ALLOC_IN_RC(LR_regP));
2543   match(iRegP);
2544 
2545   format %{ %}
2546   interface(REG_INTER);
2547 %}
2548 
2549 operand R0RegI() %{
2550   constraint(ALLOC_IN_RC(R0_regI));
2551   match(iRegI);
2552 
2553   format %{ %}
2554   interface(REG_INTER);
2555 %}
2556 
2557 operand R1RegI() %{
2558   constraint(ALLOC_IN_RC(R1_regI));
2559   match(iRegI);
2560 
2561   format %{ %}
2562   interface(REG_INTER);
2563 %}
2564 
2565 operand R2RegI() %{
2566   constraint(ALLOC_IN_RC(R2_regI));
2567   match(iRegI);
2568 
2569   format %{ %}
2570   interface(REG_INTER);
2571 %}
2572 
2573 operand R3RegI() %{
2574   constraint(ALLOC_IN_RC(R3_regI));
2575   match(iRegI);
2576 
2577   format %{ %}
2578   interface(REG_INTER);
2579 %}
2580 
2581 #ifndef AARCH64
2582 operand R12RegI() %{
2583   constraint(ALLOC_IN_RC(R12_regI));
2584   match(iRegI);
2585 
2586   format %{ %}
2587   interface(REG_INTER);
2588 %}
2589 #endif
2590 
2591 // Long Register
2592 operand iRegL() %{
2593   constraint(ALLOC_IN_RC(long_reg));
2594   match(RegL);
2595 #ifdef AARCH64
2596   match(iRegLd);
2597 #else
2598   match(R0R1RegL);
2599   match(R2R3RegL);
2600 #endif
2601 //match(iRegLex);
2602 
2603   format %{ %}
2604   interface(REG_INTER);
2605 %}
2606 
2607 operand iRegLd() %{
2608   constraint(ALLOC_IN_RC(long_reg_align));
2609   match(iRegL); // FIXME: allows unaligned R11/R12?
2610 
2611   format %{ %}
2612   interface(REG_INTER);
2613 %}
2614 
2615 #ifndef AARCH64
2616 // first long arg, or return value
2617 operand R0R1RegL() %{
2618   constraint(ALLOC_IN_RC(R0R1_regL));
2619   match(iRegL);
2620 
2621   format %{ %}
2622   interface(REG_INTER);
2623 %}
2624 
2625 operand R2R3RegL() %{
2626   constraint(ALLOC_IN_RC(R2R3_regL));
2627   match(iRegL);
2628 
2629   format %{ %}
2630   interface(REG_INTER);
2631 %}
2632 #endif
2633 
2634 // Condition Code Flag Register
2635 operand flagsReg() %{
2636   constraint(ALLOC_IN_RC(int_flags));
2637   match(RegFlags);
2638 
2639   format %{ "apsr" %}
2640   interface(REG_INTER);
2641 %}
2642 
2643 // Result of compare to 0 (TST)
2644 operand flagsReg_EQNELTGE() %{
2645   constraint(ALLOC_IN_RC(int_flags));
2646   match(RegFlags);
2647 
2648   format %{ "apsr_EQNELTGE" %}
2649   interface(REG_INTER);
2650 %}
2651 
2652 // Condition Code Register, unsigned comparisons.
2653 operand flagsRegU() %{
2654   constraint(ALLOC_IN_RC(int_flags));
2655   match(RegFlags);
2656 #ifdef TODO
2657   match(RegFlagsP);
2658 #endif
2659 
2660   format %{ "apsr_U" %}
2661   interface(REG_INTER);
2662 %}
2663 
2664 // Condition Code Register, pointer comparisons.
2665 operand flagsRegP() %{
2666   constraint(ALLOC_IN_RC(int_flags));
2667   match(RegFlags);
2668 
2669   format %{ "apsr_P" %}
2670   interface(REG_INTER);
2671 %}
2672 
2673 // Condition Code Register, long comparisons.
2674 #ifndef AARCH64
2675 operand flagsRegL_LTGE() %{
2676   constraint(ALLOC_IN_RC(int_flags));
2677   match(RegFlags);
2678 
2679   format %{ "apsr_L_LTGE" %}
2680   interface(REG_INTER);
2681 %}
2682 
2683 operand flagsRegL_EQNE() %{
2684   constraint(ALLOC_IN_RC(int_flags));
2685   match(RegFlags);
2686 
2687   format %{ "apsr_L_EQNE" %}
2688   interface(REG_INTER);
2689 %}
2690 
2691 operand flagsRegL_LEGT() %{
2692   constraint(ALLOC_IN_RC(int_flags));
2693   match(RegFlags);
2694 
2695   format %{ "apsr_L_LEGT" %}
2696   interface(REG_INTER);
2697 %}
2698 
2699 operand flagsRegUL_LTGE() %{
2700   constraint(ALLOC_IN_RC(int_flags));
2701   match(RegFlags);
2702 
2703   format %{ "apsr_UL_LTGE" %}
2704   interface(REG_INTER);
2705 %}
2706 
2707 operand flagsRegUL_EQNE() %{
2708   constraint(ALLOC_IN_RC(int_flags));
2709   match(RegFlags);
2710 
2711   format %{ "apsr_UL_EQNE" %}
2712   interface(REG_INTER);
2713 %}
2714 
2715 operand flagsRegUL_LEGT() %{
2716   constraint(ALLOC_IN_RC(int_flags));
2717   match(RegFlags);
2718 
2719   format %{ "apsr_UL_LEGT" %}
2720   interface(REG_INTER);
2721 %}
2722 #endif
2723 
2724 // Condition Code Register, floating comparisons, unordered same as "less".
2725 operand flagsRegF() %{
2726   constraint(ALLOC_IN_RC(float_flags));
2727   match(RegFlags);
2728 
2729   format %{ "fpscr_F" %}
2730   interface(REG_INTER);
2731 %}
2732 
2733 // Vectors
2734 operand vecD() %{
2735   constraint(ALLOC_IN_RC(actual_dflt_reg));
2736   match(VecD);
2737 
2738   format %{ %}
2739   interface(REG_INTER);
2740 %}
2741 
2742 operand vecX() %{
2743   constraint(ALLOC_IN_RC(vectorx_reg));
2744   match(VecX);
2745 
2746   format %{ %}
2747   interface(REG_INTER);
2748 %}
2749 
2750 operand regD() %{
2751   constraint(ALLOC_IN_RC(actual_dflt_reg));
2752   match(RegD);
2753   match(regD_low);
2754 
2755   format %{ %}
2756   interface(REG_INTER);
2757 %}
2758 
2759 operand regF() %{
2760   constraint(ALLOC_IN_RC(sflt_reg));
2761   match(RegF);
2762 
2763   format %{ %}
2764   interface(REG_INTER);
2765 %}
2766 
2767 operand regD_low() %{
2768   constraint(ALLOC_IN_RC(dflt_low_reg));
2769   match(RegD);
2770 
2771   format %{ %}
2772   interface(REG_INTER);
2773 %}
2774 
2775 // Special Registers
2776 
2777 // Method Register
2778 operand inline_cache_regP(iRegP reg) %{
2779   constraint(ALLOC_IN_RC(Ricklass_regP));
2780   match(reg);
2781   format %{ %}
2782   interface(REG_INTER);
2783 %}
2784 
2785 operand interpreter_method_oop_regP(iRegP reg) %{
2786   constraint(ALLOC_IN_RC(Rmethod_regP));
2787   match(reg);
2788   format %{ %}
2789   interface(REG_INTER);
2790 %}
2791 
2792 
2793 //----------Complex Operands---------------------------------------------------
2794 // Indirect Memory Reference
2795 operand indirect(sp_ptr_RegP reg) %{
2796   constraint(ALLOC_IN_RC(sp_ptr_reg));
2797   match(reg);
2798 
2799   op_cost(100);
2800   format %{ "[$reg]" %}
2801   interface(MEMORY_INTER) %{
2802     base($reg);
2803 #ifdef AARCH64
2804     index(0xff); // 0xff => no index
2805 #else
2806     index(0xf); // PC => no index
2807 #endif
2808     scale(0x0);
2809     disp(0x0);
2810   %}
2811 %}
2812 
2813 #ifdef AARCH64
2814 // Indirect with scaled*1 uimm12 offset
2815 operand indOffsetU12ScaleB(sp_ptr_RegP reg, immUL12 offset) %{
2816   constraint(ALLOC_IN_RC(sp_ptr_reg));
2817   match(AddP reg offset);
2818 
2819   op_cost(100);
2820   format %{ "[$reg + $offset]" %}
2821   interface(MEMORY_INTER) %{
2822     base($reg);
2823 #ifdef AARCH64
2824     index(0xff); // 0xff => no index
2825 #else
2826     index(0xf); // PC => no index
2827 #endif
2828     scale(0x0);
2829     disp($offset);
2830   %}
2831 %}
2832 
2833 // Indirect with scaled*2 uimm12 offset
2834 operand indOffsetU12ScaleS(sp_ptr_RegP reg, immUL12x2 offset) %{
2835   constraint(ALLOC_IN_RC(sp_ptr_reg));
2836   match(AddP reg offset);
2837 
2838   op_cost(100);
2839   format %{ "[$reg + $offset]" %}
2840   interface(MEMORY_INTER) %{
2841     base($reg);
2842 #ifdef AARCH64
2843     index(0xff); // 0xff => no index
2844 #else
2845     index(0xf); // PC => no index
2846 #endif
2847     scale(0x0);
2848     disp($offset);
2849   %}
2850 %}
2851 
2852 // Indirect with scaled*4 uimm12 offset
2853 operand indOffsetU12ScaleI(sp_ptr_RegP reg, immUL12x4 offset) %{
2854   constraint(ALLOC_IN_RC(sp_ptr_reg));
2855   match(AddP reg offset);
2856 
2857   op_cost(100);
2858   format %{ "[$reg + $offset]" %}
2859   interface(MEMORY_INTER) %{
2860     base($reg);
2861 #ifdef AARCH64
2862     index(0xff); // 0xff => no index
2863 #else
2864     index(0xf); // PC => no index
2865 #endif
2866     scale(0x0);
2867     disp($offset);
2868   %}
2869 %}
2870 
2871 // Indirect with scaled*8 uimm12 offset
2872 operand indOffsetU12ScaleL(sp_ptr_RegP reg, immUL12x8 offset) %{
2873   constraint(ALLOC_IN_RC(sp_ptr_reg));
2874   match(AddP reg offset);
2875 
2876   op_cost(100);
2877   format %{ "[$reg + $offset]" %}
2878   interface(MEMORY_INTER) %{
2879     base($reg);
2880 #ifdef AARCH64
2881     index(0xff); // 0xff => no index
2882 #else
2883     index(0xf); // PC => no index
2884 #endif
2885     scale(0x0);
2886     disp($offset);
2887   %}
2888 %}
2889 
2890 // Indirect with scaled*16 uimm12 offset
2891 operand indOffsetU12ScaleQ(sp_ptr_RegP reg, immUL12x16 offset) %{
2892   constraint(ALLOC_IN_RC(sp_ptr_reg));
2893   match(AddP reg offset);
2894 
2895   op_cost(100);
2896   format %{ "[$reg + $offset]" %}
2897   interface(MEMORY_INTER) %{
2898     base($reg);
2899 #ifdef AARCH64
2900     index(0xff); // 0xff => no index
2901 #else
2902     index(0xf); // PC => no index
2903 #endif
2904     scale(0x0);
2905     disp($offset);
2906   %}
2907 %}
2908 
2909 #else // ! AARCH64
2910 
2911 // Indirect with Offset in ]-4096, 4096[
2912 operand indOffset12(sp_ptr_RegP reg, immI12 offset) %{
2913   constraint(ALLOC_IN_RC(sp_ptr_reg));
2914   match(AddP reg offset);
2915 
2916   op_cost(100);
2917   format %{ "[$reg + $offset]" %}
2918   interface(MEMORY_INTER) %{
2919     base($reg);
2920 #ifdef AARCH64
2921     index(0xff); // 0xff => no index
2922 #else
2923     index(0xf); // PC => no index
2924 #endif
2925     scale(0x0);
2926     disp($offset);
2927   %}
2928 %}
2929 
2930 // Indirect with offset for float load/store
2931 operand indOffsetFP(sp_ptr_RegP reg, immIFP offset) %{
2932   constraint(ALLOC_IN_RC(sp_ptr_reg));
2933   match(AddP reg offset);
2934 
2935   op_cost(100);
2936   format %{ "[$reg + $offset]" %}
2937   interface(MEMORY_INTER) %{
2938     base($reg);
2939 #ifdef AARCH64
2940     index(0xff); // 0xff => no index
2941 #else
2942     index(0xf); // PC => no index
2943 #endif
2944     scale(0x0);
2945     disp($offset);
2946   %}
2947 %}
2948 
2949 // Indirect with Offset for half and double words
2950 operand indOffsetHD(sp_ptr_RegP reg, immIHD offset) %{
2951   constraint(ALLOC_IN_RC(sp_ptr_reg));
2952   match(AddP reg offset);
2953 
2954   op_cost(100);
2955   format %{ "[$reg + $offset]" %}
2956   interface(MEMORY_INTER) %{
2957     base($reg);
2958 #ifdef AARCH64
2959     index(0xff); // 0xff => no index
2960 #else
2961     index(0xf); // PC => no index
2962 #endif
2963     scale(0x0);
2964     disp($offset);
2965   %}
2966 %}
2967 
2968 // Indirect with Offset and Offset+4 in ]-1024, 1024[
2969 operand indOffsetFPx2(sp_ptr_RegP reg, immX10x2 offset) %{
2970   constraint(ALLOC_IN_RC(sp_ptr_reg));
2971   match(AddP reg offset);
2972 
2973   op_cost(100);
2974   format %{ "[$reg + $offset]" %}
2975   interface(MEMORY_INTER) %{
2976     base($reg);
2977 #ifdef AARCH64
2978     index(0xff); // 0xff => no index
2979 #else
2980     index(0xf); // PC => no index
2981 #endif
2982     scale(0x0);
2983     disp($offset);
2984   %}
2985 %}
2986 
2987 // Indirect with Offset and Offset+4 in ]-4096, 4096[
2988 operand indOffset12x2(sp_ptr_RegP reg, immI12x2 offset) %{
2989   constraint(ALLOC_IN_RC(sp_ptr_reg));
2990   match(AddP reg offset);
2991 
2992   op_cost(100);
2993   format %{ "[$reg + $offset]" %}
2994   interface(MEMORY_INTER) %{
2995     base($reg);
2996 #ifdef AARCH64
2997     index(0xff); // 0xff => no index
2998 #else
2999     index(0xf); // PC => no index
3000 #endif
3001     scale(0x0);
3002     disp($offset);
3003   %}
3004 %}
3005 #endif // !AARCH64
3006 
3007 // Indirect with Register Index
3008 operand indIndex(iRegP addr, iRegX index) %{
3009   constraint(ALLOC_IN_RC(ptr_reg));
3010   match(AddP addr index);
3011 
3012   op_cost(100);
3013   format %{ "[$addr + $index]" %}
3014   interface(MEMORY_INTER) %{
3015     base($addr);
3016     index($index);
3017     scale(0x0);
3018     disp(0x0);
3019   %}
3020 %}
3021 
3022 #ifdef AARCH64
3023 // Indirect Memory Times Scale Plus Index Register
3024 operand indIndexScaleS(iRegP addr, iRegX index, immI_1 scale) %{
3025   constraint(ALLOC_IN_RC(ptr_reg));
3026   match(AddP addr (LShiftX index scale));
3027 
3028   op_cost(100);
3029   format %{"[$addr + $index << $scale]" %}
3030   interface(MEMORY_INTER) %{
3031     base($addr);
3032     index($index);
3033     scale($scale);
3034     disp(0x0);
3035   %}
3036 %}
3037 
3038 // Indirect Memory Times Scale Plus 32-bit Index Register
3039 operand indIndexIScaleS(iRegP addr, iRegI index, immI_1 scale) %{
3040   constraint(ALLOC_IN_RC(ptr_reg));
3041   match(AddP addr (LShiftX (ConvI2L index) scale));
3042 
3043   op_cost(100);
3044   format %{"[$addr + $index.w << $scale]" %}
3045   interface(MEMORY_INTER) %{
3046     base($addr);
3047     index($index);
3048     scale($scale);
3049     disp(0x7fffffff); // sxtw
3050   %}
3051 %}
3052 
3053 // Indirect Memory Times Scale Plus Index Register
3054 operand indIndexScaleI(iRegP addr, iRegX index, immI_2 scale) %{
3055   constraint(ALLOC_IN_RC(ptr_reg));
3056   match(AddP addr (LShiftX index scale));
3057 
3058   op_cost(100);
3059   format %{"[$addr + $index << $scale]" %}
3060   interface(MEMORY_INTER) %{
3061     base($addr);
3062     index($index);
3063     scale($scale);
3064     disp(0x0);
3065   %}
3066 %}
3067 
3068 // Indirect Memory Times Scale Plus 32-bit Index Register
3069 operand indIndexIScaleI(iRegP addr, iRegI index, immI_2 scale) %{
3070   constraint(ALLOC_IN_RC(ptr_reg));
3071   match(AddP addr (LShiftX (ConvI2L index) scale));
3072 
3073   op_cost(100);
3074   format %{"[$addr + $index.w << $scale]" %}
3075   interface(MEMORY_INTER) %{
3076     base($addr);
3077     index($index);
3078     scale($scale);
3079     disp(0x7fffffff); // sxtw
3080   %}
3081 %}
3082 
3083 // Indirect Memory Times Scale Plus Index Register
3084 operand indIndexScaleL(iRegP addr, iRegX index, immI_3 scale) %{
3085   constraint(ALLOC_IN_RC(ptr_reg));
3086   match(AddP addr (LShiftX index scale));
3087 
3088   op_cost(100);
3089   format %{"[$addr + $index << $scale]" %}
3090   interface(MEMORY_INTER) %{
3091     base($addr);
3092     index($index);
3093     scale($scale);
3094     disp(0x0);
3095   %}
3096 %}
3097 
3098 // Indirect Memory Times Scale Plus 32-bit Index Register
3099 operand indIndexIScaleL(iRegP addr, iRegI index, immI_3 scale) %{
3100   constraint(ALLOC_IN_RC(ptr_reg));
3101   match(AddP addr (LShiftX (ConvI2L index) scale));
3102 
3103   op_cost(100);
3104   format %{"[$addr + $index.w << $scale]" %}
3105   interface(MEMORY_INTER) %{
3106     base($addr);
3107     index($index);
3108     scale($scale);
3109     disp(0x7fffffff); // sxtw
3110   %}
3111 %}
3112 
3113 // Indirect Memory Times Scale Plus Index Register
3114 operand indIndexScaleQ(iRegP addr, iRegX index, immI_4 scale) %{
3115   constraint(ALLOC_IN_RC(ptr_reg));
3116   match(AddP addr (LShiftX index scale));
3117 
3118   op_cost(100);
3119   format %{"[$addr + $index << $scale]" %}
3120   interface(MEMORY_INTER) %{
3121     base($addr);
3122     index($index);
3123     scale($scale);
3124     disp(0x0);
3125   %}
3126 %}
3127 
3128 // Indirect Memory Times Scale Plus 32-bit Index Register
3129 operand indIndexIScaleQ(iRegP addr, iRegI index, immI_4 scale) %{
3130   constraint(ALLOC_IN_RC(ptr_reg));
3131   match(AddP addr (LShiftX (ConvI2L index) scale));
3132 
3133   op_cost(100);
3134   format %{"[$addr + $index.w << $scale]" %}
3135   interface(MEMORY_INTER) %{
3136     base($addr);
3137     index($index);
3138     scale($scale);
3139     disp(0x7fffffff); // sxtw
3140   %}
3141 %}
3142 #else
3143 // Indirect Memory Times Scale Plus Index Register
3144 operand indIndexScale(iRegP addr, iRegX index, immU5 scale) %{
3145   constraint(ALLOC_IN_RC(ptr_reg));
3146   match(AddP addr (LShiftX index scale));
3147 
3148   op_cost(100);
3149   format %{"[$addr + $index << $scale]" %}
3150   interface(MEMORY_INTER) %{
3151     base($addr);
3152     index($index);
3153     scale($scale);
3154     disp(0x0);
3155   %}
3156 %}
3157 #endif
3158 
3159 // Operands for expressing Control Flow
3160 // NOTE:  Label is a predefined operand which should not be redefined in
3161 //        the AD file.  It is generically handled within the ADLC.
3162 
3163 //----------Conditional Branch Operands----------------------------------------
3164 // Comparison Op  - This is the operation of the comparison, and is limited to
3165 //                  the following set of codes:
3166 //                  L (<), LE (<=), G (>), GE (>=), E (==), NE (!=)
3167 //
3168 // Other attributes of the comparison, such as unsignedness, are specified
3169 // by the comparison instruction that sets a condition code flags register.
3170 // That result is represented by a flags operand whose subtype is appropriate
3171 // to the unsignedness (etc.) of the comparison.
3172 //
3173 // Later, the instruction which matches both the Comparison Op (a Bool) and
3174 // the flags (produced by the Cmp) specifies the coding of the comparison op
3175 // by matching a specific subtype of Bool operand below, such as cmpOpU.
3176 
3177 operand cmpOp() %{
3178   match(Bool);
3179 
3180   format %{ "" %}
3181   interface(COND_INTER) %{
3182     equal(0x0);
3183     not_equal(0x1);
3184     less(0xb);
3185     greater_equal(0xa);
3186     less_equal(0xd);
3187     greater(0xc);
3188     overflow(0x0); // unsupported/unimplemented
3189     no_overflow(0x0); // unsupported/unimplemented
3190   %}
3191 %}
3192 
3193 // integer comparison with 0, signed
3194 operand cmpOp0() %{
3195   match(Bool);
3196 
3197   format %{ "" %}
3198   interface(COND_INTER) %{
3199     equal(0x0);
3200     not_equal(0x1);
3201     less(0x4);
3202     greater_equal(0x5);
3203     less_equal(0xd); // unsupported
3204     greater(0xc); // unsupported
3205     overflow(0x0); // unsupported/unimplemented
3206     no_overflow(0x0); // unsupported/unimplemented
3207   %}
3208 %}
3209 
3210 // Comparison Op, unsigned
3211 operand cmpOpU() %{
3212   match(Bool);
3213 
3214   format %{ "u" %}
3215   interface(COND_INTER) %{
3216     equal(0x0);
3217     not_equal(0x1);
3218     less(0x3);
3219     greater_equal(0x2);
3220     less_equal(0x9);
3221     greater(0x8);
3222     overflow(0x0); // unsupported/unimplemented
3223     no_overflow(0x0); // unsupported/unimplemented
3224   %}
3225 %}
3226 
3227 // Comparison Op, pointer (same as unsigned)
3228 operand cmpOpP() %{
3229   match(Bool);
3230 
3231   format %{ "p" %}
3232   interface(COND_INTER) %{
3233     equal(0x0);
3234     not_equal(0x1);
3235     less(0x3);
3236     greater_equal(0x2);
3237     less_equal(0x9);
3238     greater(0x8);
3239     overflow(0x0); // unsupported/unimplemented
3240     no_overflow(0x0); // unsupported/unimplemented
3241   %}
3242 %}
3243 
3244 operand cmpOpL() %{
3245   match(Bool);
3246 
3247   format %{ "L" %}
3248   interface(COND_INTER) %{
3249     equal(0x0);
3250     not_equal(0x1);
3251     less(0xb);
3252     greater_equal(0xa);
3253     less_equal(0xd);
3254     greater(0xc);
3255     overflow(0x0); // unsupported/unimplemented
3256     no_overflow(0x0); // unsupported/unimplemented
3257   %}
3258 %}
3259 
3260 operand cmpOpL_commute() %{
3261   match(Bool);
3262 
3263   format %{ "L" %}
3264   interface(COND_INTER) %{
3265     equal(0x0);
3266     not_equal(0x1);
3267     less(0xc);
3268     greater_equal(0xd);
3269     less_equal(0xa);
3270     greater(0xb);
3271     overflow(0x0); // unsupported/unimplemented
3272     no_overflow(0x0); // unsupported/unimplemented
3273   %}
3274 %}
3275 
3276 operand cmpOpUL() %{
3277   match(Bool);
3278 
3279   format %{ "UL" %}
3280   interface(COND_INTER) %{
3281     equal(0x0);
3282     not_equal(0x1);
3283     less(0x3);
3284     greater_equal(0x2);
3285     less_equal(0x9);
3286     greater(0x8);
3287     overflow(0x0); // unsupported/unimplemented
3288     no_overflow(0x0); // unsupported/unimplemented
3289   %}
3290 %}
3291 
3292 operand cmpOpUL_commute() %{
3293   match(Bool);
3294 
3295   format %{ "UL" %}
3296   interface(COND_INTER) %{
3297     equal(0x0);
3298     not_equal(0x1);
3299     less(0x8);
3300     greater_equal(0x9);
3301     less_equal(0x2);
3302     greater(0x3);
3303     overflow(0x0); // unsupported/unimplemented
3304     no_overflow(0x0); // unsupported/unimplemented
3305   %}
3306 %}
3307 
3308 
3309 //----------OPERAND CLASSES----------------------------------------------------
3310 // Operand Classes are groups of operands that are used to simplify
3311 // instruction definitions by not requiring the AD writer to specify separate
3312 // instructions for every form of operand when the instruction accepts
3313 // multiple operand types with the same basic encoding and format.  The classic
3314 // case of this is memory operands.
3315 #ifdef AARCH64
3316 opclass memoryB(indirect, indIndex, indOffsetU12ScaleB);
3317 opclass memoryS(indirect, indIndex, indIndexScaleS, indIndexIScaleS, indOffsetU12ScaleS);
3318 opclass memoryI(indirect, indIndex, indIndexScaleI, indIndexIScaleI, indOffsetU12ScaleI);
3319 opclass memoryL(indirect, indIndex, indIndexScaleL, indIndexIScaleL, indOffsetU12ScaleL);
3320 opclass memoryP(indirect, indIndex, indIndexScaleL, indIndexIScaleL, indOffsetU12ScaleL);
3321 opclass memoryQ(indirect, indIndex, indIndexScaleQ, indIndexIScaleQ, indOffsetU12ScaleQ);
3322 opclass memoryF(indirect, indIndex, indIndexScaleI, indIndexIScaleI, indOffsetU12ScaleI);
3323 opclass memoryD(indirect, indIndex, indIndexScaleL, indIndexIScaleL, indOffsetU12ScaleL);
3324 
3325 opclass memoryScaledS(indIndexScaleS, indIndexIScaleS);
3326 opclass memoryScaledI(indIndexScaleI, indIndexIScaleI);
3327 opclass memoryScaledL(indIndexScaleL, indIndexIScaleL);
3328 opclass memoryScaledP(indIndexScaleL, indIndexIScaleL);
3329 opclass memoryScaledQ(indIndexScaleQ, indIndexIScaleQ);
3330 opclass memoryScaledF(indIndexScaleI, indIndexIScaleI);
3331 opclass memoryScaledD(indIndexScaleL, indIndexIScaleL);
3332 // when ldrex/strex is used:
3333 opclass memoryex ( indirect );
3334 opclass indIndexMemory( indIndex );
3335 opclass memoryvld ( indirect /* , write back mode not implemented */ );
3336 
3337 #else
3338 
3339 opclass memoryI ( indirect, indOffset12, indIndex, indIndexScale );
3340 opclass memoryP ( indirect, indOffset12, indIndex, indIndexScale );
3341 opclass memoryF ( indirect, indOffsetFP );
3342 opclass memoryF2 ( indirect, indOffsetFPx2 );
3343 opclass memoryD ( indirect, indOffsetFP );
3344 opclass memoryfp( indirect, indOffsetFP );
3345 opclass memoryB ( indirect, indIndex, indOffsetHD );
3346 opclass memoryS ( indirect, indIndex, indOffsetHD );
3347 opclass memoryL ( indirect, indIndex, indOffsetHD );
3348 
3349 opclass memoryScaledI(indIndexScale);
3350 opclass memoryScaledP(indIndexScale);
3351 
3352 // when ldrex/strex is used:
3353 opclass memoryex ( indirect );
3354 opclass indIndexMemory( indIndex );
3355 opclass memorylong ( indirect, indOffset12x2 );
3356 opclass memoryvld ( indirect /* , write back mode not implemented */ );
3357 #endif
3358 
3359 //----------PIPELINE-----------------------------------------------------------
3360 pipeline %{
3361 
3362 //----------ATTRIBUTES---------------------------------------------------------
3363 attributes %{
3364   fixed_size_instructions;           // Fixed size instructions
3365   max_instructions_per_bundle = 4;   // Up to 4 instructions per bundle
3366   instruction_unit_size = 4;         // An instruction is 4 bytes long
3367   instruction_fetch_unit_size = 16;  // The processor fetches one line
3368   instruction_fetch_units = 1;       // of 16 bytes
3369 
3370   // List of nop instructions
3371   nops( Nop_A0, Nop_A1, Nop_MS, Nop_FA, Nop_BR );
3372 %}
3373 
3374 //----------RESOURCES----------------------------------------------------------
3375 // Resources are the functional units available to the machine
3376 resources(A0, A1, MS, BR, FA, FM, IDIV, FDIV, IALU = A0 | A1);
3377 
3378 //----------PIPELINE DESCRIPTION-----------------------------------------------
3379 // Pipeline Description specifies the stages in the machine's pipeline
3380 
3381 pipe_desc(A, P, F, B, I, J, S, R, E, C, M, W, X, T, D);
3382 
3383 //----------PIPELINE CLASSES---------------------------------------------------
3384 // Pipeline Classes describe the stages in which input and output are
3385 // referenced by the hardware pipeline.
3386 
3387 // Integer ALU reg-reg operation
3388 pipe_class ialu_reg_reg(iRegI dst, iRegI src1, iRegI src2) %{
3389     single_instruction;
3390     dst   : E(write);
3391     src1  : R(read);
3392     src2  : R(read);
3393     IALU  : R;
3394 %}
3395 
3396 // Integer ALU reg-reg long operation
3397 pipe_class ialu_reg_reg_2(iRegL dst, iRegL src1, iRegL src2) %{
3398     instruction_count(2);
3399     dst   : E(write);
3400     src1  : R(read);
3401     src2  : R(read);
3402     IALU  : R;
3403     IALU  : R;
3404 %}
3405 
3406 // Integer ALU reg-reg long dependent operation
3407 pipe_class ialu_reg_reg_2_dep(iRegL dst, iRegL src1, iRegL src2, flagsReg cr) %{
3408     instruction_count(1); multiple_bundles;
3409     dst   : E(write);
3410     src1  : R(read);
3411     src2  : R(read);
3412     cr    : E(write);
3413     IALU  : R(2);
3414 %}
3415 
3416 // Integer ALU reg-imm operaion
3417 pipe_class ialu_reg_imm(iRegI dst, iRegI src1) %{
3418     single_instruction;
3419     dst   : E(write);
3420     src1  : R(read);
3421     IALU  : R;
3422 %}
3423 
3424 // Integer ALU reg-reg operation with condition code
3425 pipe_class ialu_cc_reg_reg(iRegI dst, iRegI src1, iRegI src2, flagsReg cr) %{
3426     single_instruction;
3427     dst   : E(write);
3428     cr    : E(write);
3429     src1  : R(read);
3430     src2  : R(read);
3431     IALU  : R;
3432 %}
3433 
3434 // Integer ALU zero-reg operation
3435 pipe_class ialu_zero_reg(iRegI dst, immI0 zero, iRegI src2) %{
3436     single_instruction;
3437     dst   : E(write);
3438     src2  : R(read);
3439     IALU  : R;
3440 %}
3441 
3442 // Integer ALU zero-reg operation with condition code only
3443 pipe_class ialu_cconly_zero_reg(flagsReg cr, iRegI src) %{
3444     single_instruction;
3445     cr    : E(write);
3446     src   : R(read);
3447     IALU  : R;
3448 %}
3449 
3450 // Integer ALU reg-reg operation with condition code only
3451 pipe_class ialu_cconly_reg_reg(flagsReg cr, iRegI src1, iRegI src2) %{
3452     single_instruction;
3453     cr    : E(write);
3454     src1  : R(read);
3455     src2  : R(read);
3456     IALU  : R;
3457 %}
3458 
3459 // Integer ALU reg-imm operation with condition code only
3460 pipe_class ialu_cconly_reg_imm(flagsReg cr, iRegI src1) %{
3461     single_instruction;
3462     cr    : E(write);
3463     src1  : R(read);
3464     IALU  : R;
3465 %}
3466 
3467 // Integer ALU reg-reg-zero operation with condition code only
3468 pipe_class ialu_cconly_reg_reg_zero(flagsReg cr, iRegI src1, iRegI src2, immI0 zero) %{
3469     single_instruction;
3470     cr    : E(write);
3471     src1  : R(read);
3472     src2  : R(read);
3473     IALU  : R;
3474 %}
3475 
3476 // Integer ALU reg-imm-zero operation with condition code only
3477 pipe_class ialu_cconly_reg_imm_zero(flagsReg cr, iRegI src1, immI0 zero) %{
3478     single_instruction;
3479     cr    : E(write);
3480     src1  : R(read);
3481     IALU  : R;
3482 %}
3483 
3484 // Integer ALU reg-reg operation with condition code, src1 modified
3485 pipe_class ialu_cc_rwreg_reg(flagsReg cr, iRegI src1, iRegI src2) %{
3486     single_instruction;
3487     cr    : E(write);
3488     src1  : E(write);
3489     src1  : R(read);
3490     src2  : R(read);
3491     IALU  : R;
3492 %}
3493 
3494 pipe_class cmpL_reg(iRegI dst, iRegL src1, iRegL src2, flagsReg cr ) %{
3495     multiple_bundles;
3496     dst   : E(write)+4;
3497     cr    : E(write);
3498     src1  : R(read);
3499     src2  : R(read);
3500     IALU  : R(3);
3501     BR    : R(2);
3502 %}
3503 
3504 // Integer ALU operation
3505 pipe_class ialu_none(iRegI dst) %{
3506     single_instruction;
3507     dst   : E(write);
3508     IALU  : R;
3509 %}
3510 
3511 // Integer ALU reg operation
3512 pipe_class ialu_reg(iRegI dst, iRegI src) %{
3513     single_instruction; may_have_no_code;
3514     dst   : E(write);
3515     src   : R(read);
3516     IALU  : R;
3517 %}
3518 
3519 // Integer ALU reg conditional operation
3520 // This instruction has a 1 cycle stall, and cannot execute
3521 // in the same cycle as the instruction setting the condition
3522 // code. We kludge this by pretending to read the condition code
3523 // 1 cycle earlier, and by marking the functional units as busy
3524 // for 2 cycles with the result available 1 cycle later than
3525 // is really the case.
3526 pipe_class ialu_reg_flags( iRegI op2_out, iRegI op2_in, iRegI op1, flagsReg cr ) %{
3527     single_instruction;
3528     op2_out : C(write);
3529     op1     : R(read);
3530     cr      : R(read);       // This is really E, with a 1 cycle stall
3531     BR      : R(2);
3532     MS      : R(2);
3533 %}
3534 
3535 // Integer ALU reg operation
3536 pipe_class ialu_move_reg_L_to_I(iRegI dst, iRegL src) %{
3537     single_instruction; may_have_no_code;
3538     dst   : E(write);
3539     src   : R(read);
3540     IALU  : R;
3541 %}
3542 pipe_class ialu_move_reg_I_to_L(iRegL dst, iRegI src) %{
3543     single_instruction; may_have_no_code;
3544     dst   : E(write);
3545     src   : R(read);
3546     IALU  : R;
3547 %}
3548 
3549 // Two integer ALU reg operations
3550 pipe_class ialu_reg_2(iRegL dst, iRegL src) %{
3551     instruction_count(2);
3552     dst   : E(write);
3553     src   : R(read);
3554     A0    : R;
3555     A1    : R;
3556 %}
3557 
3558 // Two integer ALU reg operations
3559 pipe_class ialu_move_reg_L_to_L(iRegL dst, iRegL src) %{
3560     instruction_count(2); may_have_no_code;
3561     dst   : E(write);
3562     src   : R(read);
3563     A0    : R;
3564     A1    : R;
3565 %}
3566 
3567 // Integer ALU imm operation
3568 pipe_class ialu_imm(iRegI dst) %{
3569     single_instruction;
3570     dst   : E(write);
3571     IALU  : R;
3572 %}
3573 
3574 pipe_class ialu_imm_n(iRegI dst) %{
3575     single_instruction;
3576     dst   : E(write);
3577     IALU  : R;
3578 %}
3579 
3580 // Integer ALU reg-reg with carry operation
3581 pipe_class ialu_reg_reg_cy(iRegI dst, iRegI src1, iRegI src2, iRegI cy) %{
3582     single_instruction;
3583     dst   : E(write);
3584     src1  : R(read);
3585     src2  : R(read);
3586     IALU  : R;
3587 %}
3588 
3589 // Integer ALU cc operation
3590 pipe_class ialu_cc(iRegI dst, flagsReg cc) %{
3591     single_instruction;
3592     dst   : E(write);
3593     cc    : R(read);
3594     IALU  : R;
3595 %}
3596 
3597 // Integer ALU cc / second IALU operation
3598 pipe_class ialu_reg_ialu( iRegI dst, iRegI src ) %{
3599     instruction_count(1); multiple_bundles;
3600     dst   : E(write)+1;
3601     src   : R(read);
3602     IALU  : R;
3603 %}
3604 
3605 // Integer ALU cc / second IALU operation
3606 pipe_class ialu_reg_reg_ialu( iRegI dst, iRegI p, iRegI q ) %{
3607     instruction_count(1); multiple_bundles;
3608     dst   : E(write)+1;
3609     p     : R(read);
3610     q     : R(read);
3611     IALU  : R;
3612 %}
3613 
3614 // Integer ALU hi-lo-reg operation
3615 pipe_class ialu_hi_lo_reg(iRegI dst, immI src) %{
3616     instruction_count(1); multiple_bundles;
3617     dst   : E(write)+1;
3618     IALU  : R(2);
3619 %}
3620 
3621 // Long Constant
3622 pipe_class loadConL( iRegL dst, immL src ) %{
3623     instruction_count(2); multiple_bundles;
3624     dst   : E(write)+1;
3625     IALU  : R(2);
3626     IALU  : R(2);
3627 %}
3628 
3629 // Pointer Constant
3630 pipe_class loadConP( iRegP dst, immP src ) %{
3631     instruction_count(0); multiple_bundles;
3632     fixed_latency(6);
3633 %}
3634 
3635 // Polling Address
3636 pipe_class loadConP_poll( iRegP dst, immP_poll src ) %{
3637     dst   : E(write);
3638     IALU  : R;
3639 %}
3640 
3641 // Long Constant small
3642 pipe_class loadConLlo( iRegL dst, immL src ) %{
3643     instruction_count(2);
3644     dst   : E(write);
3645     IALU  : R;
3646     IALU  : R;
3647 %}
3648 
3649 // [PHH] This is wrong for 64-bit.  See LdImmF/D.
3650 pipe_class loadConFD(regF dst, immF src, iRegP tmp) %{
3651     instruction_count(1); multiple_bundles;
3652     src   : R(read);
3653     dst   : M(write)+1;
3654     IALU  : R;
3655     MS    : E;
3656 %}
3657 
3658 // Integer ALU nop operation
3659 pipe_class ialu_nop() %{
3660     single_instruction;
3661     IALU  : R;
3662 %}
3663 
3664 // Integer ALU nop operation
3665 pipe_class ialu_nop_A0() %{
3666     single_instruction;
3667     A0    : R;
3668 %}
3669 
3670 // Integer ALU nop operation
3671 pipe_class ialu_nop_A1() %{
3672     single_instruction;
3673     A1    : R;
3674 %}
3675 
3676 // Integer Multiply reg-reg operation
3677 pipe_class imul_reg_reg(iRegI dst, iRegI src1, iRegI src2) %{
3678     single_instruction;
3679     dst   : E(write);
3680     src1  : R(read);
3681     src2  : R(read);
3682     MS    : R(5);
3683 %}
3684 
3685 pipe_class mulL_reg_reg(iRegL dst, iRegL src1, iRegL src2) %{
3686     single_instruction;
3687     dst   : E(write)+4;
3688     src1  : R(read);
3689     src2  : R(read);
3690     MS    : R(6);
3691 %}
3692 
3693 // Integer Divide reg-reg
3694 pipe_class sdiv_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI temp, flagsReg cr) %{
3695     instruction_count(1); multiple_bundles;
3696     dst   : E(write);
3697     temp  : E(write);
3698     src1  : R(read);
3699     src2  : R(read);
3700     temp  : R(read);
3701     MS    : R(38);
3702 %}
3703 
3704 // Long Divide
3705 pipe_class divL_reg_reg(iRegL dst, iRegL src1, iRegL src2) %{
3706     dst  : E(write)+71;
3707     src1 : R(read);
3708     src2 : R(read)+1;
3709     MS   : R(70);
3710 %}
3711 
3712 // Floating Point Add Float
3713 pipe_class faddF_reg_reg(regF dst, regF src1, regF src2) %{
3714     single_instruction;
3715     dst   : X(write);
3716     src1  : E(read);
3717     src2  : E(read);
3718     FA    : R;
3719 %}
3720 
3721 // Floating Point Add Double
3722 pipe_class faddD_reg_reg(regD dst, regD src1, regD src2) %{
3723     single_instruction;
3724     dst   : X(write);
3725     src1  : E(read);
3726     src2  : E(read);
3727     FA    : R;
3728 %}
3729 
3730 // Floating Point Conditional Move based on integer flags
3731 pipe_class int_conditional_float_move (cmpOp cmp, flagsReg cr, regF dst, regF src) %{
3732     single_instruction;
3733     dst   : X(write);
3734     src   : E(read);
3735     cr    : R(read);
3736     FA    : R(2);
3737     BR    : R(2);
3738 %}
3739 
3740 // Floating Point Conditional Move based on integer flags
3741 pipe_class int_conditional_double_move (cmpOp cmp, flagsReg cr, regD dst, regD src) %{
3742     single_instruction;
3743     dst   : X(write);
3744     src   : E(read);
3745     cr    : R(read);
3746     FA    : R(2);
3747     BR    : R(2);
3748 %}
3749 
3750 // Floating Point Multiply Float
3751 pipe_class fmulF_reg_reg(regF dst, regF src1, regF src2) %{
3752     single_instruction;
3753     dst   : X(write);
3754     src1  : E(read);
3755     src2  : E(read);
3756     FM    : R;
3757 %}
3758 
3759 // Floating Point Multiply Double
3760 pipe_class fmulD_reg_reg(regD dst, regD src1, regD src2) %{
3761     single_instruction;
3762     dst   : X(write);
3763     src1  : E(read);
3764     src2  : E(read);
3765     FM    : R;
3766 %}
3767 
3768 // Floating Point Divide Float
3769 pipe_class fdivF_reg_reg(regF dst, regF src1, regF src2) %{
3770     single_instruction;
3771     dst   : X(write);
3772     src1  : E(read);
3773     src2  : E(read);
3774     FM    : R;
3775     FDIV  : C(14);
3776 %}
3777 
3778 // Floating Point Divide Double
3779 pipe_class fdivD_reg_reg(regD dst, regD src1, regD src2) %{
3780     single_instruction;
3781     dst   : X(write);
3782     src1  : E(read);
3783     src2  : E(read);
3784     FM    : R;
3785     FDIV  : C(17);
3786 %}
3787 
3788 // Floating Point Move/Negate/Abs Float
3789 pipe_class faddF_reg(regF dst, regF src) %{
3790     single_instruction;
3791     dst   : W(write);
3792     src   : E(read);
3793     FA    : R(1);
3794 %}
3795 
3796 // Floating Point Move/Negate/Abs Double
3797 pipe_class faddD_reg(regD dst, regD src) %{
3798     single_instruction;
3799     dst   : W(write);
3800     src   : E(read);
3801     FA    : R;
3802 %}
3803 
3804 // Floating Point Convert F->D
3805 pipe_class fcvtF2D(regD dst, regF src) %{
3806     single_instruction;
3807     dst   : X(write);
3808     src   : E(read);
3809     FA    : R;
3810 %}
3811 
3812 // Floating Point Convert I->D
3813 pipe_class fcvtI2D(regD dst, regF src) %{
3814     single_instruction;
3815     dst   : X(write);
3816     src   : E(read);
3817     FA    : R;
3818 %}
3819 
3820 // Floating Point Convert LHi->D
3821 pipe_class fcvtLHi2D(regD dst, regD src) %{
3822     single_instruction;
3823     dst   : X(write);
3824     src   : E(read);
3825     FA    : R;
3826 %}
3827 
3828 // Floating Point Convert L->D
3829 pipe_class fcvtL2D(regD dst, iRegL src) %{
3830     single_instruction;
3831     dst   : X(write);
3832     src   : E(read);
3833     FA    : R;
3834 %}
3835 
3836 // Floating Point Convert L->F
3837 pipe_class fcvtL2F(regF dst, iRegL src) %{
3838     single_instruction;
3839     dst   : X(write);
3840     src   : E(read);
3841     FA    : R;
3842 %}
3843 
3844 // Floating Point Convert D->F
3845 pipe_class fcvtD2F(regD dst, regF src) %{
3846     single_instruction;
3847     dst   : X(write);
3848     src   : E(read);
3849     FA    : R;
3850 %}
3851 
3852 // Floating Point Convert I->L
3853 pipe_class fcvtI2L(regD dst, regF src) %{
3854     single_instruction;
3855     dst   : X(write);
3856     src   : E(read);
3857     FA    : R;
3858 %}
3859 
3860 // Floating Point Convert D->F
3861 pipe_class fcvtD2I(iRegI dst, regD src, flagsReg cr) %{
3862     instruction_count(1); multiple_bundles;
3863     dst   : X(write)+6;
3864     src   : E(read);
3865     FA    : R;
3866 %}
3867 
3868 // Floating Point Convert D->L
3869 pipe_class fcvtD2L(regD dst, regD src, flagsReg cr) %{
3870     instruction_count(1); multiple_bundles;
3871     dst   : X(write)+6;
3872     src   : E(read);
3873     FA    : R;
3874 %}
3875 
3876 // Floating Point Convert F->I
3877 pipe_class fcvtF2I(regF dst, regF src, flagsReg cr) %{
3878     instruction_count(1); multiple_bundles;
3879     dst   : X(write)+6;
3880     src   : E(read);
3881     FA    : R;
3882 %}
3883 
3884 // Floating Point Convert F->L
3885 pipe_class fcvtF2L(regD dst, regF src, flagsReg cr) %{
3886     instruction_count(1); multiple_bundles;
3887     dst   : X(write)+6;
3888     src   : E(read);
3889     FA    : R;
3890 %}
3891 
3892 // Floating Point Convert I->F
3893 pipe_class fcvtI2F(regF dst, regF src) %{
3894     single_instruction;
3895     dst   : X(write);
3896     src   : E(read);
3897     FA    : R;
3898 %}
3899 
3900 // Floating Point Compare
3901 pipe_class faddF_fcc_reg_reg_zero(flagsRegF cr, regF src1, regF src2, immI0 zero) %{
3902     single_instruction;
3903     cr    : X(write);
3904     src1  : E(read);
3905     src2  : E(read);
3906     FA    : R;
3907 %}
3908 
3909 // Floating Point Compare
3910 pipe_class faddD_fcc_reg_reg_zero(flagsRegF cr, regD src1, regD src2, immI0 zero) %{
3911     single_instruction;
3912     cr    : X(write);
3913     src1  : E(read);
3914     src2  : E(read);
3915     FA    : R;
3916 %}
3917 
3918 // Floating Add Nop
3919 pipe_class fadd_nop() %{
3920     single_instruction;
3921     FA  : R;
3922 %}
3923 
3924 // Integer Store to Memory
3925 pipe_class istore_mem_reg(memoryI mem, iRegI src) %{
3926     single_instruction;
3927     mem   : R(read);
3928     src   : C(read);
3929     MS    : R;
3930 %}
3931 
3932 // Integer Store to Memory
3933 pipe_class istore_mem_spORreg(memoryI mem, sp_ptr_RegP src) %{
3934     single_instruction;
3935     mem   : R(read);
3936     src   : C(read);
3937     MS    : R;
3938 %}
3939 
3940 // Float Store
3941 pipe_class fstoreF_mem_reg(memoryF mem, RegF src) %{
3942     single_instruction;
3943     mem : R(read);
3944     src : C(read);
3945     MS  : R;
3946 %}
3947 
3948 // Float Store
3949 pipe_class fstoreF_mem_zero(memoryF mem, immF0 src) %{
3950     single_instruction;
3951     mem : R(read);
3952     MS  : R;
3953 %}
3954 
3955 // Double Store
3956 pipe_class fstoreD_mem_reg(memoryD mem, RegD src) %{
3957     instruction_count(1);
3958     mem : R(read);
3959     src : C(read);
3960     MS  : R;
3961 %}
3962 
3963 // Double Store
3964 pipe_class fstoreD_mem_zero(memoryD mem, immD0 src) %{
3965     single_instruction;
3966     mem : R(read);
3967     MS  : R;
3968 %}
3969 
3970 // Integer Load (when sign bit propagation not needed)
3971 pipe_class iload_mem(iRegI dst, memoryI mem) %{
3972     single_instruction;
3973     mem : R(read);
3974     dst : C(write);
3975     MS  : R;
3976 %}
3977 
3978 // Integer Load (when sign bit propagation or masking is needed)
3979 pipe_class iload_mask_mem(iRegI dst, memoryI mem) %{
3980     single_instruction;
3981     mem : R(read);
3982     dst : M(write);
3983     MS  : R;
3984 %}
3985 
3986 // Float Load
3987 pipe_class floadF_mem(regF dst, memoryF mem) %{
3988     single_instruction;
3989     mem : R(read);
3990     dst : M(write);
3991     MS  : R;
3992 %}
3993 
3994 // Float Load
3995 pipe_class floadD_mem(regD dst, memoryD mem) %{
3996     instruction_count(1); multiple_bundles; // Again, unaligned argument is only multiple case
3997     mem : R(read);
3998     dst : M(write);
3999     MS  : R;
4000 %}
4001 
4002 // Memory Nop
4003 pipe_class mem_nop() %{
4004     single_instruction;
4005     MS  : R;
4006 %}
4007 
4008 pipe_class sethi(iRegP dst, immI src) %{
4009     single_instruction;
4010     dst  : E(write);
4011     IALU : R;
4012 %}
4013 
4014 pipe_class loadPollP(iRegP poll) %{
4015     single_instruction;
4016     poll : R(read);
4017     MS   : R;
4018 %}
4019 
4020 pipe_class br(Universe br, label labl) %{
4021     single_instruction_with_delay_slot;
4022     BR  : R;
4023 %}
4024 
4025 pipe_class br_cc(Universe br, cmpOp cmp, flagsReg cr, label labl) %{
4026     single_instruction_with_delay_slot;
4027     cr    : E(read);
4028     BR    : R;
4029 %}
4030 
4031 pipe_class br_reg(Universe br, cmpOp cmp, iRegI op1, label labl) %{
4032     single_instruction_with_delay_slot;
4033     op1 : E(read);
4034     BR  : R;
4035     MS  : R;
4036 %}
4037 
4038 pipe_class br_nop() %{
4039     single_instruction;
4040     BR  : R;
4041 %}
4042 
4043 pipe_class simple_call(method meth) %{
4044     instruction_count(2); multiple_bundles; force_serialization;
4045     fixed_latency(100);
4046     BR  : R(1);
4047     MS  : R(1);
4048     A0  : R(1);
4049 %}
4050 
4051 pipe_class compiled_call(method meth) %{
4052     instruction_count(1); multiple_bundles; force_serialization;
4053     fixed_latency(100);
4054     MS  : R(1);
4055 %}
4056 
4057 pipe_class call(method meth) %{
4058     instruction_count(0); multiple_bundles; force_serialization;
4059     fixed_latency(100);
4060 %}
4061 
4062 pipe_class tail_call(Universe ignore, label labl) %{
4063     single_instruction; has_delay_slot;
4064     fixed_latency(100);
4065     BR  : R(1);
4066     MS  : R(1);
4067 %}
4068 
4069 pipe_class ret(Universe ignore) %{
4070     single_instruction; has_delay_slot;
4071     BR  : R(1);
4072     MS  : R(1);
4073 %}
4074 
4075 // The real do-nothing guy
4076 pipe_class empty( ) %{
4077     instruction_count(0);
4078 %}
4079 
4080 pipe_class long_memory_op() %{
4081     instruction_count(0); multiple_bundles; force_serialization;
4082     fixed_latency(25);
4083     MS  : R(1);
4084 %}
4085 
4086 // Check-cast
4087 pipe_class partial_subtype_check_pipe(Universe ignore, iRegP array, iRegP match ) %{
4088     array : R(read);
4089     match  : R(read);
4090     IALU   : R(2);
4091     BR     : R(2);
4092     MS     : R;
4093 %}
4094 
4095 // Convert FPU flags into +1,0,-1
4096 pipe_class floating_cmp( iRegI dst, regF src1, regF src2 ) %{
4097     src1  : E(read);
4098     src2  : E(read);
4099     dst   : E(write);
4100     FA    : R;
4101     MS    : R(2);
4102     BR    : R(2);
4103 %}
4104 
4105 // Compare for p < q, and conditionally add y
4106 pipe_class cadd_cmpltmask( iRegI p, iRegI q, iRegI y ) %{
4107     p     : E(read);
4108     q     : E(read);
4109     y     : E(read);
4110     IALU  : R(3)
4111 %}
4112 
4113 // Perform a compare, then move conditionally in a branch delay slot.
4114 pipe_class min_max( iRegI src2, iRegI srcdst ) %{
4115     src2   : E(read);
4116     srcdst : E(read);
4117     IALU   : R;
4118     BR     : R;
4119 %}
4120 
4121 // Define the class for the Nop node
4122 define %{
4123    MachNop = ialu_nop;
4124 %}
4125 
4126 %}
4127 
4128 //----------INSTRUCTIONS-------------------------------------------------------
4129 
4130 //------------Special Nop instructions for bundling - no match rules-----------
4131 // Nop using the A0 functional unit
4132 instruct Nop_A0() %{
4133   ins_pipe(ialu_nop_A0);
4134 %}
4135 
4136 // Nop using the A1 functional unit
4137 instruct Nop_A1( ) %{
4138   ins_pipe(ialu_nop_A1);
4139 %}
4140 
4141 // Nop using the memory functional unit
4142 instruct Nop_MS( ) %{
4143   ins_pipe(mem_nop);
4144 %}
4145 
4146 // Nop using the floating add functional unit
4147 instruct Nop_FA( ) %{
4148   ins_pipe(fadd_nop);
4149 %}
4150 
4151 // Nop using the branch functional unit
4152 instruct Nop_BR( ) %{
4153   ins_pipe(br_nop);
4154 %}
4155 
4156 //----------Load/Store/Move Instructions---------------------------------------
4157 //----------Load Instructions--------------------------------------------------
4158 // Load Byte (8bit signed)
4159 instruct loadB(iRegI dst, memoryB mem) %{
4160   match(Set dst (LoadB mem));
4161   ins_cost(MEMORY_REF_COST);
4162 
4163   size(4);
4164   format %{ "LDRSB   $dst,$mem\t! byte -> int" %}
4165   ins_encode %{
4166     // High 32 bits are harmlessly set on Aarch64
4167     __ ldrsb($dst$$Register, $mem$$Address);
4168   %}
4169   ins_pipe(iload_mask_mem);
4170 %}
4171 
4172 // Load Byte (8bit signed) into a Long Register
4173 instruct loadB2L(iRegL dst, memoryB mem) %{
4174   match(Set dst (ConvI2L (LoadB mem)));
4175   ins_cost(MEMORY_REF_COST);
4176 
4177 #ifdef AARCH64
4178   size(4);
4179   format %{ "LDRSB $dst,$mem\t! byte -> long"  %}
4180   ins_encode %{
4181     __ ldrsb($dst$$Register, $mem$$Address);
4182   %}
4183 #else
4184   size(8);
4185   format %{ "LDRSB $dst.lo,$mem\t! byte -> long\n\t"
4186             "ASR   $dst.hi,$dst.lo,31" %}
4187   ins_encode %{
4188     __ ldrsb($dst$$Register, $mem$$Address);
4189     __ mov($dst$$Register->successor(), AsmOperand($dst$$Register, asr, 31));
4190   %}
4191 #endif
4192   ins_pipe(iload_mask_mem);
4193 %}
4194 
4195 // Load Unsigned Byte (8bit UNsigned) into an int reg
4196 instruct loadUB(iRegI dst, memoryB mem) %{
4197   match(Set dst (LoadUB mem));
4198   ins_cost(MEMORY_REF_COST);
4199 
4200   size(4);
4201   format %{ "LDRB   $dst,$mem\t! ubyte -> int" %}
4202   ins_encode %{
4203     __ ldrb($dst$$Register, $mem$$Address);
4204   %}
4205   ins_pipe(iload_mem);
4206 %}
4207 
4208 // Load Unsigned Byte (8bit UNsigned) into a Long Register
4209 instruct loadUB2L(iRegL dst, memoryB mem) %{
4210   match(Set dst (ConvI2L (LoadUB mem)));
4211   ins_cost(MEMORY_REF_COST);
4212 
4213 #ifdef AARCH64
4214   size(4);
4215   format %{ "LDRB  $dst,$mem\t! ubyte -> long"  %}
4216   ins_encode %{
4217     __ ldrb($dst$$Register, $mem$$Address);
4218   %}
4219 #else
4220   size(8);
4221   format %{ "LDRB  $dst.lo,$mem\t! ubyte -> long\n\t"
4222             "MOV   $dst.hi,0" %}
4223   ins_encode %{
4224     __ ldrb($dst$$Register, $mem$$Address);
4225     __ mov($dst$$Register->successor(), 0);
4226   %}
4227 #endif
4228   ins_pipe(iload_mem);
4229 %}
4230 
4231 // Load Unsigned Byte (8 bit UNsigned) with immediate mask into Long Register
4232 instruct loadUB2L_limmI(iRegL dst, memoryB mem, limmIlow8 mask) %{
4233   match(Set dst (ConvI2L (AndI (LoadUB mem) mask)));
4234 
4235 #ifdef AARCH64
4236   ins_cost(MEMORY_REF_COST + DEFAULT_COST);
4237   size(8);
4238   format %{ "LDRB  $dst,$mem\t! ubyte -> long\n\t"
4239             "AND  $dst,$dst,$mask" %}
4240   ins_encode %{
4241     __ ldrb($dst$$Register, $mem$$Address);
4242     __ andr($dst$$Register, $dst$$Register, limmI_low($mask$$constant, 8));
4243   %}
4244 #else
4245   ins_cost(MEMORY_REF_COST + 2*DEFAULT_COST);
4246   size(12);
4247   format %{ "LDRB  $dst.lo,$mem\t! ubyte -> long\n\t"
4248             "MOV   $dst.hi,0\n\t"
4249             "AND  $dst.lo,$dst.lo,$mask" %}
4250   ins_encode %{
4251     __ ldrb($dst$$Register, $mem$$Address);
4252     __ mov($dst$$Register->successor(), 0);
4253     __ andr($dst$$Register, $dst$$Register, limmI_low($mask$$constant, 8));
4254   %}
4255 #endif
4256   ins_pipe(iload_mem);
4257 %}
4258 
4259 // Load Short (16bit signed)
4260 #ifdef AARCH64
4261 // XXX This variant shouldn't be necessary if 6217251 is implemented
4262 instruct loadSoff(iRegI dst, memoryScaledS mem, aimmX off, iRegP tmp) %{
4263   match(Set dst (LoadS (AddP mem off)));
4264   ins_cost(MEMORY_REF_COST + DEFAULT_COST); // assume shift/sign-extend is free
4265   effect(TEMP tmp);
4266   size(4 * 2);
4267 
4268   format %{ "LDRSH   $dst,$mem+$off\t! short temp=$tmp" %}
4269   ins_encode %{
4270     Register base = reg_to_register_object($mem$$base);
4271     __ add($tmp$$Register, base, $off$$constant);
4272     Address nmem = Address::make_raw($tmp$$reg, $mem$$index, $mem$$scale, $mem$$disp, relocInfo::none);
4273     __ ldrsh($dst$$Register, nmem);
4274   %}
4275   ins_pipe(iload_mask_mem);
4276 %}
4277 #endif
4278 
4279 instruct loadS(iRegI dst, memoryS mem) %{
4280   match(Set dst (LoadS mem));
4281   ins_cost(MEMORY_REF_COST);
4282 
4283   size(4);
4284   format %{ "LDRSH   $dst,$mem\t! short" %}
4285   ins_encode %{
4286     __ ldrsh($dst$$Register, $mem$$Address);
4287   %}
4288   ins_pipe(iload_mask_mem);
4289 %}
4290 
4291 // Load Short (16 bit signed) to Byte (8 bit signed)
4292 instruct loadS2B(iRegI dst, memoryS mem, immI_24 twentyfour) %{
4293   match(Set dst (RShiftI (LShiftI (LoadS mem) twentyfour) twentyfour));
4294   ins_cost(MEMORY_REF_COST);
4295 
4296   size(4);
4297 
4298   format %{ "LDRSB   $dst,$mem\t! short -> byte" %}
4299   ins_encode %{
4300     // High 32 bits are harmlessly set on Aarch64
4301     __ ldrsb($dst$$Register, $mem$$Address);
4302   %}
4303   ins_pipe(iload_mask_mem);
4304 %}
4305 
4306 // Load Short (16bit signed) into a Long Register
4307 instruct loadS2L(iRegL dst, memoryS mem) %{
4308   match(Set dst (ConvI2L (LoadS mem)));
4309   ins_cost(MEMORY_REF_COST);
4310 
4311 #ifdef AARCH64
4312   size(4);
4313   format %{ "LDRSH $dst,$mem\t! short -> long"  %}
4314   ins_encode %{
4315     __ ldrsh($dst$$Register, $mem$$Address);
4316   %}
4317 #else
4318   size(8);
4319   format %{ "LDRSH $dst.lo,$mem\t! short -> long\n\t"
4320             "ASR   $dst.hi,$dst.lo,31" %}
4321   ins_encode %{
4322     __ ldrsh($dst$$Register, $mem$$Address);
4323     __ mov($dst$$Register->successor(), AsmOperand($dst$$Register, asr, 31));
4324   %}
4325 #endif
4326   ins_pipe(iload_mask_mem);
4327 %}
4328 
4329 // Load Unsigned Short/Char (16bit UNsigned)
4330 
4331 #ifdef AARCH64
4332 // XXX This variant shouldn't be necessary if 6217251 is implemented
4333 instruct loadUSoff(iRegI dst, memoryScaledS mem, aimmX off, iRegP tmp) %{
4334   match(Set dst (LoadUS (AddP mem off)));
4335   ins_cost(MEMORY_REF_COST + DEFAULT_COST); // assume shift/sign-extend is free
4336   effect(TEMP tmp);
4337   size(4 * 2);
4338 
4339   format %{ "LDRH   $dst,$mem+$off\t! ushort/char temp=$tmp" %}
4340   ins_encode %{
4341     Register base = reg_to_register_object($mem$$base);
4342     __ add($tmp$$Register, base, $off$$constant);
4343     Address nmem = Address::make_raw($tmp$$reg, $mem$$index, $mem$$scale, $mem$$disp, relocInfo::none);
4344     __ ldrh($dst$$Register, nmem);
4345   %}
4346   ins_pipe(iload_mem);
4347 %}
4348 #endif
4349 
4350 instruct loadUS(iRegI dst, memoryS mem) %{
4351   match(Set dst (LoadUS mem));
4352   ins_cost(MEMORY_REF_COST);
4353 
4354   size(4);
4355   format %{ "LDRH   $dst,$mem\t! ushort/char" %}
4356   ins_encode %{
4357     __ ldrh($dst$$Register, $mem$$Address);
4358   %}
4359   ins_pipe(iload_mem);
4360 %}
4361 
4362 // Load Unsigned Short/Char (16 bit UNsigned) to Byte (8 bit signed)
4363 instruct loadUS2B(iRegI dst, memoryB mem, immI_24 twentyfour) %{
4364   match(Set dst (RShiftI (LShiftI (LoadUS mem) twentyfour) twentyfour));
4365   ins_cost(MEMORY_REF_COST);
4366 
4367   size(4);
4368   format %{ "LDRSB   $dst,$mem\t! ushort -> byte" %}
4369   ins_encode %{
4370     __ ldrsb($dst$$Register, $mem$$Address);
4371   %}
4372   ins_pipe(iload_mask_mem);
4373 %}
4374 
4375 // Load Unsigned Short/Char (16bit UNsigned) into a Long Register
4376 instruct loadUS2L(iRegL dst, memoryS mem) %{
4377   match(Set dst (ConvI2L (LoadUS mem)));
4378   ins_cost(MEMORY_REF_COST);
4379 
4380 #ifdef AARCH64
4381   size(4);
4382   format %{ "LDRH  $dst,$mem\t! short -> long"  %}
4383   ins_encode %{
4384     __ ldrh($dst$$Register, $mem$$Address);
4385   %}
4386 #else
4387   size(8);
4388   format %{ "LDRH  $dst.lo,$mem\t! short -> long\n\t"
4389             "MOV   $dst.hi, 0" %}
4390   ins_encode %{
4391     __ ldrh($dst$$Register, $mem$$Address);
4392     __ mov($dst$$Register->successor(), 0);
4393   %}
4394 #endif
4395   ins_pipe(iload_mem);
4396 %}
4397 
4398 // Load Unsigned Short/Char (16bit UNsigned) with mask 0xFF into a Long Register
4399 instruct loadUS2L_immI_255(iRegL dst, memoryB mem, immI_255 mask) %{
4400   match(Set dst (ConvI2L (AndI (LoadUS mem) mask)));
4401   ins_cost(MEMORY_REF_COST);
4402 
4403 #ifdef AARCH64
4404   size(4);
4405   format %{ "LDRB  $dst,$mem"  %}
4406   ins_encode %{
4407     __ ldrb($dst$$Register, $mem$$Address);
4408   %}
4409 #else
4410   size(8);
4411   format %{ "LDRB  $dst.lo,$mem\t! \n\t"
4412             "MOV   $dst.hi, 0" %}
4413   ins_encode %{
4414     __ ldrb($dst$$Register, $mem$$Address);
4415     __ mov($dst$$Register->successor(), 0);
4416   %}
4417 #endif
4418   ins_pipe(iload_mem);
4419 %}
4420 
4421 // Load Unsigned Short/Char (16bit UNsigned) with a immediate mask into a Long Register
4422 instruct loadUS2L_limmI(iRegL dst, memoryS mem, limmI mask) %{
4423   match(Set dst (ConvI2L (AndI (LoadUS mem) mask)));
4424 #ifdef AARCH64
4425   ins_cost(MEMORY_REF_COST + 1*DEFAULT_COST);
4426 
4427   size(8);
4428   format %{ "LDRH   $dst,$mem\t! ushort/char & mask -> long\n\t"
4429             "AND    $dst,$dst,$mask" %}
4430   ins_encode %{
4431     __ ldrh($dst$$Register, $mem$$Address);
4432     __ andr($dst$$Register, $dst$$Register, (uintx)$mask$$constant);
4433   %}
4434 #else
4435   ins_cost(MEMORY_REF_COST + 2*DEFAULT_COST);
4436 
4437   size(12);
4438   format %{ "LDRH   $dst,$mem\t! ushort/char & mask -> long\n\t"
4439             "MOV    $dst.hi, 0\n\t"
4440             "AND    $dst,$dst,$mask" %}
4441   ins_encode %{
4442     __ ldrh($dst$$Register, $mem$$Address);
4443     __ mov($dst$$Register->successor(), 0);
4444     __ andr($dst$$Register, $dst$$Register, $mask$$constant);
4445   %}
4446 #endif
4447   ins_pipe(iload_mem);
4448 %}
4449 
4450 // Load Integer
4451 
4452 #ifdef AARCH64
4453 // XXX This variant shouldn't be necessary if 6217251 is implemented
4454 instruct loadIoff(iRegI dst, memoryScaledI mem, aimmX off, iRegP tmp) %{
4455   match(Set dst (LoadI (AddP mem off)));
4456   ins_cost(MEMORY_REF_COST + DEFAULT_COST); // assume shift/sign-extend is free
4457   effect(TEMP tmp);
4458   size(4 * 2);
4459 
4460   format %{ "ldr_s32 $dst,$mem+$off\t! int temp=$tmp" %}
4461   ins_encode %{
4462     Register base = reg_to_register_object($mem$$base);
4463     __ add($tmp$$Register, base, $off$$constant);
4464     Address nmem = Address::make_raw($tmp$$reg, $mem$$index, $mem$$scale, $mem$$disp, relocInfo::none);
4465     __ ldr_s32($dst$$Register, nmem);
4466   %}
4467   ins_pipe(iload_mem);
4468 %}
4469 #endif
4470 
4471 instruct loadI(iRegI dst, memoryI mem) %{
4472   match(Set dst (LoadI mem));
4473   ins_cost(MEMORY_REF_COST);
4474 
4475   size(4);
4476   format %{ "ldr_s32 $dst,$mem\t! int" %}
4477   ins_encode %{
4478     __ ldr_s32($dst$$Register, $mem$$Address);
4479   %}
4480   ins_pipe(iload_mem);
4481 %}
4482 
4483 // Load Integer to Byte (8 bit signed)
4484 instruct loadI2B(iRegI dst, memoryS mem, immI_24 twentyfour) %{
4485   match(Set dst (RShiftI (LShiftI (LoadI mem) twentyfour) twentyfour));
4486   ins_cost(MEMORY_REF_COST);
4487 
4488   size(4);
4489 
4490   format %{ "LDRSB   $dst,$mem\t! int -> byte" %}
4491   ins_encode %{
4492     __ ldrsb($dst$$Register, $mem$$Address);
4493   %}
4494   ins_pipe(iload_mask_mem);
4495 %}
4496 
4497 // Load Integer to Unsigned Byte (8 bit UNsigned)
4498 instruct loadI2UB(iRegI dst, memoryB mem, immI_255 mask) %{
4499   match(Set dst (AndI (LoadI mem) mask));
4500   ins_cost(MEMORY_REF_COST);
4501 
4502   size(4);
4503 
4504   format %{ "LDRB   $dst,$mem\t! int -> ubyte" %}
4505   ins_encode %{
4506     __ ldrb($dst$$Register, $mem$$Address);
4507   %}
4508   ins_pipe(iload_mask_mem);
4509 %}
4510 
4511 // Load Integer to Short (16 bit signed)
4512 instruct loadI2S(iRegI dst, memoryS mem, immI_16 sixteen) %{
4513   match(Set dst (RShiftI (LShiftI (LoadI mem) sixteen) sixteen));
4514   ins_cost(MEMORY_REF_COST);
4515 
4516   size(4);
4517   format %{ "LDRSH   $dst,$mem\t! int -> short" %}
4518   ins_encode %{
4519     __ ldrsh($dst$$Register, $mem$$Address);
4520   %}
4521   ins_pipe(iload_mask_mem);
4522 %}
4523 
4524 // Load Integer to Unsigned Short (16 bit UNsigned)
4525 instruct loadI2US(iRegI dst, memoryS mem, immI_65535 mask) %{
4526   match(Set dst (AndI (LoadI mem) mask));
4527   ins_cost(MEMORY_REF_COST);
4528 
4529   size(4);
4530   format %{ "LDRH   $dst,$mem\t! int -> ushort/char" %}
4531   ins_encode %{
4532     __ ldrh($dst$$Register, $mem$$Address);
4533   %}
4534   ins_pipe(iload_mask_mem);
4535 %}
4536 
4537 // Load Integer into a Long Register
4538 instruct loadI2L(iRegL dst, memoryI mem) %{
4539   match(Set dst (ConvI2L (LoadI mem)));
4540 #ifdef AARCH64
4541   ins_cost(MEMORY_REF_COST);
4542 
4543   size(4);
4544   format %{ "LDRSW $dst.lo,$mem\t! int -> long"  %}
4545   ins_encode %{
4546     __ ldr_s32($dst$$Register, $mem$$Address);
4547   %}
4548 #else
4549   ins_cost(MEMORY_REF_COST);
4550 
4551   size(8);
4552   format %{ "LDR   $dst.lo,$mem\t! int -> long\n\t"
4553             "ASR   $dst.hi,$dst.lo,31\t! int->long" %}
4554   ins_encode %{
4555     __ ldr($dst$$Register, $mem$$Address);
4556     __ mov($dst$$Register->successor(), AsmOperand($dst$$Register, asr, 31));
4557   %}
4558 #endif
4559   ins_pipe(iload_mask_mem);
4560 %}
4561 
4562 // Load Integer with mask 0xFF into a Long Register
4563 instruct loadI2L_immI_255(iRegL dst, memoryB mem, immI_255 mask) %{
4564   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
4565 #ifdef AARCH64
4566   ins_cost(MEMORY_REF_COST);
4567 
4568   size(4);
4569   format %{ "LDRB   $dst.lo,$mem\t! int & 0xFF -> long"  %}
4570   ins_encode %{
4571     __ ldrb($dst$$Register, $mem$$Address);
4572   %}
4573 #else
4574   ins_cost(MEMORY_REF_COST);
4575 
4576   size(8);
4577   format %{ "LDRB   $dst.lo,$mem\t! int & 0xFF -> long\n\t"
4578             "MOV    $dst.hi, 0" %}
4579   ins_encode %{
4580     __ ldrb($dst$$Register, $mem$$Address);
4581     __ mov($dst$$Register->successor(), 0);
4582   %}
4583 #endif
4584   ins_pipe(iload_mem);
4585 %}
4586 
4587 // Load Integer with mask 0xFFFF into a Long Register
4588 instruct loadI2L_immI_65535(iRegL dst, memoryS mem, immI_65535 mask) %{
4589   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
4590   ins_cost(MEMORY_REF_COST);
4591 
4592 #ifdef AARCH64
4593   size(4);
4594   format %{ "LDRH   $dst,$mem\t! int & 0xFFFF -> long" %}
4595   ins_encode %{
4596     __ ldrh($dst$$Register, $mem$$Address);
4597   %}
4598 #else
4599   size(8);
4600   format %{ "LDRH   $dst,$mem\t! int & 0xFFFF -> long\n\t"
4601             "MOV    $dst.hi, 0" %}
4602   ins_encode %{
4603     __ ldrh($dst$$Register, $mem$$Address);
4604     __ mov($dst$$Register->successor(), 0);
4605   %}
4606 #endif
4607   ins_pipe(iload_mask_mem);
4608 %}
4609 
4610 #ifdef AARCH64
4611 // Load Integer with an immediate mask into a Long Register
4612 instruct loadI2L_limmI(iRegL dst, memoryI mem, limmI mask) %{
4613   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
4614   ins_cost(MEMORY_REF_COST + 1*DEFAULT_COST);
4615 
4616   size(8);
4617   format %{ "LDRSW $dst,$mem\t! int -> long\n\t"
4618             "AND   $dst,$dst,$mask" %}
4619 
4620   ins_encode %{
4621     __ ldr_s32($dst$$Register, $mem$$Address);
4622     __ andr($dst$$Register, $dst$$Register, (uintx)$mask$$constant);
4623   %}
4624   ins_pipe(iload_mem);
4625 %}
4626 #else
4627 // Load Integer with a 31-bit immediate mask into a Long Register
4628 instruct loadI2L_limmU31(iRegL dst, memoryI mem, limmU31 mask) %{
4629   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
4630   ins_cost(MEMORY_REF_COST + 2*DEFAULT_COST);
4631 
4632   size(12);
4633   format %{ "LDR   $dst.lo,$mem\t! int -> long\n\t"
4634             "MOV    $dst.hi, 0\n\t"
4635             "AND   $dst,$dst,$mask" %}
4636 
4637   ins_encode %{
4638     __ ldr($dst$$Register, $mem$$Address);
4639     __ mov($dst$$Register->successor(), 0);
4640     __ andr($dst$$Register, $dst$$Register, $mask$$constant);
4641   %}
4642   ins_pipe(iload_mem);
4643 %}
4644 #endif
4645 
4646 #ifdef AARCH64
4647 // Load Integer with mask into a Long Register
4648 // FIXME: use signedRegI mask, remove tmp?
4649 instruct loadI2L_immI(iRegL dst, memoryI mem, immI mask, iRegI tmp) %{
4650   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
4651   effect(TEMP dst, TEMP tmp);
4652 
4653   ins_cost(MEMORY_REF_COST + 3*DEFAULT_COST);
4654   format %{ "LDRSW    $mem,$dst\t! int & 31-bit mask -> long\n\t"
4655             "MOV_SLOW $tmp,$mask\n\t"
4656             "AND      $dst,$tmp,$dst" %}
4657   ins_encode %{
4658     __ ldrsw($dst$$Register, $mem$$Address);
4659     __ mov_slow($tmp$$Register, $mask$$constant);
4660     __ andr($dst$$Register, $dst$$Register, $tmp$$Register);
4661   %}
4662   ins_pipe(iload_mem);
4663 %}
4664 #else
4665 // Load Integer with a 31-bit mask into a Long Register
4666 // FIXME: use iRegI mask, remove tmp?
4667 instruct loadI2L_immU31(iRegL dst, memoryI mem, immU31 mask, iRegI tmp) %{
4668   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
4669   effect(TEMP dst, TEMP tmp);
4670 
4671   ins_cost(MEMORY_REF_COST + 4*DEFAULT_COST);
4672   size(20);
4673   format %{ "LDR      $mem,$dst\t! int & 31-bit mask -> long\n\t"
4674             "MOV      $dst.hi, 0\n\t"
4675             "MOV_SLOW $tmp,$mask\n\t"
4676             "AND      $dst,$tmp,$dst" %}
4677   ins_encode %{
4678     __ ldr($dst$$Register, $mem$$Address);
4679     __ mov($dst$$Register->successor(), 0);
4680     __ mov_slow($tmp$$Register, $mask$$constant);
4681     __ andr($dst$$Register, $dst$$Register, $tmp$$Register);
4682   %}
4683   ins_pipe(iload_mem);
4684 %}
4685 #endif
4686 
4687 // Load Unsigned Integer into a Long Register
4688 instruct loadUI2L(iRegL dst, memoryI mem, immL_32bits mask) %{
4689   match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
4690   ins_cost(MEMORY_REF_COST);
4691 
4692 #ifdef AARCH64
4693 //size(4);
4694   format %{ "LDR_w $dst,$mem\t! uint -> long" %}
4695   ins_encode %{
4696     __ ldr_w($dst$$Register, $mem$$Address);
4697   %}
4698 #else
4699   size(8);
4700   format %{ "LDR   $dst.lo,$mem\t! uint -> long\n\t"
4701             "MOV   $dst.hi,0" %}
4702   ins_encode %{
4703     __ ldr($dst$$Register, $mem$$Address);
4704     __ mov($dst$$Register->successor(), 0);
4705   %}
4706 #endif
4707   ins_pipe(iload_mem);
4708 %}
4709 
4710 // Load Long
4711 
4712 #ifdef AARCH64
4713 // XXX This variant shouldn't be necessary if 6217251 is implemented
4714 instruct loadLoff(iRegLd dst, memoryScaledL mem, aimmX off, iRegP tmp) %{
4715   match(Set dst (LoadL (AddP mem off)));
4716   ins_cost(MEMORY_REF_COST + DEFAULT_COST); // assume shift/sign-extend is free
4717   effect(TEMP tmp);
4718   size(4 * 2);
4719 
4720   format %{ "LDR    $dst,$mem+$off\t! long temp=$tmp" %}
4721   ins_encode %{
4722     Register base = reg_to_register_object($mem$$base);
4723     __ add($tmp$$Register, base, $off$$constant);
4724     Address nmem = Address::make_raw($tmp$$reg, $mem$$index, $mem$$scale, $mem$$disp, relocInfo::none);
4725     __ ldr($dst$$Register, nmem);
4726   %}
4727   ins_pipe(iload_mem);
4728 %}
4729 #endif
4730 
4731 instruct loadL(iRegLd dst, memoryL mem ) %{
4732 #ifdef AARCH64
4733   // already atomic for Aarch64
4734 #else
4735   predicate(!((LoadLNode*)n)->require_atomic_access());
4736 #endif
4737   match(Set dst (LoadL mem));
4738   effect(TEMP dst);
4739   ins_cost(MEMORY_REF_COST);
4740 
4741   size(4);
4742   format %{ "ldr_64  $dst,$mem\t! long" %}
4743   ins_encode %{
4744     __ ldr_64($dst$$Register, $mem$$Address);
4745   %}
4746   ins_pipe(iload_mem);
4747 %}
4748 
4749 #ifndef AARCH64
4750 instruct loadL_2instr(iRegL dst, memorylong mem ) %{
4751   predicate(!((LoadLNode*)n)->require_atomic_access());
4752   match(Set dst (LoadL mem));
4753   ins_cost(MEMORY_REF_COST + DEFAULT_COST);
4754 
4755   size(8);
4756   format %{ "LDR    $dst.lo,$mem \t! long order of instrs reversed if $dst.lo == base($mem)\n\t"
4757             "LDR    $dst.hi,$mem+4 or $mem" %}
4758   ins_encode %{
4759     Address Amemlo = Address::make_raw($mem$$base, $mem$$index, $mem$$scale, $mem$$disp, relocInfo::none);
4760     Address Amemhi = Address::make_raw($mem$$base, $mem$$index, $mem$$scale, $mem$$disp + 4, relocInfo::none);
4761 
4762     if ($dst$$Register == reg_to_register_object($mem$$base)) {
4763       __ ldr($dst$$Register->successor(), Amemhi);
4764       __ ldr($dst$$Register, Amemlo);
4765     } else {
4766       __ ldr($dst$$Register, Amemlo);
4767       __ ldr($dst$$Register->successor(), Amemhi);
4768     }
4769   %}
4770   ins_pipe(iload_mem);
4771 %}
4772 
4773 instruct loadL_volatile(iRegL dst, indirect mem ) %{
4774   predicate(((LoadLNode*)n)->require_atomic_access());
4775   match(Set dst (LoadL mem));
4776   ins_cost(MEMORY_REF_COST);
4777 
4778   size(4);
4779   format %{ "LDMIA    $dst,$mem\t! long" %}
4780   ins_encode %{
4781     // FIXME: why is ldmia considered atomic?  Should be ldrexd
4782     RegisterSet set($dst$$Register);
4783     set = set | reg_to_register_object($dst$$reg + 1);
4784     __ ldmia(reg_to_register_object($mem$$base), set);
4785   %}
4786   ins_pipe(iload_mem);
4787 %}
4788 
4789 instruct loadL_volatile_fp(iRegL dst, memoryD mem ) %{
4790   predicate(((LoadLNode*)n)->require_atomic_access());
4791   match(Set dst (LoadL mem));
4792   ins_cost(MEMORY_REF_COST);
4793 
4794   size(8);
4795   format %{ "FLDD      S14, $mem"
4796             "FMRRD    $dst, S14\t! long \n't" %}
4797   ins_encode %{
4798     __ fldd(S14, $mem$$Address);
4799     __ fmrrd($dst$$Register, $dst$$Register->successor(), S14);
4800   %}
4801   ins_pipe(iload_mem);
4802 %}
4803 
4804 instruct loadL_unaligned(iRegL dst, memorylong mem ) %{
4805   match(Set dst (LoadL_unaligned mem));
4806   ins_cost(MEMORY_REF_COST);
4807 
4808   size(8);
4809   format %{ "LDR    $dst.lo,$mem\t! long order of instrs reversed if $dst.lo == base($mem)\n\t"
4810             "LDR    $dst.hi,$mem+4" %}
4811   ins_encode %{
4812     Address Amemlo = Address::make_raw($mem$$base, $mem$$index, $mem$$scale, $mem$$disp, relocInfo::none);
4813     Address Amemhi = Address::make_raw($mem$$base, $mem$$index, $mem$$scale, $mem$$disp + 4, relocInfo::none);
4814 
4815     if ($dst$$Register == reg_to_register_object($mem$$base)) {
4816       __ ldr($dst$$Register->successor(), Amemhi);
4817       __ ldr($dst$$Register, Amemlo);
4818     } else {
4819       __ ldr($dst$$Register, Amemlo);
4820       __ ldr($dst$$Register->successor(), Amemhi);
4821     }
4822   %}
4823   ins_pipe(iload_mem);
4824 %}
4825 #endif // !AARCH64
4826 
4827 // Load Range
4828 instruct loadRange(iRegI dst, memoryI mem) %{
4829   match(Set dst (LoadRange mem));
4830   ins_cost(MEMORY_REF_COST);
4831 
4832   size(4);
4833   format %{ "LDR_u32 $dst,$mem\t! range" %}
4834   ins_encode %{
4835     __ ldr_u32($dst$$Register, $mem$$Address);
4836   %}
4837   ins_pipe(iload_mem);
4838 %}
4839 
4840 // Load Pointer
4841 
4842 #ifdef AARCH64
4843 // XXX This variant shouldn't be necessary if 6217251 is implemented
4844 instruct loadPoff(iRegP dst, memoryScaledP mem, aimmX off, iRegP tmp) %{
4845   match(Set dst (LoadP (AddP mem off)));
4846   ins_cost(MEMORY_REF_COST + DEFAULT_COST); // assume shift/sign-extend is free
4847   effect(TEMP tmp);
4848   size(4 * 2);
4849 
4850   format %{ "LDR    $dst,$mem+$off\t! ptr temp=$tmp" %}
4851   ins_encode %{
4852     Register base = reg_to_register_object($mem$$base);
4853     __ add($tmp$$Register, base, $off$$constant);
4854     Address nmem = Address::make_raw($tmp$$reg, $mem$$index, $mem$$scale, $mem$$disp, relocInfo::none);
4855     __ ldr($dst$$Register, nmem);
4856   %}
4857   ins_pipe(iload_mem);
4858 %}
4859 #endif
4860 
4861 instruct loadP(iRegP dst, memoryP mem) %{
4862   match(Set dst (LoadP mem));
4863   ins_cost(MEMORY_REF_COST);
4864   size(4);
4865 
4866   format %{ "LDR   $dst,$mem\t! ptr" %}
4867   ins_encode %{
4868     __ ldr($dst$$Register, $mem$$Address);
4869   %}
4870   ins_pipe(iload_mem);
4871 %}
4872 
4873 #ifdef XXX
4874 // FIXME XXXX
4875 //instruct loadSP(iRegP dst, memoryP mem) %{
4876 instruct loadSP(SPRegP dst, memoryP mem, iRegP tmp) %{
4877   match(Set dst (LoadP mem));
4878   effect(TEMP tmp);
4879   ins_cost(MEMORY_REF_COST+1);
4880   size(8);
4881 
4882   format %{ "LDR   $tmp,$mem\t! ptr\n\t"
4883             "MOV   $dst,$tmp\t! ptr" %}
4884   ins_encode %{
4885     __ ldr($tmp$$Register, $mem$$Address);
4886     __ mov($dst$$Register, $tmp$$Register);
4887   %}
4888   ins_pipe(iload_mem);
4889 %}
4890 #endif
4891 
4892 #ifdef _LP64
4893 // Load Compressed Pointer
4894 
4895 // XXX This variant shouldn't be necessary if 6217251 is implemented
4896 instruct loadNoff(iRegN dst, memoryScaledI mem, aimmX off, iRegP tmp) %{
4897   match(Set dst (LoadN (AddP mem off)));
4898   ins_cost(MEMORY_REF_COST + DEFAULT_COST); // assume shift/sign-extend is free
4899   effect(TEMP tmp);
4900   size(4 * 2);
4901 
4902   format %{ "ldr_u32 $dst,$mem+$off\t! compressed ptr temp=$tmp" %}
4903   ins_encode %{
4904     Register base = reg_to_register_object($mem$$base);
4905     __ add($tmp$$Register, base, $off$$constant);
4906     Address nmem = Address::make_raw($tmp$$reg, $mem$$index, $mem$$scale, $mem$$disp, relocInfo::none);
4907     __ ldr_u32($dst$$Register, nmem);
4908   %}
4909   ins_pipe(iload_mem);
4910 %}
4911 
4912 instruct loadN(iRegN dst, memoryI mem) %{
4913   match(Set dst (LoadN mem));
4914   ins_cost(MEMORY_REF_COST);
4915   size(4);
4916 
4917   format %{ "ldr_u32 $dst,$mem\t! compressed ptr" %}
4918   ins_encode %{
4919     __ ldr_u32($dst$$Register, $mem$$Address);
4920   %}
4921   ins_pipe(iload_mem);
4922 %}
4923 #endif
4924 
4925 // Load Klass Pointer
4926 instruct loadKlass(iRegP dst, memoryI mem) %{
4927   match(Set dst (LoadKlass mem));
4928   ins_cost(MEMORY_REF_COST);
4929   size(4);
4930 
4931   format %{ "LDR   $dst,$mem\t! klass ptr" %}
4932   ins_encode %{
4933     __ ldr($dst$$Register, $mem$$Address);
4934   %}
4935   ins_pipe(iload_mem);
4936 %}
4937 
4938 #ifdef _LP64
4939 // Load narrow Klass Pointer
4940 instruct loadNKlass(iRegN dst, memoryI mem) %{
4941   match(Set dst (LoadNKlass mem));
4942   ins_cost(MEMORY_REF_COST);
4943   size(4);
4944 
4945   format %{ "ldr_u32 $dst,$mem\t! compressed klass ptr" %}
4946   ins_encode %{
4947     __ ldr_u32($dst$$Register, $mem$$Address);
4948   %}
4949   ins_pipe(iload_mem);
4950 %}
4951 #endif
4952 
4953 #ifdef AARCH64
4954 // XXX This variant shouldn't be necessary if 6217251 is implemented
4955 instruct loadDoff(regD dst, memoryScaledD mem, aimmX off, iRegP tmp) %{
4956   match(Set dst (LoadD (AddP mem off)));
4957   ins_cost(MEMORY_REF_COST + DEFAULT_COST); // assume shift/sign-extend is free
4958   effect(TEMP tmp);
4959   size(4 * 2);
4960 
4961   format %{ "ldr    $dst,$mem+$off\t! double temp=$tmp" %}
4962   ins_encode %{
4963     Register base = reg_to_register_object($mem$$base);
4964     __ add($tmp$$Register, base, $off$$constant);
4965     Address nmem = Address::make_raw($tmp$$reg, $mem$$index, $mem$$scale, $mem$$disp, relocInfo::none);
4966     __ ldr_d($dst$$FloatRegister, nmem);
4967   %}
4968   ins_pipe(floadD_mem);
4969 %}
4970 #endif
4971 
4972 instruct loadD(regD dst, memoryD mem) %{
4973   match(Set dst (LoadD mem));
4974   ins_cost(MEMORY_REF_COST);
4975 
4976   size(4);
4977   // FIXME: needs to be atomic, but  ARMv7 A.R.M. guarantees
4978   // only LDREXD and STREXD are 64-bit single-copy atomic
4979   format %{ "FLDD   $dst,$mem" %}
4980   ins_encode %{
4981     __ ldr_double($dst$$FloatRegister, $mem$$Address);
4982   %}
4983   ins_pipe(floadD_mem);
4984 %}
4985 
4986 #ifndef AARCH64
4987 // Load Double - UNaligned
4988 instruct loadD_unaligned(regD_low dst, memoryF2 mem ) %{
4989   match(Set dst (LoadD_unaligned mem));
4990   ins_cost(MEMORY_REF_COST*2+DEFAULT_COST);
4991   size(8);
4992   format %{ "FLDS    $dst.lo,$mem\t! misaligned double\n"
4993           "\tFLDS    $dst.hi,$mem+4\t!" %}
4994   ins_encode %{
4995     Address Amemlo = Address::make_raw($mem$$base, $mem$$index, $mem$$scale, $mem$$disp, relocInfo::none);
4996     Address Amemhi = Address::make_raw($mem$$base, $mem$$index, $mem$$scale, $mem$$disp + 4, relocInfo::none);
4997       __ flds($dst$$FloatRegister, Amemlo);
4998       __ flds($dst$$FloatRegister->successor(), Amemhi);
4999   %}
5000   ins_pipe(iload_mem);
5001 %}
5002 #endif
5003 
5004 #ifdef AARCH64
5005 // XXX This variant shouldn't be necessary if 6217251 is implemented
5006 instruct loadFoff(regF dst, memoryScaledF mem, aimmX off, iRegP tmp) %{
5007   match(Set dst (LoadF (AddP mem off)));
5008   ins_cost(MEMORY_REF_COST + DEFAULT_COST); // assume shift/sign-extend is free
5009   effect(TEMP tmp);
5010   size(4 * 2);
5011 
5012   format %{ "ldr    $dst,$mem+$off\t! float temp=$tmp" %}
5013   ins_encode %{
5014     Register base = reg_to_register_object($mem$$base);
5015     __ add($tmp$$Register, base, $off$$constant);
5016     Address nmem = Address::make_raw($tmp$$reg, $mem$$index, $mem$$scale, $mem$$disp, relocInfo::none);
5017     __ ldr_s($dst$$FloatRegister, nmem);
5018   %}
5019   ins_pipe(floadF_mem);
5020 %}
5021 #endif
5022 
5023 instruct loadF(regF dst, memoryF mem) %{
5024   match(Set dst (LoadF mem));
5025 
5026   ins_cost(MEMORY_REF_COST);
5027   size(4);
5028   format %{ "FLDS    $dst,$mem" %}
5029   ins_encode %{
5030     __ ldr_float($dst$$FloatRegister, $mem$$Address);
5031   %}
5032   ins_pipe(floadF_mem);
5033 %}
5034 
5035 #ifdef AARCH64
5036 instruct load_limmI(iRegI dst, limmI src) %{
5037   match(Set dst src);
5038   ins_cost(DEFAULT_COST + 1); // + 1 because MOV is preferred
5039   format %{ "ORR_w  $dst, ZR, $src\t! int"  %}
5040   ins_encode %{
5041     __ orr_w($dst$$Register, ZR, (uintx)$src$$constant);
5042   %}
5043   ins_pipe(ialu_imm);
5044 %}
5045 #endif
5046 
5047 // // Load Constant
5048 instruct loadConI( iRegI dst, immI src ) %{
5049   match(Set dst src);
5050   ins_cost(DEFAULT_COST * 3/2);
5051   format %{ "MOV_SLOW    $dst, $src" %}
5052   ins_encode %{
5053     __ mov_slow($dst$$Register, $src$$constant);
5054   %}
5055   ins_pipe(ialu_hi_lo_reg);
5056 %}
5057 
5058 instruct loadConIMov( iRegI dst, immIMov src ) %{
5059   match(Set dst src);
5060   size(4);
5061   format %{ "MOV    $dst, $src" %}
5062   ins_encode %{
5063     __ mov($dst$$Register, $src$$constant);
5064   %}
5065   ins_pipe(ialu_imm);
5066 %}
5067 
5068 #ifndef AARCH64
5069 instruct loadConIMovn( iRegI dst, immIRotn src ) %{
5070   match(Set dst src);
5071   size(4);
5072   format %{ "MVN    $dst, ~$src" %}
5073   ins_encode %{
5074     __ mvn($dst$$Register, ~$src$$constant);
5075   %}
5076   ins_pipe(ialu_imm_n);
5077 %}
5078 #endif
5079 
5080 instruct loadConI16( iRegI dst, immI16 src ) %{
5081   match(Set dst src);
5082   size(4);
5083 #ifdef AARCH64
5084   format %{ "MOVZ_w  $dst, $src" %}
5085 #else
5086   format %{ "MOVW    $dst, $src" %}
5087 #endif
5088   ins_encode %{
5089 #ifdef AARCH64
5090     __ mov_w($dst$$Register, $src$$constant);
5091 #else
5092     __ movw($dst$$Register, $src$$constant);
5093 #endif
5094   %}
5095   ins_pipe(ialu_imm_n);
5096 %}
5097 
5098 instruct loadConP(iRegP dst, immP src) %{
5099   match(Set dst src);
5100   ins_cost(DEFAULT_COST * 3/2);
5101   format %{ "MOV_SLOW    $dst,$src\t!ptr" %}
5102   ins_encode %{
5103     relocInfo::relocType constant_reloc = _opnds[1]->constant_reloc();
5104     intptr_t val = $src$$constant;
5105     if (constant_reloc == relocInfo::oop_type) {
5106       __ mov_oop($dst$$Register, (jobject)val);
5107     } else if (constant_reloc == relocInfo::metadata_type) {
5108       __ mov_metadata($dst$$Register, (Metadata*)val);
5109     } else {
5110       __ mov_slow($dst$$Register, val);
5111     }
5112   %}
5113   ins_pipe(loadConP);
5114 %}
5115 
5116 
5117 instruct loadConP_poll(iRegP dst, immP_poll src) %{
5118   match(Set dst src);
5119   ins_cost(DEFAULT_COST);
5120   format %{ "MOV_SLOW    $dst,$src\t!ptr" %}
5121   ins_encode %{
5122       __ mov_slow($dst$$Register, $src$$constant);
5123   %}
5124   ins_pipe(loadConP_poll);
5125 %}
5126 
5127 #ifdef AARCH64
5128 instruct loadConP0(iRegP dst, immP0 src) %{
5129   match(Set dst src);
5130   ins_cost(DEFAULT_COST);
5131   format %{ "MOV    $dst,ZR\t!ptr" %}
5132   ins_encode %{
5133     __ mov($dst$$Register, ZR);
5134   %}
5135   ins_pipe(ialu_none);
5136 %}
5137 
5138 instruct loadConN(iRegN dst, immN src) %{
5139   match(Set dst src);
5140   ins_cost(DEFAULT_COST * 3/2);
5141   format %{ "SET    $dst,$src\t! compressed ptr" %}
5142   ins_encode %{
5143     Register dst = $dst$$Register;
5144     // FIXME: use $constanttablebase?
5145     __ set_narrow_oop(dst, (jobject)$src$$constant);
5146   %}
5147   ins_pipe(ialu_hi_lo_reg);
5148 %}
5149 
5150 instruct loadConN0(iRegN dst, immN0 src) %{
5151   match(Set dst src);
5152   ins_cost(DEFAULT_COST);
5153   format %{ "MOV    $dst,ZR\t! compressed ptr" %}
5154   ins_encode %{
5155     __ mov($dst$$Register, ZR);
5156   %}
5157   ins_pipe(ialu_none);
5158 %}
5159 
5160 instruct loadConNKlass(iRegN dst, immNKlass src) %{
5161   match(Set dst src);
5162   ins_cost(DEFAULT_COST * 3/2);
5163   format %{ "SET    $dst,$src\t! compressed klass ptr" %}
5164   ins_encode %{
5165     Register dst = $dst$$Register;
5166     // FIXME: use $constanttablebase?
5167     __ set_narrow_klass(dst, (Klass*)$src$$constant);
5168   %}
5169   ins_pipe(ialu_hi_lo_reg);
5170 %}
5171 
5172 instruct load_limmL(iRegL dst, limmL src) %{
5173   match(Set dst src);
5174   ins_cost(DEFAULT_COST);
5175   format %{ "ORR    $dst, ZR, $src\t! long"  %}
5176   ins_encode %{
5177     __ orr($dst$$Register, ZR, (uintx)$src$$constant);
5178   %}
5179   ins_pipe(loadConL);
5180 %}
5181 instruct load_immLMov(iRegL dst, immLMov src) %{
5182   match(Set dst src);
5183   ins_cost(DEFAULT_COST);
5184   format %{ "MOV    $dst, $src\t! long"  %}
5185   ins_encode %{
5186     __ mov($dst$$Register, $src$$constant);
5187   %}
5188   ins_pipe(loadConL);
5189 %}
5190 instruct loadConL(iRegL dst, immL src) %{
5191   match(Set dst src);
5192   ins_cost(DEFAULT_COST * 4); // worst case
5193   format %{ "mov_slow   $dst, $src\t! long"  %}
5194   ins_encode %{
5195     // FIXME: use $constanttablebase?
5196     __ mov_slow($dst$$Register, $src$$constant);
5197   %}
5198   ins_pipe(loadConL);
5199 %}
5200 #else
5201 instruct loadConL(iRegL dst, immL src) %{
5202   match(Set dst src);
5203   ins_cost(DEFAULT_COST * 4);
5204   format %{ "MOV_SLOW   $dst.lo, $src & 0x0FFFFFFFFL \t! long\n\t"
5205             "MOV_SLOW   $dst.hi, $src >> 32" %}
5206   ins_encode %{
5207     __ mov_slow(reg_to_register_object($dst$$reg), $src$$constant & 0x0FFFFFFFFL);
5208     __ mov_slow(reg_to_register_object($dst$$reg + 1), ((julong)($src$$constant)) >> 32);
5209   %}
5210   ins_pipe(loadConL);
5211 %}
5212 
5213 instruct loadConL16( iRegL dst, immL16 src ) %{
5214   match(Set dst src);
5215   ins_cost(DEFAULT_COST * 2);
5216 
5217   size(8);
5218   format %{ "MOVW    $dst.lo, $src \n\t"
5219             "MOVW    $dst.hi, 0 \n\t" %}
5220   ins_encode %{
5221     __ movw($dst$$Register, $src$$constant);
5222     __ movw($dst$$Register->successor(), 0);
5223   %}
5224   ins_pipe(ialu_imm);
5225 %}
5226 #endif
5227 
5228 instruct loadConF_imm8(regF dst, imm8F src) %{
5229   match(Set dst src);
5230   ins_cost(DEFAULT_COST);
5231   size(4);
5232 
5233   format %{ "FCONSTS      $dst, $src"%}
5234 
5235   ins_encode %{
5236     __ fconsts($dst$$FloatRegister, Assembler::float_num($src$$constant).imm8());
5237   %}
5238   ins_pipe(loadConFD); // FIXME
5239 %}
5240 
5241 #ifdef AARCH64
5242 instruct loadIConF(iRegI dst, immF src) %{
5243   match(Set dst src);
5244   ins_cost(DEFAULT_COST * 2);
5245 
5246   format %{ "MOV_SLOW  $dst, $src\t! loadIConF"  %}
5247 
5248   ins_encode %{
5249     // FIXME revisit once 6961697 is in
5250     union {
5251       jfloat f;
5252       int i;
5253     } v;
5254     v.f = $src$$constant;
5255     __ mov_slow($dst$$Register, v.i);
5256   %}
5257   ins_pipe(ialu_imm);
5258 %}
5259 #endif
5260 
5261 instruct loadConF(regF dst, immF src, iRegI tmp) %{
5262   match(Set dst src);
5263   ins_cost(DEFAULT_COST * 2);
5264   effect(TEMP tmp);
5265   size(3*4);
5266 
5267   format %{ "MOV_SLOW  $tmp, $src\n\t"
5268             "FMSR      $dst, $tmp"%}
5269 
5270   ins_encode %{
5271     // FIXME revisit once 6961697 is in
5272     union {
5273       jfloat f;
5274       int i;
5275     } v;
5276     v.f = $src$$constant;
5277     __ mov_slow($tmp$$Register, v.i);
5278     __ fmsr($dst$$FloatRegister, $tmp$$Register);
5279   %}
5280   ins_pipe(loadConFD); // FIXME
5281 %}
5282 
5283 instruct loadConD_imm8(regD dst, imm8D src) %{
5284   match(Set dst src);
5285   ins_cost(DEFAULT_COST);
5286   size(4);
5287 
5288   format %{ "FCONSTD      $dst, $src"%}
5289 
5290   ins_encode %{
5291     __ fconstd($dst$$FloatRegister, Assembler::double_num($src$$constant).imm8());
5292   %}
5293   ins_pipe(loadConFD); // FIXME
5294 %}
5295 
5296 instruct loadConD(regD dst, immD src, iRegP tmp) %{
5297   match(Set dst src);
5298   effect(TEMP tmp);
5299   ins_cost(MEMORY_REF_COST);
5300   format %{ "FLDD  $dst, [$constanttablebase + $constantoffset]\t! load from constant table: double=$src" %}
5301 
5302   ins_encode %{
5303     Register r = $constanttablebase;
5304     int offset  = $constantoffset($src);
5305     if (!is_memoryD(offset)) {                // can't use a predicate
5306                                               // in load constant instructs
5307       __ add_slow($tmp$$Register, r, offset);
5308       r = $tmp$$Register;
5309       offset = 0;
5310     }
5311     __ ldr_double($dst$$FloatRegister, Address(r, offset));
5312   %}
5313   ins_pipe(loadConFD);
5314 %}
5315 
5316 // Prefetch instructions.
5317 // Must be safe to execute with invalid address (cannot fault).
5318 
5319 instruct prefetchAlloc( memoryP mem ) %{
5320   predicate(os::is_MP());
5321   match( PrefetchAllocation mem );
5322   ins_cost(MEMORY_REF_COST);
5323   size(4);
5324 
5325   format %{ "PLDW $mem\t! Prefetch allocation" %}
5326   ins_encode %{
5327 #ifdef AARCH64
5328     __ prfm(pstl1keep, $mem$$Address);
5329 #else
5330     __ pldw($mem$$Address);
5331 #endif
5332   %}
5333   ins_pipe(iload_mem);
5334 %}
5335 
5336 //----------Store Instructions-------------------------------------------------
5337 // Store Byte
5338 instruct storeB(memoryB mem, store_RegI src) %{
5339   match(Set mem (StoreB mem src));
5340   ins_cost(MEMORY_REF_COST);
5341 
5342   size(4);
5343   format %{ "STRB    $src,$mem\t! byte" %}
5344   ins_encode %{
5345     __ strb($src$$Register, $mem$$Address);
5346   %}
5347   ins_pipe(istore_mem_reg);
5348 %}
5349 
5350 instruct storeCM(memoryB mem, store_RegI src) %{
5351   match(Set mem (StoreCM mem src));
5352   ins_cost(MEMORY_REF_COST);
5353 
5354   size(4);
5355   format %{ "STRB    $src,$mem\t! CMS card-mark byte" %}
5356   ins_encode %{
5357     __ strb($src$$Register, $mem$$Address);
5358   %}
5359   ins_pipe(istore_mem_reg);
5360 %}
5361 
5362 // Store Char/Short
5363 
5364 #ifdef AARCH64
5365 // XXX This variant shouldn't be necessary if 6217251 is implemented
5366 instruct storeCoff(store_RegI src, memoryScaledS mem, aimmX off, iRegP tmp) %{
5367   match(Set mem (StoreC (AddP mem off) src));
5368   ins_cost(MEMORY_REF_COST + DEFAULT_COST); // assume shift/sign-extend is free
5369   effect(TEMP tmp);
5370   size(4 * 2);
5371 
5372   format %{ "STRH    $src,$mem+$off\t! short temp=$tmp" %}
5373   ins_encode %{
5374     Register base = reg_to_register_object($mem$$base);
5375     __ add($tmp$$Register, base, $off$$constant);
5376     Address nmem = Address::make_raw($tmp$$reg, $mem$$index, $mem$$scale, $mem$$disp, relocInfo::none);
5377     __ strh($src$$Register, nmem);
5378   %}
5379   ins_pipe(istore_mem_reg);
5380 %}
5381 #endif
5382 
5383 instruct storeC(memoryS mem, store_RegI src) %{
5384   match(Set mem (StoreC mem src));
5385   ins_cost(MEMORY_REF_COST);
5386 
5387   size(4);
5388   format %{ "STRH    $src,$mem\t! short" %}
5389   ins_encode %{
5390     __ strh($src$$Register, $mem$$Address);
5391   %}
5392   ins_pipe(istore_mem_reg);
5393 %}
5394 
5395 // Store Integer
5396 
5397 #ifdef AARCH64
5398 // XXX This variant shouldn't be necessary if 6217251 is implemented
5399 instruct storeIoff(store_RegI src, memoryScaledI mem, aimmX off, iRegP tmp) %{
5400   match(Set mem (StoreI (AddP mem off) src));
5401   ins_cost(MEMORY_REF_COST + DEFAULT_COST); // assume shift/sign-extend is free
5402   effect(TEMP tmp);
5403   size(4 * 2);
5404 
5405   format %{ "str_32 $src,$mem+$off\t! int temp=$tmp" %}
5406   ins_encode %{
5407     Register base = reg_to_register_object($mem$$base);
5408     __ add($tmp$$Register, base, $off$$constant);
5409     Address nmem = Address::make_raw($tmp$$reg, $mem$$index, $mem$$scale, $mem$$disp, relocInfo::none);
5410     __ str_32($src$$Register, nmem);
5411   %}
5412   ins_pipe(istore_mem_reg);
5413 %}
5414 #endif
5415 
5416 instruct storeI(memoryI mem, store_RegI src) %{
5417   match(Set mem (StoreI mem src));
5418   ins_cost(MEMORY_REF_COST);
5419 
5420   size(4);
5421   format %{ "str_32 $src,$mem" %}
5422   ins_encode %{
5423     __ str_32($src$$Register, $mem$$Address);
5424   %}
5425   ins_pipe(istore_mem_reg);
5426 %}
5427 
5428 // Store Long
5429 
5430 #ifdef AARCH64
5431 // XXX This variant shouldn't be necessary if 6217251 is implemented
5432 instruct storeLoff(store_RegLd src, memoryScaledL mem, aimmX off, iRegP tmp) %{
5433   match(Set mem (StoreL (AddP mem off) src));
5434   ins_cost(MEMORY_REF_COST + DEFAULT_COST); // assume shift/sign-extend is free
5435   effect(TEMP tmp);
5436   size(4 * 2);
5437 
5438   format %{ "str_64 $src,$mem+$off\t! long temp=$tmp" %}
5439   ins_encode %{
5440     Register base = reg_to_register_object($mem$$base);
5441     __ add($tmp$$Register, base, $off$$constant);
5442     Address nmem = Address::make_raw($tmp$$reg, $mem$$index, $mem$$scale, $mem$$disp, relocInfo::none);
5443     __ str_64($src$$Register, nmem);
5444   %}
5445   ins_pipe(istore_mem_reg);
5446 %}
5447 #endif
5448 
5449 instruct storeL(memoryL mem, store_RegLd src) %{
5450 #ifdef AARCH64
5451   // already atomic for Aarch64
5452 #else
5453   predicate(!((StoreLNode*)n)->require_atomic_access());
5454 #endif
5455   match(Set mem (StoreL mem src));
5456   ins_cost(MEMORY_REF_COST);
5457 
5458   size(4);
5459   format %{ "str_64  $src,$mem\t! long\n\t" %}
5460 
5461   ins_encode %{
5462     __ str_64($src$$Register, $mem$$Address);
5463   %}
5464   ins_pipe(istore_mem_reg);
5465 %}
5466 
5467 #ifndef AARCH64
5468 instruct storeL_2instr(memorylong mem, iRegL src) %{
5469   predicate(!((StoreLNode*)n)->require_atomic_access());
5470   match(Set mem (StoreL mem src));
5471   ins_cost(MEMORY_REF_COST + DEFAULT_COST);
5472 
5473   size(8);
5474   format %{ "STR    $src.lo,$mem\t! long\n\t"
5475             "STR    $src.hi,$mem+4" %}
5476 
5477   ins_encode %{
5478     Address Amemlo = Address::make_raw($mem$$base, $mem$$index, $mem$$scale, $mem$$disp, relocInfo::none);
5479     Address Amemhi = Address::make_raw($mem$$base, $mem$$index, $mem$$scale, $mem$$disp + 4, relocInfo::none);
5480     __ str($src$$Register, Amemlo);
5481     __ str($src$$Register->successor(), Amemhi);
5482   %}
5483   ins_pipe(istore_mem_reg);
5484 %}
5485 
5486 instruct storeL_volatile(indirect mem, iRegL src) %{
5487   predicate(((StoreLNode*)n)->require_atomic_access());
5488   match(Set mem (StoreL mem src));
5489   ins_cost(MEMORY_REF_COST);
5490   size(4);
5491   format %{ "STMIA    $src,$mem\t! long" %}
5492   ins_encode %{
5493     // FIXME: why is stmia considered atomic?  Should be strexd
5494     RegisterSet set($src$$Register);
5495     set = set | reg_to_register_object($src$$reg + 1);
5496     __ stmia(reg_to_register_object($mem$$base), set);
5497   %}
5498   ins_pipe(istore_mem_reg);
5499 %}
5500 #endif // !AARCH64
5501 
5502 #ifndef AARCH64
5503 instruct storeL_volatile_fp(memoryD mem, iRegL src) %{
5504   predicate(((StoreLNode*)n)->require_atomic_access());
5505   match(Set mem (StoreL mem src));
5506   ins_cost(MEMORY_REF_COST);
5507   size(8);
5508   format %{ "FMDRR    S14, $src\t! long \n\t"
5509             "FSTD     S14, $mem" %}
5510   ins_encode %{
5511     __ fmdrr(S14, $src$$Register, $src$$Register->successor());
5512     __ fstd(S14, $mem$$Address);
5513   %}
5514   ins_pipe(istore_mem_reg);
5515 %}
5516 #endif
5517 
5518 #ifdef XXX
5519 // Move SP Pointer
5520 //instruct movSP(sp_ptr_RegP dst, SPRegP src) %{
5521 //instruct movSP(iRegP dst, SPRegP src) %{
5522 instruct movSP(store_ptr_RegP dst, SPRegP src) %{
5523   match(Set dst src);
5524 //predicate(!_kids[1]->_leaf->is_Proj() || _kids[1]->_leaf->as_Proj()->_con == TypeFunc::FramePtr);
5525   ins_cost(MEMORY_REF_COST);
5526   size(4);
5527 
5528   format %{ "MOV    $dst,$src\t! SP ptr\n\t" %}
5529   ins_encode %{
5530     assert(false, "XXX1 got here");
5531     __ mov($dst$$Register, SP);
5532     __ mov($dst$$Register, $src$$Register);
5533   %}
5534   ins_pipe(ialu_reg);
5535 %}
5536 #endif
5537 
5538 #ifdef AARCH64
5539 // FIXME
5540 // Store SP Pointer
5541 instruct storeSP(memoryP mem, SPRegP src, iRegP tmp) %{
5542   match(Set mem (StoreP mem src));
5543   predicate(_kids[1]->_leaf->is_Proj() && _kids[1]->_leaf->as_Proj()->_con == TypeFunc::FramePtr);
5544   // Multiple StoreP rules, different only in register mask.
5545   // Matcher makes the last always valid.  The others will
5546   // only be valid if they cost less than the last valid
5547   // rule.  So cost(rule1) < cost(rule2) < cost(last)
5548   // Unlike immediates, register constraints are not checked
5549   // at match time.
5550   ins_cost(MEMORY_REF_COST+DEFAULT_COST+4);
5551   effect(TEMP tmp);
5552   size(8);
5553 
5554   format %{ "MOV    $tmp,$src\t! SP ptr\n\t"
5555             "STR    $tmp,$mem\t! SP ptr" %}
5556   ins_encode %{
5557     assert($src$$Register == SP, "SP expected");
5558     __ mov($tmp$$Register, $src$$Register);
5559     __ str($tmp$$Register, $mem$$Address);
5560   %}
5561   ins_pipe(istore_mem_spORreg); // FIXME
5562 %}
5563 #endif // AARCH64
5564 
5565 // Store Pointer
5566 
5567 #ifdef AARCH64
5568 // XXX This variant shouldn't be necessary if 6217251 is implemented
5569 instruct storePoff(store_ptr_RegP src, memoryScaledP mem, aimmX off, iRegP tmp) %{
5570   predicate(!_kids[1]->_leaf->is_Proj() || _kids[1]->_leaf->as_Proj()->_con != TypeFunc::FramePtr);
5571   match(Set mem (StoreP (AddP mem off) src));
5572   ins_cost(MEMORY_REF_COST + DEFAULT_COST); // assume shift/sign-extend is free
5573   effect(TEMP tmp);
5574   size(4 * 2);
5575 
5576   format %{ "STR    $src,$mem+$off\t! ptr temp=$tmp" %}
5577   ins_encode %{
5578     Register base = reg_to_register_object($mem$$base);
5579     __ add($tmp$$Register, base, $off$$constant);
5580     Address nmem = Address::make_raw($tmp$$reg, $mem$$index, $mem$$scale, $mem$$disp, relocInfo::none);
5581     __ str($src$$Register, nmem);
5582   %}
5583   ins_pipe(istore_mem_reg);
5584 %}
5585 #endif
5586 
5587 instruct storeP(memoryP mem, store_ptr_RegP src) %{
5588   match(Set mem (StoreP mem src));
5589 #ifdef AARCH64
5590   predicate(!_kids[1]->_leaf->is_Proj() || _kids[1]->_leaf->as_Proj()->_con != TypeFunc::FramePtr);
5591 #endif
5592   ins_cost(MEMORY_REF_COST);
5593   size(4);
5594 
5595   format %{ "STR    $src,$mem\t! ptr" %}
5596   ins_encode %{
5597     __ str($src$$Register, $mem$$Address);
5598   %}
5599   ins_pipe(istore_mem_spORreg);
5600 %}
5601 
5602 #ifdef AARCH64
5603 // Store NULL Pointer
5604 instruct storeP0(memoryP mem, immP0 src) %{
5605   match(Set mem (StoreP mem src));
5606   ins_cost(MEMORY_REF_COST);
5607   size(4);
5608 
5609   format %{ "STR    ZR,$mem\t! ptr" %}
5610   ins_encode %{
5611     __ str(ZR, $mem$$Address);
5612   %}
5613   ins_pipe(istore_mem_spORreg);
5614 %}
5615 #endif // AARCH64
5616 
5617 #ifdef _LP64
5618 // Store Compressed Pointer
5619 
5620 #ifdef AARCH64
5621 // XXX This variant shouldn't be necessary if 6217251 is implemented
5622 instruct storeNoff(store_RegN src, memoryScaledI mem, aimmX off, iRegP tmp) %{
5623   match(Set mem (StoreN (AddP mem off) src));
5624   ins_cost(MEMORY_REF_COST + DEFAULT_COST); // assume shift/sign-extend is free
5625   effect(TEMP tmp);
5626   size(4 * 2);
5627 
5628   format %{ "str_32 $src,$mem+$off\t! compressed ptr temp=$tmp" %}
5629   ins_encode %{
5630     Register base = reg_to_register_object($mem$$base);
5631     __ add($tmp$$Register, base, $off$$constant);
5632     Address nmem = Address::make_raw($tmp$$reg, $mem$$index, $mem$$scale, $mem$$disp, relocInfo::none);
5633     __ str_32($src$$Register, nmem);
5634   %}
5635   ins_pipe(istore_mem_reg);
5636 %}
5637 #endif
5638 
5639 instruct storeN(memoryI mem, store_RegN src) %{
5640   match(Set mem (StoreN mem src));
5641   ins_cost(MEMORY_REF_COST);
5642   size(4);
5643 
5644   format %{ "str_32 $src,$mem\t! compressed ptr" %}
5645   ins_encode %{
5646     __ str_32($src$$Register, $mem$$Address);
5647   %}
5648   ins_pipe(istore_mem_reg);
5649 %}
5650 
5651 #ifdef AARCH64
5652 // Store NULL Pointer
5653 instruct storeN0(memoryI mem, immN0 src) %{
5654   match(Set mem (StoreN mem src));
5655   ins_cost(MEMORY_REF_COST);
5656   size(4);
5657 
5658   format %{ "str_32 ZR,$mem\t! compressed ptr" %}
5659   ins_encode %{
5660     __ str_32(ZR, $mem$$Address);
5661   %}
5662   ins_pipe(istore_mem_reg);
5663 %}
5664 #endif
5665 
5666 // Store Compressed Klass Pointer
5667 instruct storeNKlass(memoryI mem, store_RegN src) %{
5668   match(Set mem (StoreNKlass mem src));
5669   ins_cost(MEMORY_REF_COST);
5670   size(4);
5671 
5672   format %{ "str_32 $src,$mem\t! compressed klass ptr" %}
5673   ins_encode %{
5674     __ str_32($src$$Register, $mem$$Address);
5675   %}
5676   ins_pipe(istore_mem_reg);
5677 %}
5678 #endif
5679 
5680 // Store Double
5681 
5682 #ifdef AARCH64
5683 // XXX This variant shouldn't be necessary if 6217251 is implemented
5684 instruct storeDoff(regD src, memoryScaledD mem, aimmX off, iRegP tmp) %{
5685   match(Set mem (StoreD (AddP mem off) src));
5686   ins_cost(MEMORY_REF_COST + DEFAULT_COST); // assume shift/sign-extend is free
5687   effect(TEMP tmp);
5688   size(4 * 2);
5689 
5690   format %{ "STR    $src,$mem+$off\t! double temp=$tmp" %}
5691   ins_encode %{
5692     Register base = reg_to_register_object($mem$$base);
5693     __ add($tmp$$Register, base, $off$$constant);
5694     Address nmem = Address::make_raw($tmp$$reg, $mem$$index, $mem$$scale, $mem$$disp, relocInfo::none);
5695     __ str_d($src$$FloatRegister, nmem);
5696   %}
5697   ins_pipe(fstoreD_mem_reg);
5698 %}
5699 #endif
5700 
5701 instruct storeD(memoryD mem, regD src) %{
5702   match(Set mem (StoreD mem src));
5703   ins_cost(MEMORY_REF_COST);
5704 
5705   size(4);
5706   // FIXME: needs to be atomic, but  ARMv7 A.R.M. guarantees
5707   // only LDREXD and STREXD are 64-bit single-copy atomic
5708   format %{ "FSTD   $src,$mem" %}
5709   ins_encode %{
5710     __ str_double($src$$FloatRegister, $mem$$Address);
5711   %}
5712   ins_pipe(fstoreD_mem_reg);
5713 %}
5714 
5715 #ifdef AARCH64
5716 instruct movI2F(regF dst, iRegI src) %{
5717   match(Set dst src);
5718   size(4);
5719 
5720   format %{ "FMOV_sw $dst,$src\t! movI2F" %}
5721   ins_encode %{
5722     __ fmov_sw($dst$$FloatRegister, $src$$Register);
5723   %}
5724   ins_pipe(ialu_reg); // FIXME
5725 %}
5726 
5727 instruct movF2I(iRegI dst, regF src) %{
5728   match(Set dst src);
5729   size(4);
5730 
5731   format %{ "FMOV_ws $dst,$src\t! movF2I" %}
5732   ins_encode %{
5733     __ fmov_ws($dst$$Register, $src$$FloatRegister);
5734   %}
5735   ins_pipe(ialu_reg); // FIXME
5736 %}
5737 #endif
5738 
5739 // Store Float
5740 
5741 #ifdef AARCH64
5742 // XXX This variant shouldn't be necessary if 6217251 is implemented
5743 instruct storeFoff(regF src, memoryScaledF mem, aimmX off, iRegP tmp) %{
5744   match(Set mem (StoreF (AddP mem off) src));
5745   ins_cost(MEMORY_REF_COST + DEFAULT_COST); // assume shift/sign-extend is free
5746   effect(TEMP tmp);
5747   size(4 * 2);
5748 
5749   format %{ "str_s  $src,$mem+$off\t! float temp=$tmp" %}
5750   ins_encode %{
5751     Register base = reg_to_register_object($mem$$base);
5752     __ add($tmp$$Register, base, $off$$constant);
5753     Address nmem = Address::make_raw($tmp$$reg, $mem$$index, $mem$$scale, $mem$$disp, relocInfo::none);
5754     __ str_s($src$$FloatRegister, nmem);
5755   %}
5756   ins_pipe(fstoreF_mem_reg);
5757 %}
5758 #endif
5759 
5760 instruct storeF( memoryF mem, regF src) %{
5761   match(Set mem (StoreF mem src));
5762   ins_cost(MEMORY_REF_COST);
5763 
5764   size(4);
5765   format %{ "FSTS    $src,$mem" %}
5766   ins_encode %{
5767     __ str_float($src$$FloatRegister, $mem$$Address);
5768   %}
5769   ins_pipe(fstoreF_mem_reg);
5770 %}
5771 
5772 #ifdef AARCH64
5773 // Convert oop pointer into compressed form
5774 instruct encodeHeapOop(iRegN dst, iRegP src, flagsReg ccr) %{
5775   predicate(n->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
5776   match(Set dst (EncodeP src));
5777   effect(KILL ccr);
5778   format %{ "encode_heap_oop $dst, $src" %}
5779   ins_encode %{
5780     __ encode_heap_oop($dst$$Register, $src$$Register);
5781   %}
5782   ins_pipe(ialu_reg);
5783 %}
5784 
5785 instruct encodeHeapOop_not_null(iRegN dst, iRegP src) %{
5786   predicate(n->bottom_type()->make_ptr()->ptr() == TypePtr::NotNull);
5787   match(Set dst (EncodeP src));
5788   format %{ "encode_heap_oop_not_null $dst, $src" %}
5789   ins_encode %{
5790     __ encode_heap_oop_not_null($dst$$Register, $src$$Register);
5791   %}
5792   ins_pipe(ialu_reg);
5793 %}
5794 
5795 instruct decodeHeapOop(iRegP dst, iRegN src, flagsReg ccr) %{
5796   predicate(n->bottom_type()->is_oopptr()->ptr() != TypePtr::NotNull &&
5797             n->bottom_type()->is_oopptr()->ptr() != TypePtr::Constant);
5798   match(Set dst (DecodeN src));
5799   effect(KILL ccr);
5800   format %{ "decode_heap_oop $dst, $src" %}
5801   ins_encode %{
5802     __ decode_heap_oop($dst$$Register, $src$$Register);
5803   %}
5804   ins_pipe(ialu_reg);
5805 %}
5806 
5807 instruct decodeHeapOop_not_null(iRegP dst, iRegN src) %{
5808   predicate(n->bottom_type()->is_oopptr()->ptr() == TypePtr::NotNull ||
5809             n->bottom_type()->is_oopptr()->ptr() == TypePtr::Constant);
5810   match(Set dst (DecodeN src));
5811   format %{ "decode_heap_oop_not_null $dst, $src" %}
5812   ins_encode %{
5813     __ decode_heap_oop_not_null($dst$$Register, $src$$Register);
5814   %}
5815   ins_pipe(ialu_reg);
5816 %}
5817 
5818 instruct encodeKlass_not_null(iRegN dst, iRegP src) %{
5819   match(Set dst (EncodePKlass src));
5820   format %{ "encode_klass_not_null $dst, $src" %}
5821   ins_encode %{
5822     __ encode_klass_not_null($dst$$Register, $src$$Register);
5823   %}
5824   ins_pipe(ialu_reg);
5825 %}
5826 
5827 instruct decodeKlass_not_null(iRegP dst, iRegN src) %{
5828   match(Set dst (DecodeNKlass src));
5829   format %{ "decode_klass_not_null $dst, $src" %}
5830   ins_encode %{
5831     __ decode_klass_not_null($dst$$Register, $src$$Register);
5832   %}
5833   ins_pipe(ialu_reg);
5834 %}
5835 #endif // AARCH64
5836 
5837 //----------MemBar Instructions-----------------------------------------------
5838 // Memory barrier flavors
5839 
5840 // TODO: take advantage of Aarch64 load-acquire, store-release, etc
5841 // pattern-match out unnecessary membars
5842 instruct membar_storestore() %{
5843   match(MemBarStoreStore);
5844   ins_cost(4*MEMORY_REF_COST);
5845 
5846   size(4);
5847   format %{ "MEMBAR-storestore" %}
5848   ins_encode %{
5849     __ membar(MacroAssembler::Membar_mask_bits(MacroAssembler::StoreStore), noreg);
5850   %}
5851   ins_pipe(long_memory_op);
5852 %}
5853 
5854 instruct membar_acquire() %{
5855   match(MemBarAcquire);
5856   match(LoadFence);
5857   ins_cost(4*MEMORY_REF_COST);
5858 
5859   size(4);
5860   format %{ "MEMBAR-acquire" %}
5861   ins_encode %{
5862     __ membar(MacroAssembler::Membar_mask_bits(MacroAssembler::LoadLoad | MacroAssembler::LoadStore), noreg);
5863   %}
5864   ins_pipe(long_memory_op);
5865 %}
5866 
5867 instruct membar_acquire_lock() %{
5868   match(MemBarAcquireLock);
5869   ins_cost(0);
5870 
5871   size(0);
5872   format %{ "!MEMBAR-acquire (CAS in prior FastLock so empty encoding)" %}
5873   ins_encode( );
5874   ins_pipe(empty);
5875 %}
5876 
5877 instruct membar_release() %{
5878   match(MemBarRelease);
5879   match(StoreFence);
5880   ins_cost(4*MEMORY_REF_COST);
5881 
5882   size(4);
5883   format %{ "MEMBAR-release" %}
5884   ins_encode %{
5885     __ membar(MacroAssembler::Membar_mask_bits(MacroAssembler::StoreStore | MacroAssembler::LoadStore), noreg);
5886   %}
5887   ins_pipe(long_memory_op);
5888 %}
5889 
5890 instruct membar_release_lock() %{
5891   match(MemBarReleaseLock);
5892   ins_cost(0);
5893 
5894   size(0);
5895   format %{ "!MEMBAR-release (CAS in succeeding FastUnlock so empty encoding)" %}
5896   ins_encode( );
5897   ins_pipe(empty);
5898 %}
5899 
5900 instruct membar_volatile() %{
5901   match(MemBarVolatile);
5902   ins_cost(4*MEMORY_REF_COST);
5903 
5904   size(4);
5905   format %{ "MEMBAR-volatile" %}
5906   ins_encode %{
5907     __ membar(MacroAssembler::StoreLoad, noreg);
5908   %}
5909   ins_pipe(long_memory_op);
5910 %}
5911 
5912 instruct unnecessary_membar_volatile() %{
5913   match(MemBarVolatile);
5914   predicate(Matcher::post_store_load_barrier(n));
5915   ins_cost(0);
5916 
5917   size(0);
5918   format %{ "!MEMBAR-volatile (unnecessary so empty encoding)" %}
5919   ins_encode( );
5920   ins_pipe(empty);
5921 %}
5922 
5923 //----------Register Move Instructions-----------------------------------------
5924 // instruct roundDouble_nop(regD dst) %{
5925 //   match(Set dst (RoundDouble dst));
5926 //   ins_pipe(empty);
5927 // %}
5928 
5929 
5930 // instruct roundFloat_nop(regF dst) %{
5931 //   match(Set dst (RoundFloat dst));
5932 //   ins_pipe(empty);
5933 // %}
5934 
5935 
5936 #ifdef AARCH64
5937 // 0 constant in register
5938 instruct zrImmI0(ZRRegI dst, immI0 imm) %{
5939   match(Set dst imm);
5940   size(0);
5941   ins_cost(0);
5942 
5943   format %{ "! ZR (int 0)" %}
5944   ins_encode( /*empty encoding*/ );
5945   ins_pipe(ialu_none);
5946 %}
5947 
5948 // 0 constant in register
5949 instruct zrImmL0(ZRRegL dst, immL0 imm) %{
5950   match(Set dst imm);
5951   size(0);
5952   ins_cost(0);
5953 
5954   format %{ "! ZR (long 0)" %}
5955   ins_encode( /*empty encoding*/ );
5956   ins_pipe(ialu_none);
5957 %}
5958 
5959 #ifdef XXX
5960 // 0 constant in register
5961 instruct zrImmN0(ZRRegN dst, immN0 imm) %{
5962   match(Set dst imm);
5963   size(0);
5964   ins_cost(0);
5965 
5966   format %{ "! ZR (compressed pointer NULL)" %}
5967   ins_encode( /*empty encoding*/ );
5968   ins_pipe(ialu_none);
5969 %}
5970 
5971 // 0 constant in register
5972 instruct zrImmP0(ZRRegP dst, immP0 imm) %{
5973   match(Set dst imm);
5974   size(0);
5975   ins_cost(0);
5976 
5977   format %{ "! ZR (NULL)" %}
5978   ins_encode( /*empty encoding*/ );
5979   ins_pipe(ialu_none);
5980 %}
5981 #endif
5982 #endif // AARCH64
5983 
5984 // Cast Index to Pointer for unsafe natives
5985 instruct castX2P(iRegX src, iRegP dst) %{
5986   match(Set dst (CastX2P src));
5987 
5988   format %{ "MOV    $dst,$src\t! IntX->Ptr if $dst != $src" %}
5989   ins_encode %{
5990     if ($dst$$Register !=  $src$$Register) {
5991       __ mov($dst$$Register, $src$$Register);
5992     }
5993   %}
5994   ins_pipe(ialu_reg);
5995 %}
5996 
5997 // Cast Pointer to Index for unsafe natives
5998 instruct castP2X(iRegP src, iRegX dst) %{
5999   match(Set dst (CastP2X src));
6000 
6001   format %{ "MOV    $dst,$src\t! Ptr->IntX if $dst != $src" %}
6002   ins_encode %{
6003     if ($dst$$Register !=  $src$$Register) {
6004       __ mov($dst$$Register, $src$$Register);
6005     }
6006   %}
6007   ins_pipe(ialu_reg);
6008 %}
6009 
6010 #ifndef AARCH64
6011 //----------Conditional Move---------------------------------------------------
6012 // Conditional move
6013 instruct cmovIP_reg(cmpOpP cmp, flagsRegP pcc, iRegI dst, iRegI src) %{
6014   match(Set dst (CMoveI (Binary cmp pcc) (Binary dst src)));
6015   ins_cost(150);
6016   size(4);
6017   format %{ "MOV$cmp  $dst,$src\t! int" %}
6018   ins_encode %{
6019     __ mov($dst$$Register, $src$$Register, (AsmCondition)($cmp$$cmpcode));
6020   %}
6021   ins_pipe(ialu_reg);
6022 %}
6023 #endif
6024 
6025 #ifdef AARCH64
6026 instruct cmovI_reg3(cmpOp cmp, flagsReg icc, iRegI dst, iRegI src1, iRegI src2) %{
6027   match(Set dst (CMoveI (Binary cmp icc) (Binary src2 src1)));
6028   ins_cost(150);
6029   size(4);
6030   format %{ "CSEL $dst,$src1,$src2,$cmp\t! int" %}
6031   ins_encode %{
6032     __ csel($dst$$Register, $src1$$Register, $src2$$Register, (AsmCondition)($cmp$$cmpcode));
6033   %}
6034   ins_pipe(ialu_reg);
6035 %}
6036 
6037 instruct cmovL_reg3(cmpOp cmp, flagsReg icc, iRegL dst, iRegL src1, iRegL src2) %{
6038   match(Set dst (CMoveL (Binary cmp icc) (Binary src2 src1)));
6039   ins_cost(150);
6040   size(4);
6041   format %{ "CSEL $dst,$src1,$src2,$cmp\t! long" %}
6042   ins_encode %{
6043     __ csel($dst$$Register, $src1$$Register, $src2$$Register, (AsmCondition)($cmp$$cmpcode));
6044   %}
6045   ins_pipe(ialu_reg);
6046 %}
6047 
6048 instruct cmovP_reg3(cmpOp cmp, flagsReg icc, iRegP dst, iRegP src1, iRegP src2) %{
6049   match(Set dst (CMoveP (Binary cmp icc) (Binary src2 src1)));
6050   ins_cost(150);
6051   size(4);
6052   format %{ "CSEL $dst,$src1,$src2,$cmp\t! ptr" %}
6053   ins_encode %{
6054     __ csel($dst$$Register, $src1$$Register, $src2$$Register, (AsmCondition)($cmp$$cmpcode));
6055   %}
6056   ins_pipe(ialu_reg);
6057 %}
6058 
6059 instruct cmovN_reg3(cmpOp cmp, flagsReg icc, iRegN dst, iRegN src1, iRegN src2) %{
6060   match(Set dst (CMoveN (Binary cmp icc) (Binary src2 src1)));
6061   ins_cost(150);
6062   size(4);
6063   format %{ "CSEL $dst,$src1,$src2,$cmp\t! compressed ptr" %}
6064   ins_encode %{
6065     __ csel($dst$$Register, $src1$$Register, $src2$$Register, (AsmCondition)($cmp$$cmpcode));
6066   %}
6067   ins_pipe(ialu_reg);
6068 %}
6069 
6070 instruct cmovIP_reg3(cmpOpP cmp, flagsRegP icc, iRegI dst, iRegI src1, iRegI src2) %{
6071   match(Set dst (CMoveI (Binary cmp icc) (Binary src2 src1)));
6072   ins_cost(150);
6073   size(4);
6074   format %{ "CSEL $dst,$src1,$src2,$cmp\t! int" %}
6075   ins_encode %{
6076     __ csel($dst$$Register, $src1$$Register, $src2$$Register, (AsmCondition)($cmp$$cmpcode));
6077   %}
6078   ins_pipe(ialu_reg);
6079 %}
6080 
6081 instruct cmovLP_reg3(cmpOpP cmp, flagsRegP icc, iRegL dst, iRegL src1, iRegL src2) %{
6082   match(Set dst (CMoveL (Binary cmp icc) (Binary src2 src1)));
6083   ins_cost(150);
6084   size(4);
6085   format %{ "CSEL $dst,$src1,$src2,$cmp\t! long" %}
6086   ins_encode %{
6087     __ csel($dst$$Register, $src1$$Register, $src2$$Register, (AsmCondition)($cmp$$cmpcode));
6088   %}
6089   ins_pipe(ialu_reg);
6090 %}
6091 
6092 instruct cmovPP_reg3(cmpOpP cmp, flagsRegP icc, iRegP dst, iRegP src1, iRegP src2) %{
6093   match(Set dst (CMoveP (Binary cmp icc) (Binary src2 src1)));
6094   ins_cost(150);
6095   size(4);
6096   format %{ "CSEL $dst,$src1,$src2,$cmp\t! ptr" %}
6097   ins_encode %{
6098     __ csel($dst$$Register, $src1$$Register, $src2$$Register, (AsmCondition)($cmp$$cmpcode));
6099   %}
6100   ins_pipe(ialu_reg);
6101 %}
6102 
6103 instruct cmovNP_reg3(cmpOpP cmp, flagsRegP icc, iRegN dst, iRegN src1, iRegN src2) %{
6104   match(Set dst (CMoveN (Binary cmp icc) (Binary src2 src1)));
6105   ins_cost(150);
6106   size(4);
6107   format %{ "CSEL $dst,$src1,$src2,$cmp\t! compressed ptr" %}
6108   ins_encode %{
6109     __ csel($dst$$Register, $src1$$Register, $src2$$Register, (AsmCondition)($cmp$$cmpcode));
6110   %}
6111   ins_pipe(ialu_reg);
6112 %}
6113 
6114 instruct cmovIU_reg3(cmpOpU cmp, flagsRegU icc, iRegI dst, iRegI src1, iRegI src2) %{
6115   match(Set dst (CMoveI (Binary cmp icc) (Binary src2 src1)));
6116   ins_cost(150);
6117   size(4);
6118   format %{ "CSEL $dst,$src1,$src2,$cmp\t! int" %}
6119   ins_encode %{
6120     __ csel($dst$$Register, $src1$$Register, $src2$$Register, (AsmCondition)($cmp$$cmpcode));
6121   %}
6122   ins_pipe(ialu_reg);
6123 %}
6124 
6125 instruct cmovLU_reg3(cmpOpU cmp, flagsRegU icc, iRegL dst, iRegL src1, iRegL src2) %{
6126   match(Set dst (CMoveL (Binary cmp icc) (Binary src2 src1)));
6127   ins_cost(150);
6128   size(4);
6129   format %{ "CSEL $dst,$src1,$src2,$cmp\t! long" %}
6130   ins_encode %{
6131     __ csel($dst$$Register, $src1$$Register, $src2$$Register, (AsmCondition)($cmp$$cmpcode));
6132   %}
6133   ins_pipe(ialu_reg);
6134 %}
6135 
6136 instruct cmovPU_reg3(cmpOpU cmp, flagsRegU icc, iRegP dst, iRegP src1, iRegP src2) %{
6137   match(Set dst (CMoveP (Binary cmp icc) (Binary src2 src1)));
6138   ins_cost(150);
6139   size(4);
6140   format %{ "CSEL $dst,$src1,$src2,$cmp\t! ptr" %}
6141   ins_encode %{
6142     __ csel($dst$$Register, $src1$$Register, $src2$$Register, (AsmCondition)($cmp$$cmpcode));
6143   %}
6144   ins_pipe(ialu_reg);
6145 %}
6146 
6147 instruct cmovNU_reg3(cmpOpU cmp, flagsRegU icc, iRegN dst, iRegN src1, iRegN src2) %{
6148   match(Set dst (CMoveN (Binary cmp icc) (Binary src2 src1)));
6149   ins_cost(150);
6150   size(4);
6151   format %{ "CSEL $dst,$src1,$src2,$cmp\t! compressed ptr" %}
6152   ins_encode %{
6153     __ csel($dst$$Register, $src1$$Register, $src2$$Register, (AsmCondition)($cmp$$cmpcode));
6154   %}
6155   ins_pipe(ialu_reg);
6156 %}
6157 
6158 instruct cmovIZ_reg3(cmpOp0 cmp, flagsReg_EQNELTGE icc, iRegI dst, iRegI src1, iRegI src2) %{
6159   match(Set dst (CMoveI (Binary cmp icc) (Binary src2 src1)));
6160   ins_cost(150);
6161   size(4);
6162   format %{ "CSEL $dst,$src1,$src2,$cmp\t! int" %}
6163   ins_encode %{
6164     __ csel($dst$$Register, $src1$$Register, $src2$$Register, (AsmCondition)($cmp$$cmpcode));
6165   %}
6166   ins_pipe(ialu_reg);
6167 %}
6168 
6169 instruct cmovLZ_reg3(cmpOp0 cmp, flagsReg_EQNELTGE icc, iRegL dst, iRegL src1, iRegL src2) %{
6170   match(Set dst (CMoveL (Binary cmp icc) (Binary src2 src1)));
6171   ins_cost(150);
6172   size(4);
6173   format %{ "CSEL $dst,$src1,$src2,$cmp\t! long" %}
6174   ins_encode %{
6175     __ csel($dst$$Register, $src1$$Register, $src2$$Register, (AsmCondition)($cmp$$cmpcode));
6176   %}
6177   ins_pipe(ialu_reg);
6178 %}
6179 
6180 instruct cmovPZ_reg3(cmpOp0 cmp, flagsReg_EQNELTGE icc, iRegP dst, iRegP src1, iRegP src2) %{
6181   match(Set dst (CMoveP (Binary cmp icc) (Binary src2 src1)));
6182   ins_cost(150);
6183   size(4);
6184   format %{ "CSEL $dst,$src1,$src2,$cmp\t! ptr" %}
6185   ins_encode %{
6186     __ csel($dst$$Register, $src1$$Register, $src2$$Register, (AsmCondition)($cmp$$cmpcode));
6187   %}
6188   ins_pipe(ialu_reg);
6189 %}
6190 
6191 instruct cmovNZ_reg3(cmpOp0 cmp, flagsReg_EQNELTGE icc, iRegN dst, iRegN src1, iRegN src2) %{
6192   match(Set dst (CMoveN (Binary cmp icc) (Binary src2 src1)));
6193   ins_cost(150);
6194   size(4);
6195   format %{ "CSEL $dst,$src1,$src2,$cmp\t! compressed ptr" %}
6196   ins_encode %{
6197     __ csel($dst$$Register, $src1$$Register, $src2$$Register, (AsmCondition)($cmp$$cmpcode));
6198   %}
6199   ins_pipe(ialu_reg);
6200 %}
6201 #endif // AARCH64
6202 
6203 #ifndef AARCH64
6204 instruct cmovIP_immMov(cmpOpP cmp, flagsRegP pcc, iRegI dst, immIMov src) %{
6205   match(Set dst (CMoveI (Binary cmp pcc) (Binary dst src)));
6206   ins_cost(140);
6207   size(4);
6208   format %{ "MOV$cmp  $dst,$src" %}
6209   ins_encode %{
6210     __ mov($dst$$Register, $src$$constant, (AsmCondition)($cmp$$cmpcode));
6211   %}
6212   ins_pipe(ialu_imm);
6213 %}
6214 
6215 instruct cmovIP_imm16(cmpOpP cmp, flagsRegP pcc, iRegI dst, immI16 src) %{
6216   match(Set dst (CMoveI (Binary cmp pcc) (Binary dst src)));
6217   ins_cost(140);
6218   size(4);
6219   format %{ "MOVw$cmp  $dst,$src" %}
6220   ins_encode %{
6221     __ movw($dst$$Register, $src$$constant, (AsmCondition)($cmp$$cmpcode));
6222   %}
6223   ins_pipe(ialu_imm);
6224 %}
6225 #endif
6226 
6227 instruct cmovI_reg(cmpOp cmp, flagsReg icc, iRegI dst, iRegI src) %{
6228   match(Set dst (CMoveI (Binary cmp icc) (Binary dst src)));
6229   ins_cost(150);
6230   size(4);
6231   format %{ "MOV$cmp  $dst,$src" %}
6232   ins_encode %{
6233     __ mov($dst$$Register, $src$$Register, (AsmCondition)($cmp$$cmpcode));
6234   %}
6235   ins_pipe(ialu_reg);
6236 %}
6237 
6238 #ifdef AARCH64
6239 instruct cmovL_reg(cmpOp cmp, flagsReg icc, iRegL dst, iRegL src) %{
6240   match(Set dst (CMoveL (Binary cmp icc) (Binary dst src)));
6241   ins_cost(150);
6242   size(4);
6243   format %{ "MOV$cmp  $dst,$src\t! long" %}
6244   ins_encode %{
6245     __ mov($dst$$Register, $src$$Register, (AsmCondition)($cmp$$cmpcode));
6246   %}
6247   ins_pipe(ialu_reg);
6248 %}
6249 #endif
6250 
6251 #ifndef AARCH64
6252 instruct cmovI_immMov(cmpOp cmp, flagsReg icc, iRegI dst, immIMov src) %{
6253   match(Set dst (CMoveI (Binary cmp icc) (Binary dst src)));
6254   ins_cost(140);
6255   size(4);
6256   format %{ "MOV$cmp  $dst,$src" %}
6257   ins_encode %{
6258     __ mov($dst$$Register, $src$$constant, (AsmCondition)($cmp$$cmpcode));
6259   %}
6260   ins_pipe(ialu_imm);
6261 %}
6262 
6263 instruct cmovII_imm16(cmpOp cmp, flagsReg icc, iRegI dst, immI16 src) %{
6264   match(Set dst (CMoveI (Binary cmp icc) (Binary dst src)));
6265   ins_cost(140);
6266   size(4);
6267   format %{ "MOVw$cmp  $dst,$src" %}
6268   ins_encode %{
6269     __ movw($dst$$Register, $src$$constant, (AsmCondition)($cmp$$cmpcode));
6270   %}
6271   ins_pipe(ialu_imm);
6272 %}
6273 #endif
6274 
6275 instruct cmovII_reg_EQNELTGE(cmpOp0 cmp, flagsReg_EQNELTGE icc, iRegI dst, iRegI src) %{
6276   match(Set dst (CMoveI (Binary cmp icc) (Binary dst src)));
6277   predicate(_kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::eq ||
6278             _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ne ||
6279             _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::lt ||
6280             _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ge);
6281   ins_cost(150);
6282   size(4);
6283   format %{ "MOV$cmp  $dst,$src" %}
6284   ins_encode %{
6285     __ mov($dst$$Register, $src$$Register, (AsmCondition)($cmp$$cmpcode));
6286   %}
6287   ins_pipe(ialu_reg);
6288 %}
6289 
6290 #ifndef AARCH64
6291 instruct cmovII_immMov_EQNELTGE(cmpOp0 cmp, flagsReg_EQNELTGE icc, iRegI dst, immIMov src) %{
6292   match(Set dst (CMoveI (Binary cmp icc) (Binary dst src)));
6293   predicate(_kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::eq ||
6294             _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ne ||
6295             _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::lt ||
6296             _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ge);
6297   ins_cost(140);
6298   size(4);
6299   format %{ "MOV$cmp  $dst,$src" %}
6300   ins_encode %{
6301     __ mov($dst$$Register, $src$$constant, (AsmCondition)($cmp$$cmpcode));
6302   %}
6303   ins_pipe(ialu_imm);
6304 %}
6305 
6306 instruct cmovII_imm16_EQNELTGE(cmpOp0 cmp, flagsReg_EQNELTGE icc, iRegI dst, immI16 src) %{
6307   match(Set dst (CMoveI (Binary cmp icc) (Binary dst src)));
6308   predicate(_kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::eq ||
6309             _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ne ||
6310             _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::lt ||
6311             _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ge);
6312   ins_cost(140);
6313   size(4);
6314   format %{ "MOVW$cmp  $dst,$src" %}
6315   ins_encode %{
6316     __ movw($dst$$Register, $src$$constant, (AsmCondition)($cmp$$cmpcode));
6317   %}
6318   ins_pipe(ialu_imm);
6319 %}
6320 #endif
6321 
6322 instruct cmovIIu_reg(cmpOpU cmp, flagsRegU icc, iRegI dst, iRegI src) %{
6323   match(Set dst (CMoveI (Binary cmp icc) (Binary dst src)));
6324   ins_cost(150);
6325   size(4);
6326   format %{ "MOV$cmp  $dst,$src" %}
6327   ins_encode %{
6328     __ mov($dst$$Register, $src$$Register, (AsmCondition)($cmp$$cmpcode));
6329   %}
6330   ins_pipe(ialu_reg);
6331 %}
6332 
6333 #ifndef AARCH64
6334 instruct cmovIIu_immMov(cmpOpU cmp, flagsRegU icc, iRegI dst, immIMov src) %{
6335   match(Set dst (CMoveI (Binary cmp icc) (Binary dst src)));
6336   ins_cost(140);
6337   size(4);
6338   format %{ "MOV$cmp  $dst,$src" %}
6339   ins_encode %{
6340     __ mov($dst$$Register, $src$$constant, (AsmCondition)($cmp$$cmpcode));
6341   %}
6342   ins_pipe(ialu_imm);
6343 %}
6344 
6345 instruct cmovIIu_imm16(cmpOpU cmp, flagsRegU icc, iRegI dst, immI16 src) %{
6346   match(Set dst (CMoveI (Binary cmp icc) (Binary dst src)));
6347   ins_cost(140);
6348   size(4);
6349   format %{ "MOVW$cmp  $dst,$src" %}
6350   ins_encode %{
6351     __ movw($dst$$Register, $src$$constant, (AsmCondition)($cmp$$cmpcode));
6352   %}
6353   ins_pipe(ialu_imm);
6354 %}
6355 #endif
6356 
6357 // Conditional move
6358 instruct cmovPP_reg(cmpOpP cmp, flagsRegP pcc, iRegP dst, iRegP src) %{
6359   match(Set dst (CMoveP (Binary cmp pcc) (Binary dst src)));
6360   ins_cost(150);
6361   size(4);
6362   format %{ "MOV$cmp  $dst,$src" %}
6363   ins_encode %{
6364     __ mov($dst$$Register, $src$$Register, (AsmCondition)($cmp$$cmpcode));
6365   %}
6366   ins_pipe(ialu_reg);
6367 %}
6368 
6369 instruct cmovPP_imm(cmpOpP cmp, flagsRegP pcc, iRegP dst, immP0 src) %{
6370   match(Set dst (CMoveP (Binary cmp pcc) (Binary dst src)));
6371   ins_cost(140);
6372   size(4);
6373 #ifdef AARCH64
6374   format %{ "MOV$cmp  $dst,ZR" %}
6375 #else
6376   format %{ "MOV$cmp  $dst,$src" %}
6377 #endif
6378   ins_encode %{
6379 #ifdef AARCH64
6380     __ mov($dst$$Register,             ZR, (AsmCondition)($cmp$$cmpcode));
6381 #else
6382     __ mov($dst$$Register, $src$$constant, (AsmCondition)($cmp$$cmpcode));
6383 #endif
6384   %}
6385   ins_pipe(ialu_imm);
6386 %}
6387 
6388 // This instruction also works with CmpN so we don't need cmovPN_reg.
6389 instruct cmovPI_reg(cmpOp cmp, flagsReg icc, iRegP dst, iRegP src) %{
6390   match(Set dst (CMoveP (Binary cmp icc) (Binary dst src)));
6391   ins_cost(150);
6392 
6393   size(4);
6394   format %{ "MOV$cmp  $dst,$src\t! ptr" %}
6395   ins_encode %{
6396     __ mov($dst$$Register, $src$$Register, (AsmCondition)($cmp$$cmpcode));
6397   %}
6398   ins_pipe(ialu_reg);
6399 %}
6400 
6401 instruct cmovPI_reg_EQNELTGE(cmpOp0 cmp, flagsReg_EQNELTGE icc, iRegP dst, iRegP src) %{
6402   match(Set dst (CMoveP (Binary cmp icc) (Binary dst src)));
6403   predicate(_kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::eq ||
6404             _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ne ||
6405             _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::lt ||
6406             _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ge);
6407   ins_cost(150);
6408 
6409   size(4);
6410   format %{ "MOV$cmp  $dst,$src\t! ptr" %}
6411   ins_encode %{
6412     __ mov($dst$$Register, $src$$Register, (AsmCondition)($cmp$$cmpcode));
6413   %}
6414   ins_pipe(ialu_reg);
6415 %}
6416 
6417 instruct cmovPIu_reg(cmpOpU cmp, flagsRegU icc, iRegP dst, iRegP src) %{
6418   match(Set dst (CMoveP (Binary cmp icc) (Binary dst src)));
6419   ins_cost(150);
6420 
6421   size(4);
6422   format %{ "MOV$cmp  $dst,$src\t! ptr" %}
6423   ins_encode %{
6424     __ mov($dst$$Register, $src$$Register, (AsmCondition)($cmp$$cmpcode));
6425   %}
6426   ins_pipe(ialu_reg);
6427 %}
6428 
6429 instruct cmovPI_imm(cmpOp cmp, flagsReg icc, iRegP dst, immP0 src) %{
6430   match(Set dst (CMoveP (Binary cmp icc) (Binary dst src)));
6431   ins_cost(140);
6432 
6433   size(4);
6434 #ifdef AARCH64
6435   format %{ "MOV$cmp  $dst,ZR\t! ptr" %}
6436 #else
6437   format %{ "MOV$cmp  $dst,$src\t! ptr" %}
6438 #endif
6439   ins_encode %{
6440 #ifdef AARCH64
6441     __ mov($dst$$Register,             ZR, (AsmCondition)($cmp$$cmpcode));
6442 #else
6443     __ mov($dst$$Register, $src$$constant, (AsmCondition)($cmp$$cmpcode));
6444 #endif
6445   %}
6446   ins_pipe(ialu_imm);
6447 %}
6448 
6449 instruct cmovPI_imm_EQNELTGE(cmpOp0 cmp, flagsReg_EQNELTGE icc, iRegP dst, immP0 src) %{
6450   match(Set dst (CMoveP (Binary cmp icc) (Binary dst src)));
6451   predicate(_kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::eq ||
6452             _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ne ||
6453             _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::lt ||
6454             _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ge);
6455   ins_cost(140);
6456 
6457   size(4);
6458 #ifdef AARCH64
6459   format %{ "MOV$cmp  $dst,ZR\t! ptr" %}
6460 #else
6461   format %{ "MOV$cmp  $dst,$src\t! ptr" %}
6462 #endif
6463   ins_encode %{
6464 #ifdef AARCH64
6465     __ mov($dst$$Register,             ZR, (AsmCondition)($cmp$$cmpcode));
6466 #else
6467     __ mov($dst$$Register, $src$$constant, (AsmCondition)($cmp$$cmpcode));
6468 #endif
6469   %}
6470   ins_pipe(ialu_imm);
6471 %}
6472 
6473 instruct cmovPIu_imm(cmpOpU cmp, flagsRegU icc, iRegP dst, immP0 src) %{
6474   match(Set dst (CMoveP (Binary cmp icc) (Binary dst src)));
6475   ins_cost(140);
6476 
6477   size(4);
6478 #ifdef AARCH64
6479   format %{ "MOV$cmp  $dst,ZR\t! ptr" %}
6480 #else
6481   format %{ "MOV$cmp  $dst,$src\t! ptr" %}
6482 #endif
6483   ins_encode %{
6484 #ifdef AARCH64
6485     __ mov($dst$$Register,             ZR, (AsmCondition)($cmp$$cmpcode));
6486 #else
6487     __ mov($dst$$Register, $src$$constant, (AsmCondition)($cmp$$cmpcode));
6488 #endif
6489   %}
6490   ins_pipe(ialu_imm);
6491 %}
6492 
6493 #ifdef AARCH64
6494 // Conditional move
6495 instruct cmovF_reg(cmpOp cmp, flagsReg icc, regF dst, regF src1, regF src2) %{
6496   match(Set dst (CMoveF (Binary cmp icc) (Binary src2 src1)));
6497   ins_cost(150);
6498   size(4);
6499   format %{ "FCSEL_s $dst,$src1,$src2,$cmp" %}
6500   ins_encode %{
6501     __ fcsel_s($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister, (AsmCondition)($cmp$$cmpcode));
6502   %}
6503   ins_pipe(int_conditional_float_move);
6504 %}
6505 
6506 instruct cmovD_reg(cmpOp cmp, flagsReg icc, regD dst, regD src1, regD src2) %{
6507   match(Set dst (CMoveD (Binary cmp icc) (Binary src2 src1)));
6508   ins_cost(150);
6509   size(4);
6510   format %{ "FCSEL_d $dst,$src1,$src2,$cmp" %}
6511   ins_encode %{
6512     __ fcsel_d($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister, (AsmCondition)($cmp$$cmpcode));
6513   %}
6514   ins_pipe(int_conditional_float_move);
6515 %}
6516 
6517 instruct cmovFP_reg(cmpOpP cmp, flagsRegP icc, regF dst, regF src1, regF src2) %{
6518   match(Set dst (CMoveF (Binary cmp icc) (Binary src2 src1)));
6519   ins_cost(150);
6520   size(4);
6521   format %{ "FCSEL_s $dst,$src1,$src2,$cmp" %}
6522   ins_encode %{
6523     __ fcsel_s($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister, (AsmCondition)($cmp$$cmpcode));
6524   %}
6525   ins_pipe(int_conditional_float_move);
6526 %}
6527 
6528 instruct cmovDP_reg(cmpOpP cmp, flagsRegP icc, regD dst, regD src1, regD src2) %{
6529   match(Set dst (CMoveD (Binary cmp icc) (Binary src2 src1)));
6530   ins_cost(150);
6531   size(4);
6532   format %{ "FCSEL_d $dst,$src1,$src2,$cmp" %}
6533   ins_encode %{
6534     __ fcsel_d($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister, (AsmCondition)($cmp$$cmpcode));
6535   %}
6536   ins_pipe(int_conditional_float_move);
6537 %}
6538 
6539 instruct cmovFU_reg(cmpOpU cmp, flagsRegU icc, regF dst, regF src1, regF src2) %{
6540   match(Set dst (CMoveF (Binary cmp icc) (Binary src2 src1)));
6541   ins_cost(150);
6542   size(4);
6543   format %{ "FCSEL_s $dst,$src1,$src2,$cmp" %}
6544   ins_encode %{
6545     __ fcsel_s($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister, (AsmCondition)($cmp$$cmpcode));
6546   %}
6547   ins_pipe(int_conditional_float_move);
6548 %}
6549 
6550 instruct cmovDU_reg(cmpOpU cmp, flagsRegU icc, regD dst, regD src1, regD src2) %{
6551   match(Set dst (CMoveD (Binary cmp icc) (Binary src2 src1)));
6552   ins_cost(150);
6553   size(4);
6554   format %{ "FCSEL_d $dst,$src1,$src2,$cmp" %}
6555   ins_encode %{
6556     __ fcsel_d($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister, (AsmCondition)($cmp$$cmpcode));
6557   %}
6558   ins_pipe(int_conditional_float_move);
6559 %}
6560 
6561 instruct cmovFZ_reg(cmpOp0 cmp, flagsReg_EQNELTGE icc, regF dst, regF src1, regF src2) %{
6562   match(Set dst (CMoveF (Binary cmp icc) (Binary src2 src1)));
6563   ins_cost(150);
6564   size(4);
6565   format %{ "FCSEL_s $dst,$src1,$src2,$cmp" %}
6566   ins_encode %{
6567     __ fcsel_s($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister, (AsmCondition)($cmp$$cmpcode));
6568   %}
6569   ins_pipe(int_conditional_float_move);
6570 %}
6571 
6572 instruct cmovDZ_reg(cmpOp0 cmp, flagsReg_EQNELTGE icc, regD dst, regD src1, regD src2) %{
6573   match(Set dst (CMoveD (Binary cmp icc) (Binary src2 src1)));
6574   ins_cost(150);
6575   size(4);
6576   format %{ "FCSEL_d $dst,$src1,$src2,$cmp" %}
6577   ins_encode %{
6578     __ fcsel_d($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister, (AsmCondition)($cmp$$cmpcode));
6579   %}
6580   ins_pipe(int_conditional_float_move);
6581 %}
6582 
6583 #else // !AARCH64
6584 
6585 // Conditional move
6586 instruct cmovFP_reg(cmpOpP cmp, flagsRegP pcc, regF dst, regF src) %{
6587   match(Set dst (CMoveF (Binary cmp pcc) (Binary dst src)));
6588   ins_cost(150);
6589   size(4);
6590   format %{ "FCPYS$cmp $dst,$src" %}
6591   ins_encode %{
6592     __ fcpys($dst$$FloatRegister, $src$$FloatRegister, (AsmCondition)($cmp$$cmpcode));
6593   %}
6594   ins_pipe(int_conditional_float_move);
6595 %}
6596 
6597 instruct cmovFI_reg(cmpOp cmp, flagsReg icc, regF dst, regF src) %{
6598   match(Set dst (CMoveF (Binary cmp icc) (Binary dst src)));
6599   ins_cost(150);
6600 
6601   size(4);
6602   format %{ "FCPYS$cmp $dst,$src" %}
6603   ins_encode %{
6604     __ fcpys($dst$$FloatRegister, $src$$FloatRegister, (AsmCondition)($cmp$$cmpcode));
6605   %}
6606   ins_pipe(int_conditional_float_move);
6607 %}
6608 
6609 instruct cmovFI_reg_EQNELTGE(cmpOp0 cmp, flagsReg_EQNELTGE icc, regF dst, regF src) %{
6610   match(Set dst (CMoveF (Binary cmp icc) (Binary dst src)));
6611   predicate(_kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::eq ||
6612             _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ne ||
6613             _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::lt ||
6614             _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ge);
6615   ins_cost(150);
6616 
6617   size(4);
6618   format %{ "FCPYS$cmp $dst,$src" %}
6619   ins_encode %{
6620     __ fcpys($dst$$FloatRegister, $src$$FloatRegister, (AsmCondition)($cmp$$cmpcode));
6621   %}
6622   ins_pipe(int_conditional_float_move);
6623 %}
6624 
6625 instruct cmovFIu_reg(cmpOpU cmp, flagsRegU icc, regF dst, regF src) %{
6626   match(Set dst (CMoveF (Binary cmp icc) (Binary dst src)));
6627   ins_cost(150);
6628 
6629   size(4);
6630   format %{ "FCPYS$cmp $dst,$src" %}
6631   ins_encode %{
6632     __ fcpys($dst$$FloatRegister, $src$$FloatRegister, (AsmCondition)($cmp$$cmpcode));
6633   %}
6634   ins_pipe(int_conditional_float_move);
6635 %}
6636 
6637 // Conditional move
6638 instruct cmovDP_reg(cmpOpP cmp, flagsRegP pcc, regD dst, regD src) %{
6639   match(Set dst (CMoveD (Binary cmp pcc) (Binary dst src)));
6640   ins_cost(150);
6641   size(4);
6642   format %{ "FCPYD$cmp $dst,$src" %}
6643   ins_encode %{
6644     __ fcpyd($dst$$FloatRegister, $src$$FloatRegister, (AsmCondition)($cmp$$cmpcode));
6645   %}
6646   ins_pipe(int_conditional_double_move);
6647 %}
6648 
6649 instruct cmovDI_reg(cmpOp cmp, flagsReg icc, regD dst, regD src) %{
6650   match(Set dst (CMoveD (Binary cmp icc) (Binary dst src)));
6651   ins_cost(150);
6652 
6653   size(4);
6654   format %{ "FCPYD$cmp $dst,$src" %}
6655   ins_encode %{
6656     __ fcpyd($dst$$FloatRegister, $src$$FloatRegister, (AsmCondition)($cmp$$cmpcode));
6657   %}
6658   ins_pipe(int_conditional_double_move);
6659 %}
6660 
6661 instruct cmovDI_reg_EQNELTGE(cmpOp0 cmp, flagsReg_EQNELTGE icc, regD dst, regD src) %{
6662   match(Set dst (CMoveD (Binary cmp icc) (Binary dst src)));
6663   predicate(_kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::eq || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ne || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::lt || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ge);
6664   ins_cost(150);
6665 
6666   size(4);
6667   format %{ "FCPYD$cmp $dst,$src" %}
6668   ins_encode %{
6669     __ fcpyd($dst$$FloatRegister, $src$$FloatRegister, (AsmCondition)($cmp$$cmpcode));
6670   %}
6671   ins_pipe(int_conditional_double_move);
6672 %}
6673 
6674 instruct cmovDIu_reg(cmpOpU cmp, flagsRegU icc, regD dst, regD src) %{
6675   match(Set dst (CMoveD (Binary cmp icc) (Binary dst src)));
6676   ins_cost(150);
6677 
6678   size(4);
6679   format %{ "FCPYD$cmp $dst,$src" %}
6680   ins_encode %{
6681     __ fcpyd($dst$$FloatRegister, $src$$FloatRegister, (AsmCondition)($cmp$$cmpcode));
6682   %}
6683   ins_pipe(int_conditional_double_move);
6684 %}
6685 
6686 // Conditional move
6687 instruct cmovLP_reg(cmpOpP cmp, flagsRegP pcc, iRegL dst, iRegL src) %{
6688   match(Set dst (CMoveL (Binary cmp pcc) (Binary dst src)));
6689   ins_cost(150);
6690 
6691   size(8);
6692   format %{ "MOV$cmp  $dst.lo,$src.lo\t! long\n\t"
6693             "MOV$cmp  $dst.hi,$src.hi" %}
6694   ins_encode %{
6695     __ mov($dst$$Register, $src$$Register, (AsmCondition)($cmp$$cmpcode));
6696     __ mov($dst$$Register->successor(), $src$$Register->successor(), (AsmCondition)($cmp$$cmpcode));
6697   %}
6698   ins_pipe(ialu_reg);
6699 %}
6700 
6701 // TODO: try immLRot2 instead, (0, $con$$constant) becomes
6702 // (hi($con$$constant), lo($con$$constant)) becomes
6703 instruct cmovLP_immRot(cmpOpP cmp, flagsRegP pcc, iRegL dst, immLlowRot src) %{
6704   match(Set dst (CMoveL (Binary cmp pcc) (Binary dst src)));
6705   ins_cost(140);
6706 
6707   size(8);
6708   format %{ "MOV$cmp  $dst.lo,$src\t! long\n\t"
6709             "MOV$cmp  $dst.hi,0" %}
6710   ins_encode %{
6711     __ mov($dst$$Register, $src$$constant, (AsmCondition)($cmp$$cmpcode));
6712     __ mov($dst$$Register->successor(), 0, (AsmCondition)($cmp$$cmpcode));
6713   %}
6714   ins_pipe(ialu_imm);
6715 %}
6716 
6717 instruct cmovLP_imm16(cmpOpP cmp, flagsRegP pcc, iRegL dst, immL16 src) %{
6718   match(Set dst (CMoveL (Binary cmp pcc) (Binary dst src)));
6719   ins_cost(140);
6720 
6721   size(8);
6722   format %{ "MOV$cmp  $dst.lo,$src\t! long\n\t"
6723             "MOV$cmp  $dst.hi,0" %}
6724   ins_encode %{
6725     __ movw($dst$$Register, $src$$constant, (AsmCondition)($cmp$$cmpcode));
6726     __ mov($dst$$Register->successor(), 0, (AsmCondition)($cmp$$cmpcode));
6727   %}
6728   ins_pipe(ialu_imm);
6729 %}
6730 
6731 instruct cmovLI_reg(cmpOp cmp, flagsReg icc, iRegL dst, iRegL src) %{
6732   match(Set dst (CMoveL (Binary cmp icc) (Binary dst src)));
6733   ins_cost(150);
6734 
6735   size(8);
6736   format %{ "MOV$cmp  $dst.lo,$src.lo\t! long\n\t"
6737             "MOV$cmp  $dst.hi,$src.hi" %}
6738   ins_encode %{
6739     __ mov($dst$$Register, $src$$Register, (AsmCondition)($cmp$$cmpcode));
6740     __ mov($dst$$Register->successor(), $src$$Register->successor(), (AsmCondition)($cmp$$cmpcode));
6741   %}
6742   ins_pipe(ialu_reg);
6743 %}
6744 
6745 instruct cmovLI_reg_EQNELTGE(cmpOp0 cmp, flagsReg_EQNELTGE icc, iRegL dst, iRegL src) %{
6746   match(Set dst (CMoveL (Binary cmp icc) (Binary dst src)));
6747   predicate(_kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::eq ||
6748             _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ne ||
6749             _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::lt ||
6750             _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ge);
6751   ins_cost(150);
6752 
6753   size(8);
6754   format %{ "MOV$cmp  $dst.lo,$src.lo\t! long\n\t"
6755             "MOV$cmp  $dst.hi,$src.hi" %}
6756   ins_encode %{
6757     __ mov($dst$$Register, $src$$Register, (AsmCondition)($cmp$$cmpcode));
6758     __ mov($dst$$Register->successor(), $src$$Register->successor(), (AsmCondition)($cmp$$cmpcode));
6759   %}
6760   ins_pipe(ialu_reg);
6761 %}
6762 
6763 // TODO: try immLRot2 instead, (0, $con$$constant) becomes
6764 // (hi($con$$constant), lo($con$$constant)) becomes
6765 instruct cmovLI_immRot(cmpOp cmp, flagsReg icc, iRegL dst, immLlowRot src) %{
6766   match(Set dst (CMoveL (Binary cmp icc) (Binary dst src)));
6767   ins_cost(140);
6768 
6769   size(8);
6770   format %{ "MOV$cmp  $dst.lo,$src\t! long\n\t"
6771             "MOV$cmp  $dst.hi,0" %}
6772   ins_encode %{
6773     __ mov($dst$$Register, $src$$constant, (AsmCondition)($cmp$$cmpcode));
6774     __ mov($dst$$Register->successor(), 0, (AsmCondition)($cmp$$cmpcode));
6775   %}
6776   ins_pipe(ialu_imm);
6777 %}
6778 
6779 // TODO: try immLRot2 instead, (0, $con$$constant) becomes
6780 // (hi($con$$constant), lo($con$$constant)) becomes
6781 instruct cmovLI_immRot_EQNELTGE(cmpOp0 cmp, flagsReg_EQNELTGE icc, iRegL dst, immLlowRot src) %{
6782   match(Set dst (CMoveL (Binary cmp icc) (Binary dst src)));
6783   predicate(_kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::eq ||
6784             _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ne ||
6785             _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::lt ||
6786             _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ge);
6787   ins_cost(140);
6788 
6789   size(8);
6790   format %{ "MOV$cmp  $dst.lo,$src\t! long\n\t"
6791             "MOV$cmp  $dst.hi,0" %}
6792   ins_encode %{
6793     __ mov($dst$$Register, $src$$constant, (AsmCondition)($cmp$$cmpcode));
6794     __ mov($dst$$Register->successor(), 0, (AsmCondition)($cmp$$cmpcode));
6795   %}
6796   ins_pipe(ialu_imm);
6797 %}
6798 
6799 instruct cmovLI_imm16(cmpOp cmp, flagsReg icc, iRegL dst, immL16 src) %{
6800   match(Set dst (CMoveL (Binary cmp icc) (Binary dst src)));
6801   ins_cost(140);
6802 
6803   size(8);
6804   format %{ "MOV$cmp  $dst.lo,$src\t! long\n\t"
6805             "MOV$cmp  $dst.hi,0" %}
6806   ins_encode %{
6807     __ movw($dst$$Register, $src$$constant, (AsmCondition)($cmp$$cmpcode));
6808     __ movw($dst$$Register->successor(), 0, (AsmCondition)($cmp$$cmpcode));
6809   %}
6810   ins_pipe(ialu_imm);
6811 %}
6812 
6813 instruct cmovLI_imm16_EQNELTGE(cmpOp0 cmp, flagsReg_EQNELTGE icc, iRegL dst, immL16 src) %{
6814   match(Set dst (CMoveL (Binary cmp icc) (Binary dst src)));
6815   predicate(_kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::eq ||
6816             _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ne ||
6817             _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::lt ||
6818             _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ge);
6819   ins_cost(140);
6820 
6821   size(8);
6822   format %{ "MOV$cmp  $dst.lo,$src\t! long\n\t"
6823             "MOV$cmp  $dst.hi,0" %}
6824   ins_encode %{
6825     __ movw($dst$$Register, $src$$constant, (AsmCondition)($cmp$$cmpcode));
6826     __ movw($dst$$Register->successor(), 0, (AsmCondition)($cmp$$cmpcode));
6827   %}
6828   ins_pipe(ialu_imm);
6829 %}
6830 
6831 instruct cmovLIu_reg(cmpOpU cmp, flagsRegU icc, iRegL dst, iRegL src) %{
6832   match(Set dst (CMoveL (Binary cmp icc) (Binary dst src)));
6833   ins_cost(150);
6834 
6835   size(8);
6836   format %{ "MOV$cmp  $dst.lo,$src.lo\t! long\n\t"
6837             "MOV$cmp  $dst.hi,$src.hi" %}
6838   ins_encode %{
6839     __ mov($dst$$Register, $src$$Register, (AsmCondition)($cmp$$cmpcode));
6840     __ mov($dst$$Register->successor(), $src$$Register->successor(), (AsmCondition)($cmp$$cmpcode));
6841   %}
6842   ins_pipe(ialu_reg);
6843 %}
6844 #endif // !AARCH64
6845 
6846 
6847 //----------OS and Locking Instructions----------------------------------------
6848 
6849 // This name is KNOWN by the ADLC and cannot be changed.
6850 // The ADLC forces a 'TypeRawPtr::BOTTOM' output type
6851 // for this guy.
6852 instruct tlsLoadP(RthreadRegP dst) %{
6853   match(Set dst (ThreadLocal));
6854 
6855   size(0);
6856   ins_cost(0);
6857   format %{ "! TLS is in $dst" %}
6858   ins_encode( /*empty encoding*/ );
6859   ins_pipe(ialu_none);
6860 %}
6861 
6862 instruct checkCastPP( iRegP dst ) %{
6863   match(Set dst (CheckCastPP dst));
6864 
6865   size(0);
6866   format %{ "! checkcastPP of $dst" %}
6867   ins_encode( /*empty encoding*/ );
6868   ins_pipe(empty);
6869 %}
6870 
6871 
6872 instruct castPP( iRegP dst ) %{
6873   match(Set dst (CastPP dst));
6874   format %{ "! castPP of $dst" %}
6875   ins_encode( /*empty encoding*/ );
6876   ins_pipe(empty);
6877 %}
6878 
6879 instruct castII( iRegI dst ) %{
6880   match(Set dst (CastII dst));
6881   format %{ "! castII of $dst" %}
6882   ins_encode( /*empty encoding*/ );
6883   ins_cost(0);
6884   ins_pipe(empty);
6885 %}
6886 
6887 //----------Arithmetic Instructions--------------------------------------------
6888 // Addition Instructions
6889 // Register Addition
6890 instruct addI_reg_reg(iRegI dst, iRegI src1, iRegI src2) %{
6891   match(Set dst (AddI src1 src2));
6892 
6893   size(4);
6894   format %{ "add_32 $dst,$src1,$src2\t! int" %}
6895   ins_encode %{
6896     __ add_32($dst$$Register, $src1$$Register, $src2$$Register);
6897   %}
6898   ins_pipe(ialu_reg_reg);
6899 %}
6900 
6901 #ifndef AARCH64
6902 instruct addshlI_reg_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3) %{
6903   match(Set dst (AddI (LShiftI src1 src2) src3));
6904 
6905   size(4);
6906   format %{ "add_32 $dst,$src3,$src1<<$src2\t! int" %}
6907   ins_encode %{
6908     __ add_32($dst$$Register, $src3$$Register, AsmOperand($src1$$Register, lsl, $src2$$Register));
6909   %}
6910   ins_pipe(ialu_reg_reg);
6911 %}
6912 #endif
6913 
6914 #ifdef AARCH64
6915 #ifdef TODO
6916 instruct addshlL_reg_imm_reg(iRegL dst, iRegL src1, immU6 src2, iRegL src3) %{
6917   match(Set dst (AddL (LShiftL src1 src2) src3));
6918 
6919   size(4);
6920   format %{ "ADD    $dst,$src3,$src1<<$src2\t! long" %}
6921   ins_encode %{
6922     __ add($dst$$Register, $src3$$Register, AsmOperand($src1$$Register, lsl, $src2$$constant));
6923   %}
6924   ins_pipe(ialu_reg_reg);
6925 %}
6926 #endif
6927 #endif
6928 
6929 instruct addshlI_reg_imm_reg(iRegI dst, iRegI src1, immU5 src2, iRegI src3) %{
6930   match(Set dst (AddI (LShiftI src1 src2) src3));
6931 
6932   size(4);
6933   format %{ "add_32 $dst,$src3,$src1<<$src2\t! int" %}
6934   ins_encode %{
6935     __ add_32($dst$$Register, $src3$$Register, AsmOperand($src1$$Register, lsl, $src2$$constant));
6936   %}
6937   ins_pipe(ialu_reg_reg);
6938 %}
6939 
6940 #ifndef AARCH64
6941 instruct addsarI_reg_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3) %{
6942   match(Set dst (AddI (RShiftI src1 src2) src3));
6943 
6944   size(4);
6945   format %{ "add_32 $dst,$src3,$src1>>$src2\t! int" %}
6946   ins_encode %{
6947     __ add_32($dst$$Register, $src3$$Register, AsmOperand($src1$$Register, asr, $src2$$Register));
6948   %}
6949   ins_pipe(ialu_reg_reg);
6950 %}
6951 #endif
6952 
6953 instruct addsarI_reg_imm_reg(iRegI dst, iRegI src1, immU5 src2, iRegI src3) %{
6954   match(Set dst (AddI (RShiftI src1 src2) src3));
6955 
6956   size(4);
6957   format %{ "add_32 $dst,$src3,$src1>>$src2\t! int" %}
6958   ins_encode %{
6959     __ add_32($dst$$Register, $src3$$Register, AsmOperand($src1$$Register, asr, $src2$$constant));
6960   %}
6961   ins_pipe(ialu_reg_reg);
6962 %}
6963 
6964 #ifndef AARCH64
6965 instruct addshrI_reg_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3) %{
6966   match(Set dst (AddI (URShiftI src1 src2) src3));
6967 
6968   size(4);
6969   format %{ "add_32 $dst,$src3,$src1>>>$src2\t! int" %}
6970   ins_encode %{
6971     __ add_32($dst$$Register, $src3$$Register, AsmOperand($src1$$Register, lsr, $src2$$Register));
6972   %}
6973   ins_pipe(ialu_reg_reg);
6974 %}
6975 #endif
6976 
6977 instruct addshrI_reg_imm_reg(iRegI dst, iRegI src1, immU5 src2, iRegI src3) %{
6978   match(Set dst (AddI (URShiftI src1 src2) src3));
6979 
6980   size(4);
6981   format %{ "add_32 $dst,$src3,$src1>>>$src2\t! int" %}
6982   ins_encode %{
6983     __ add_32($dst$$Register, $src3$$Register, AsmOperand($src1$$Register, lsr, $src2$$constant));
6984   %}
6985   ins_pipe(ialu_reg_reg);
6986 %}
6987 
6988 // Immediate Addition
6989 instruct addI_reg_aimmI(iRegI dst, iRegI src1, aimmI src2) %{
6990   match(Set dst (AddI src1 src2));
6991 
6992   size(4);
6993   format %{ "add_32 $dst,$src1,$src2\t! int" %}
6994   ins_encode %{
6995     __ add_32($dst$$Register, $src1$$Register, $src2$$constant);
6996   %}
6997   ins_pipe(ialu_reg_imm);
6998 %}
6999 
7000 // Pointer Register Addition
7001 instruct addP_reg_reg(iRegP dst, iRegP src1, iRegX src2) %{
7002   match(Set dst (AddP src1 src2));
7003 
7004   size(4);
7005   format %{ "ADD    $dst,$src1,$src2\t! ptr" %}
7006   ins_encode %{
7007     __ add($dst$$Register, $src1$$Register, $src2$$Register);
7008   %}
7009   ins_pipe(ialu_reg_reg);
7010 %}
7011 
7012 #ifdef AARCH64
7013 // unshifted I2L operand
7014 operand unshiftedI2L(iRegI src2) %{
7015 //constraint(ALLOC_IN_RC(sp_ptr_reg));
7016   match(ConvI2L src2);
7017 
7018   op_cost(1);
7019   format %{ "$src2.w" %}
7020   interface(MEMORY_INTER) %{
7021     base($src2);
7022     index(0xff);
7023     scale(0x0);
7024     disp(0x0);
7025   %}
7026 %}
7027 
7028 // shifted I2L operand
7029 operand shiftedI2L(iRegI src2, immI_0_4 src3) %{
7030 //constraint(ALLOC_IN_RC(sp_ptr_reg));
7031   match(LShiftX (ConvI2L src2) src3);
7032 
7033   op_cost(1);
7034   format %{ "$src2.w << $src3" %}
7035   interface(MEMORY_INTER) %{
7036     base($src2);
7037     index(0xff);
7038     scale($src3);
7039     disp(0x0);
7040   %}
7041 %}
7042 
7043 opclass shiftedRegI(shiftedI2L, unshiftedI2L);
7044 
7045 instruct shlL_reg_regI(iRegL dst, iRegI src1, immU6 src2) %{
7046   match(Set dst (LShiftL (ConvI2L src1) src2));
7047 
7048   size(4);
7049   format %{ "LSL    $dst,$src1.w,$src2\t! ptr" %}
7050   ins_encode %{
7051     int c = $src2$$constant;
7052     int r = 64 - c;
7053     int s = 31;
7054     if (s >= r) {
7055       s = r - 1;
7056     }
7057     __ sbfm($dst$$Register, $src1$$Register, r, s);
7058   %}
7059   ins_pipe(ialu_reg_reg);
7060 %}
7061 
7062 instruct addP_reg_regI(iRegP dst, iRegP src1, shiftedRegI src2) %{
7063   match(Set dst (AddP src1 src2));
7064 
7065   ins_cost(DEFAULT_COST * 3/2);
7066   size(4);
7067   format %{ "ADD    $dst,$src1,$src2, sxtw\t! ptr" %}
7068   ins_encode %{
7069     Register base = reg_to_register_object($src2$$base);
7070     __ add($dst$$Register, $src1$$Register, base, ex_sxtw, $src2$$scale);
7071   %}
7072   ins_pipe(ialu_reg_reg);
7073 %}
7074 #endif
7075 
7076 // shifted iRegX operand
7077 operand shiftedX(iRegX src2, shimmX src3) %{
7078 //constraint(ALLOC_IN_RC(sp_ptr_reg));
7079   match(LShiftX src2 src3);
7080 
7081   op_cost(1);
7082   format %{ "$src2 << $src3" %}
7083   interface(MEMORY_INTER) %{
7084     base($src2);
7085     index(0xff);
7086     scale($src3);
7087     disp(0x0);
7088   %}
7089 %}
7090 
7091 instruct addshlP_reg_reg_imm(iRegP dst, iRegP src1, shiftedX src2) %{
7092   match(Set dst (AddP src1 src2));
7093 
7094   ins_cost(DEFAULT_COST * 3/2);
7095   size(4);
7096   format %{ "ADD    $dst,$src1,$src2\t! ptr" %}
7097   ins_encode %{
7098     Register base = reg_to_register_object($src2$$base);
7099     __ add($dst$$Register, $src1$$Register, AsmOperand(base, lsl, $src2$$scale));
7100   %}
7101   ins_pipe(ialu_reg_reg);
7102 %}
7103 
7104 // Pointer Immediate Addition
7105 instruct addP_reg_aimmX(iRegP dst, iRegP src1, aimmX src2) %{
7106   match(Set dst (AddP src1 src2));
7107 
7108   size(4);
7109   format %{ "ADD    $dst,$src1,$src2\t! ptr" %}
7110   ins_encode %{
7111     __ add($dst$$Register, $src1$$Register, $src2$$constant);
7112   %}
7113   ins_pipe(ialu_reg_imm);
7114 %}
7115 
7116 // Long Addition
7117 #ifdef AARCH64
7118 instruct addL_reg_reg(iRegL dst, iRegL src1, iRegL src2) %{
7119   match(Set dst (AddL src1 src2));
7120   size(4);
7121   format %{ "ADD     $dst,$src1,$src2\t! long" %}
7122   ins_encode %{
7123     __ add($dst$$Register, $src1$$Register, $src2$$Register);
7124   %}
7125   ins_pipe(ialu_reg_reg);
7126 %}
7127 
7128 instruct addL_reg_regI(iRegL dst, iRegL src1, shiftedRegI src2) %{
7129   match(Set dst (AddL src1 src2));
7130 
7131   ins_cost(DEFAULT_COST * 3/2);
7132   size(4);
7133   format %{ "ADD    $dst,$src1,$src2, sxtw\t! long" %}
7134   ins_encode %{
7135     Register base = reg_to_register_object($src2$$base);
7136     __ add($dst$$Register, $src1$$Register, base, ex_sxtw, $src2$$scale);
7137   %}
7138   ins_pipe(ialu_reg_reg);
7139 %}
7140 #else
7141 instruct addL_reg_reg(iRegL dst, iRegL src1, iRegL src2, flagsReg ccr) %{
7142   match(Set dst (AddL src1 src2));
7143   effect(KILL ccr);
7144   size(8);
7145   format %{ "ADDS    $dst.lo,$src1.lo,$src2.lo\t! long\n\t"
7146             "ADC     $dst.hi,$src1.hi,$src2.hi" %}
7147   ins_encode %{
7148     __ adds($dst$$Register, $src1$$Register, $src2$$Register);
7149     __ adc($dst$$Register->successor(), $src1$$Register->successor(), $src2$$Register->successor());
7150   %}
7151   ins_pipe(ialu_reg_reg);
7152 %}
7153 #endif
7154 
7155 #ifdef AARCH64
7156 // Immediate Addition
7157 instruct addL_reg_aimm(iRegL dst, iRegL src1, aimmL src2) %{
7158   match(Set dst (AddL src1 src2));
7159 
7160   size(4);
7161   format %{ "ADD    $dst,$src1,$src2\t! long" %}
7162   ins_encode %{
7163     __ add($dst$$Register, $src1$$Register, $src2$$constant);
7164   %}
7165   ins_pipe(ialu_reg_imm);
7166 %}
7167 
7168 instruct addL_reg_immLneg(iRegL dst, iRegL src1, aimmLneg src2) %{
7169   match(Set dst (SubL src1 src2));
7170 
7171   size(4);
7172   format %{ "ADD    $dst,$src1,-($src2)\t! long" %}
7173   ins_encode %{
7174     __ add($dst$$Register, $src1$$Register, -$src2$$constant);
7175   %}
7176   ins_pipe(ialu_reg_imm);
7177 %}
7178 #else
7179 // TODO
7180 #endif
7181 
7182 #ifndef AARCH64
7183 // TODO: try immLRot2 instead, (0, $con$$constant) becomes
7184 // (hi($con$$constant), lo($con$$constant)) becomes
7185 instruct addL_reg_immRot(iRegL dst, iRegL src1, immLlowRot con, flagsReg ccr) %{
7186   match(Set dst (AddL src1 con));
7187   effect(KILL ccr);
7188   size(8);
7189   format %{ "ADDS    $dst.lo,$src1.lo,$con\t! long\n\t"
7190             "ADC     $dst.hi,$src1.hi,0" %}
7191   ins_encode %{
7192     __ adds($dst$$Register, $src1$$Register, $con$$constant);
7193     __ adc($dst$$Register->successor(), $src1$$Register->successor(), 0);
7194   %}
7195   ins_pipe(ialu_reg_imm);
7196 %}
7197 #endif
7198 
7199 //----------Conditional_store--------------------------------------------------
7200 // Conditional-store of the updated heap-top.
7201 // Used during allocation of the shared heap.
7202 // Sets flags (EQ) on success.
7203 
7204 // TODO: optimize out barriers with AArch64 load-acquire/store-release
7205 // LoadP-locked.
7206 instruct loadPLocked(iRegP dst, memoryex mem) %{
7207   match(Set dst (LoadPLocked mem));
7208   size(4);
7209   format %{ "LDREX  $dst,$mem" %}
7210   ins_encode %{
7211 #ifdef AARCH64
7212     Register base = reg_to_register_object($mem$$base);
7213     __ ldxr($dst$$Register, base);
7214 #else
7215     __ ldrex($dst$$Register,$mem$$Address);
7216 #endif
7217   %}
7218   ins_pipe(iload_mem);
7219 %}
7220 
7221 instruct storePConditional( memoryex heap_top_ptr, iRegP oldval, iRegP newval, iRegI tmp, flagsRegP pcc ) %{
7222   predicate(_kids[1]->_kids[0]->_leaf->Opcode() == Op_LoadPLocked); // only works in conjunction with a LoadPLocked node
7223   match(Set pcc (StorePConditional heap_top_ptr (Binary oldval newval)));
7224   effect( TEMP tmp );
7225   size(8);
7226   format %{ "STREX  $tmp,$newval,$heap_top_ptr\n\t"
7227             "CMP    $tmp, 0" %}
7228   ins_encode %{
7229 #ifdef AARCH64
7230     Register base = reg_to_register_object($heap_top_ptr$$base);
7231     __ stxr($tmp$$Register, $newval$$Register, base);
7232 #else
7233     __ strex($tmp$$Register, $newval$$Register, $heap_top_ptr$$Address);
7234 #endif
7235     __ cmp($tmp$$Register, 0);
7236   %}
7237   ins_pipe( long_memory_op );
7238 %}
7239 
7240 // Conditional-store of an intx value.
7241 instruct storeXConditional( memoryex mem, iRegX oldval, iRegX newval, iRegX tmp, flagsReg icc ) %{
7242 #ifdef AARCH64
7243   match(Set icc (StoreLConditional mem (Binary oldval newval)));
7244   effect( TEMP tmp );
7245   size(28);
7246   format %{ "loop:\n\t"
7247             "LDXR     $tmp, $mem\t! If $oldval==[$mem] Then store $newval into [$mem], DOESN'T set $newval=[$mem] in any case\n\t"
7248             "SUBS     $tmp, $tmp, $oldval\n\t"
7249             "B.ne     done\n\t"
7250             "STXR     $tmp, $newval, $mem\n\t"
7251             "CBNZ_w   $tmp, loop\n\t"
7252             "CMP      $tmp, 0\n\t"
7253             "done:\n\t"
7254             "membar   LoadStore|LoadLoad" %}
7255 #else
7256   match(Set icc (StoreIConditional mem (Binary oldval newval)));
7257   effect( TEMP tmp );
7258   size(28);
7259   format %{ "loop: \n\t"
7260             "LDREX    $tmp, $mem\t! If $oldval==[$mem] Then store $newval into [$mem], DOESN'T set $newval=[$mem] in any case\n\t"
7261             "XORS     $tmp,$tmp, $oldval\n\t"
7262             "STREX.eq $tmp, $newval, $mem\n\t"
7263             "CMP.eq   $tmp, 1 \n\t"
7264             "B.eq     loop \n\t"
7265             "TEQ      $tmp, 0\n\t"
7266             "membar   LoadStore|LoadLoad" %}
7267 #endif
7268   ins_encode %{
7269     Label loop;
7270     __ bind(loop);
7271 #ifdef AARCH64
7272 // FIXME: use load-acquire/store-release, remove membar?
7273     Label done;
7274     Register base = reg_to_register_object($mem$$base);
7275     __ ldxr($tmp$$Register, base);
7276     __ subs($tmp$$Register, $tmp$$Register, $oldval$$Register);
7277     __ b(done, ne);
7278     __ stxr($tmp$$Register, $newval$$Register, base);
7279     __ cbnz_w($tmp$$Register, loop);
7280     __ cmp($tmp$$Register, 0);
7281     __ bind(done);
7282 #else
7283     __ ldrex($tmp$$Register, $mem$$Address);
7284     __ eors($tmp$$Register, $tmp$$Register, $oldval$$Register);
7285     __ strex($tmp$$Register, $newval$$Register, $mem$$Address, eq);
7286     __ cmp($tmp$$Register, 1, eq);
7287     __ b(loop, eq);
7288     __ teq($tmp$$Register, 0);
7289 #endif
7290     // used by biased locking only. Requires a membar.
7291     __ membar(MacroAssembler::Membar_mask_bits(MacroAssembler::LoadStore | MacroAssembler::LoadLoad), noreg);
7292   %}
7293   ins_pipe( long_memory_op );
7294 %}
7295 
7296 // No flag versions for CompareAndSwap{P,I,L} because matcher can't match them
7297 
7298 #ifdef AARCH64
7299 // TODO: if combined with membar, elide membar and use
7300 // load-acquire/store-release if appropriate
7301 instruct compareAndSwapL_bool(memoryex mem, iRegL oldval, iRegL newval, iRegI res, iRegI tmp, flagsReg ccr) %{
7302   match(Set res (CompareAndSwapL mem (Binary oldval newval)));
7303   effect( KILL ccr, TEMP tmp);
7304   size(24);
7305   format %{ "loop:\n\t"
7306             "LDXR     $tmp, $mem\t! If $oldval==[$mem] Then store $newval into [$mem]\n\t"
7307             "CMP      $tmp, $oldval\n\t"
7308             "B.ne     done\n\t"
7309             "STXR     $tmp, $newval, $mem\n\t"
7310             "CBNZ_w   $tmp, loop\n\t"
7311             "done:\n\t"
7312             "CSET_w   $res, eq" %}
7313   ins_encode %{
7314     Register base = reg_to_register_object($mem$$base);
7315     Label loop, done;
7316     __ bind(loop);
7317     __ ldxr($tmp$$Register, base);
7318     __ cmp($tmp$$Register, $oldval$$Register);
7319     __ b(done, ne);
7320     __ stxr($tmp$$Register, $newval$$Register, base);
7321     __ cbnz_w($tmp$$Register, loop);
7322     __ bind(done);
7323     __ cset_w($res$$Register, eq);
7324   %}
7325   ins_pipe( long_memory_op );
7326 %}
7327 
7328 instruct compareAndSwapI_bool(memoryex mem, iRegI oldval, iRegI newval, iRegI res, iRegI tmp, flagsReg ccr) %{
7329   match(Set res (CompareAndSwapI mem (Binary oldval newval)));
7330   effect( KILL ccr, TEMP tmp);
7331   size(24);
7332   format %{ "loop:\n\t"
7333             "LDXR_w   $tmp, $mem\t! If $oldval==[$mem] Then store $newval into [$mem]\n\t"
7334             "CMP_w    $tmp, $oldval\n\t"
7335             "B.ne     done\n\t"
7336             "STXR_w   $tmp, $newval, $mem\n\t"
7337             "CBNZ_w   $tmp, loop\n\t"
7338             "done:\n\t"
7339             "CSET_w   $res, eq" %}
7340   ins_encode %{
7341     Register base = reg_to_register_object($mem$$base);
7342     Label loop, done;
7343     __ bind(loop);
7344     __ ldxr_w($tmp$$Register, base);
7345     __ cmp_w($tmp$$Register, $oldval$$Register);
7346     __ b(done, ne);
7347     __ stxr_w($tmp$$Register, $newval$$Register,  base);
7348     __ cbnz_w($tmp$$Register, loop);
7349     __ bind(done);
7350     __ cset_w($res$$Register, eq);
7351   %}
7352   ins_pipe( long_memory_op );
7353 %}
7354 
7355 // tmp must use iRegI instead of iRegN until 8051805 is fixed.
7356 instruct compareAndSwapN_bool(memoryex mem, iRegN oldval, iRegN newval, iRegI res, iRegI tmp, flagsReg ccr) %{
7357   match(Set res (CompareAndSwapN mem (Binary oldval newval)));
7358   effect( KILL ccr, TEMP tmp);
7359   size(24);
7360   format %{ "loop:\n\t"
7361             "LDXR_w   $tmp, $mem\t! If $oldval==[$mem] Then store $newval into [$mem]\n\t"
7362             "CMP_w    $tmp, $oldval\n\t"
7363             "B.ne     done\n\t"
7364             "STXR_w   $tmp, $newval, $mem\n\t"
7365             "CBNZ_w   $tmp, loop\n\t"
7366             "done:\n\t"
7367             "CSET_w   $res, eq" %}
7368   ins_encode %{
7369     Register base = reg_to_register_object($mem$$base);
7370     Label loop, done;
7371     __ bind(loop);
7372     __ ldxr_w($tmp$$Register, base);
7373     __ cmp_w($tmp$$Register, $oldval$$Register);
7374     __ b(done, ne);
7375     __ stxr_w($tmp$$Register, $newval$$Register,  base);
7376     __ cbnz_w($tmp$$Register, loop);
7377     __ bind(done);
7378     __ cset_w($res$$Register, eq);
7379   %}
7380   ins_pipe( long_memory_op );
7381 %}
7382 
7383 instruct compareAndSwapP_bool(memoryex mem, iRegP oldval, iRegP newval, iRegI res, iRegI tmp, flagsReg ccr) %{
7384   match(Set res (CompareAndSwapP mem (Binary oldval newval)));
7385   effect( KILL ccr, TEMP tmp);
7386   size(24);
7387   format %{ "loop:\n\t"
7388             "LDXR     $tmp, $mem\t! If $oldval==[$mem] Then store $newval into [$mem]\n\t"
7389             "CMP      $tmp, $oldval\n\t"
7390             "B.ne     done\n\t"
7391             "STXR     $tmp, $newval, $mem\n\t"
7392             "CBNZ_w   $tmp, loop\n\t"
7393             "done:\n\t"
7394             "CSET_w   $res, eq" %}
7395   ins_encode %{
7396     Register base = reg_to_register_object($mem$$base);
7397     Label loop, done;
7398     __ bind(loop);
7399     __ ldxr($tmp$$Register, base);
7400     __ cmp($tmp$$Register, $oldval$$Register);
7401     __ b(done, ne);
7402     __ stxr($tmp$$Register, $newval$$Register,  base);
7403     __ cbnz_w($tmp$$Register, loop);
7404     __ bind(done);
7405     __ cset_w($res$$Register, eq);
7406   %}
7407   ins_pipe( long_memory_op );
7408 %}
7409 #else // !AARCH64
7410 instruct compareAndSwapL_bool(memoryex mem, iRegL oldval, iRegLd newval, iRegI res, iRegLd tmp, flagsReg ccr ) %{
7411   match(Set res (CompareAndSwapL mem (Binary oldval newval)));
7412   effect( KILL ccr, TEMP tmp);
7413   size(32);
7414   format %{ "loop: \n\t"
7415             "LDREXD   $tmp, $mem\t! If $oldval==[$mem] Then store $newval into [$mem]\n\t"
7416             "CMP      $tmp.lo, $oldval.lo\n\t"
7417             "CMP.eq   $tmp.hi, $oldval.hi\n\t"
7418             "STREXD.eq $tmp, $newval, $mem\n\t"
7419             "MOV.ne   $tmp, 0 \n\t"
7420             "XORS.eq  $tmp,$tmp, 1 \n\t"
7421             "B.eq     loop \n\t"
7422             "MOV      $res, $tmp" %}
7423   ins_encode %{
7424     Label loop;
7425     __ bind(loop);
7426     __ ldrexd($tmp$$Register, $mem$$Address);
7427     __ cmp($tmp$$Register, $oldval$$Register);
7428     __ cmp($tmp$$Register->successor(), $oldval$$Register->successor(), eq);
7429     __ strexd($tmp$$Register, $newval$$Register, $mem$$Address, eq);
7430     __ mov($tmp$$Register, 0, ne);
7431     __ eors($tmp$$Register, $tmp$$Register, 1, eq);
7432     __ b(loop, eq);
7433     __ mov($res$$Register, $tmp$$Register);
7434   %}
7435   ins_pipe( long_memory_op );
7436 %}
7437 
7438 
7439 instruct compareAndSwapI_bool(memoryex mem, iRegI oldval, iRegI newval, iRegI res, iRegI tmp, flagsReg ccr ) %{
7440   match(Set res (CompareAndSwapI mem (Binary oldval newval)));
7441   effect( KILL ccr, TEMP tmp);
7442   size(28);
7443   format %{ "loop: \n\t"
7444             "LDREX    $tmp, $mem\t! If $oldval==[$mem] Then store $newval into [$mem]\n\t"
7445             "CMP      $tmp, $oldval\n\t"
7446             "STREX.eq $tmp, $newval, $mem\n\t"
7447             "MOV.ne   $tmp, 0 \n\t"
7448             "XORS.eq  $tmp,$tmp, 1 \n\t"
7449             "B.eq     loop \n\t"
7450             "MOV      $res, $tmp" %}
7451 
7452   ins_encode %{
7453     Label loop;
7454     __ bind(loop);
7455     __ ldrex($tmp$$Register,$mem$$Address);
7456     __ cmp($tmp$$Register, $oldval$$Register);
7457     __ strex($tmp$$Register, $newval$$Register, $mem$$Address, eq);
7458     __ mov($tmp$$Register, 0, ne);
7459     __ eors($tmp$$Register, $tmp$$Register, 1, eq);
7460     __ b(loop, eq);
7461     __ mov($res$$Register, $tmp$$Register);
7462   %}
7463   ins_pipe( long_memory_op );
7464 %}
7465 
7466 instruct compareAndSwapP_bool(memoryex mem, iRegP oldval, iRegP newval, iRegI res, iRegI tmp, flagsReg ccr ) %{
7467   match(Set res (CompareAndSwapP mem (Binary oldval newval)));
7468   effect( KILL ccr, TEMP tmp);
7469   size(28);
7470   format %{ "loop: \n\t"
7471             "LDREX    $tmp, $mem\t! If $oldval==[$mem] Then store $newval into [$mem]\n\t"
7472             "CMP      $tmp, $oldval\n\t"
7473             "STREX.eq $tmp, $newval, $mem\n\t"
7474             "MOV.ne   $tmp, 0 \n\t"
7475             "EORS.eq  $tmp,$tmp, 1 \n\t"
7476             "B.eq     loop \n\t"
7477             "MOV      $res, $tmp" %}
7478 
7479   ins_encode %{
7480     Label loop;
7481     __ bind(loop);
7482     __ ldrex($tmp$$Register,$mem$$Address);
7483     __ cmp($tmp$$Register, $oldval$$Register);
7484     __ strex($tmp$$Register, $newval$$Register, $mem$$Address, eq);
7485     __ mov($tmp$$Register, 0, ne);
7486     __ eors($tmp$$Register, $tmp$$Register, 1, eq);
7487     __ b(loop, eq);
7488     __ mov($res$$Register, $tmp$$Register);
7489   %}
7490   ins_pipe( long_memory_op );
7491 %}
7492 #endif // !AARCH64
7493 
7494 #ifdef AARCH64
7495 instruct xaddI_aimmI_no_res(memoryex mem, aimmI add, Universe dummy, iRegI tmp1, iRegI tmp2) %{
7496   predicate(n->as_LoadStore()->result_not_used());
7497   match(Set dummy (GetAndAddI mem add));
7498   effect(TEMP tmp1, TEMP tmp2);
7499   size(16);
7500   format %{ "loop:\n\t"
7501             "LDXR_w   $tmp1, $mem\n\t"
7502             "ADD_w    $tmp1, $tmp1, $add\n\t"
7503             "STXR_w   $tmp2, $tmp1, $mem\n\t"
7504             "CBNZ_w   $tmp2, loop" %}
7505 
7506   ins_encode %{
7507     Label loop;
7508     Register base = reg_to_register_object($mem$$base);
7509     __ bind(loop);
7510     __ ldxr_w($tmp1$$Register, base);
7511     __ add_w($tmp1$$Register, $tmp1$$Register, $add$$constant);
7512     __ stxr_w($tmp2$$Register, $tmp1$$Register, base);
7513     __ cbnz_w($tmp2$$Register, loop);
7514   %}
7515   ins_pipe( long_memory_op );
7516 %}
7517 #else
7518 instruct xaddI_aimmI_no_res(memoryex mem, aimmI add, Universe dummy, iRegI tmp1, iRegI tmp2, flagsReg ccr) %{
7519   predicate(n->as_LoadStore()->result_not_used());
7520   match(Set dummy (GetAndAddI mem add));
7521   effect(KILL ccr, TEMP tmp1, TEMP tmp2);
7522   size(20);
7523   format %{ "loop: \n\t"
7524             "LDREX    $tmp1, $mem\n\t"
7525             "ADD      $tmp1, $tmp1, $add\n\t"
7526             "STREX    $tmp2, $tmp1, $mem\n\t"
7527             "CMP      $tmp2, 0 \n\t"
7528             "B.ne     loop \n\t" %}
7529 
7530   ins_encode %{
7531     Label loop;
7532     __ bind(loop);
7533     __ ldrex($tmp1$$Register,$mem$$Address);
7534     __ add($tmp1$$Register, $tmp1$$Register, $add$$constant);
7535     __ strex($tmp2$$Register, $tmp1$$Register, $mem$$Address);
7536     __ cmp($tmp2$$Register, 0);
7537     __ b(loop, ne);
7538   %}
7539   ins_pipe( long_memory_op );
7540 %}
7541 #endif
7542 
7543 #ifdef AARCH64
7544 instruct xaddI_reg_no_res(memoryex mem, iRegI add, Universe dummy, iRegI tmp1, iRegI tmp2) %{
7545   predicate(n->as_LoadStore()->result_not_used());
7546   match(Set dummy (GetAndAddI mem add));
7547   effect(TEMP tmp1, TEMP tmp2);
7548   size(16);
7549   format %{ "loop:\n\t"
7550             "LDXR_w   $tmp1, $mem\n\t"
7551             "ADD_w    $tmp1, $tmp1, $add\n\t"
7552             "STXR_w   $tmp2, $tmp1, $mem\n\t"
7553             "CBNZ_w   $tmp2, loop" %}
7554 
7555   ins_encode %{
7556     Label loop;
7557     Register base = reg_to_register_object($mem$$base);
7558     __ bind(loop);
7559     __ ldxr_w($tmp1$$Register, base);
7560     __ add_w($tmp1$$Register, $tmp1$$Register, $add$$Register);
7561     __ stxr_w($tmp2$$Register, $tmp1$$Register, base);
7562     __ cbnz_w($tmp2$$Register, loop);
7563   %}
7564   ins_pipe( long_memory_op );
7565 %}
7566 #else
7567 instruct xaddI_reg_no_res(memoryex mem, iRegI add, Universe dummy, iRegI tmp1, iRegI tmp2, flagsReg ccr) %{
7568   predicate(n->as_LoadStore()->result_not_used());
7569   match(Set dummy (GetAndAddI mem add));
7570   effect(KILL ccr, TEMP tmp1, TEMP tmp2);
7571   size(20);
7572   format %{ "loop: \n\t"
7573             "LDREX    $tmp1, $mem\n\t"
7574             "ADD      $tmp1, $tmp1, $add\n\t"
7575             "STREX    $tmp2, $tmp1, $mem\n\t"
7576             "CMP      $tmp2, 0 \n\t"
7577             "B.ne     loop \n\t" %}
7578 
7579   ins_encode %{
7580     Label loop;
7581     __ bind(loop);
7582     __ ldrex($tmp1$$Register,$mem$$Address);
7583     __ add($tmp1$$Register, $tmp1$$Register, $add$$Register);
7584     __ strex($tmp2$$Register, $tmp1$$Register, $mem$$Address);
7585     __ cmp($tmp2$$Register, 0);
7586     __ b(loop, ne);
7587   %}
7588   ins_pipe( long_memory_op );
7589 %}
7590 #endif
7591 
7592 #ifdef AARCH64
7593 instruct xaddI_aimmI(memoryex mem, aimmI add, iRegI res, iRegI tmp1, iRegI tmp2) %{
7594   match(Set res (GetAndAddI mem add));
7595   effect(TEMP tmp1, TEMP tmp2, TEMP res);
7596   size(16);
7597   format %{ "loop:\n\t"
7598             "LDXR_w   $res, $mem\n\t"
7599             "ADD_w    $tmp1, $res, $add\n\t"
7600             "STXR_w   $tmp2, $tmp1, $mem\n\t"
7601             "CBNZ_w   $tmp2, loop" %}
7602 
7603   ins_encode %{
7604     Label loop;
7605     Register base = reg_to_register_object($mem$$base);
7606     __ bind(loop);
7607     __ ldxr_w($res$$Register, base);
7608     __ add_w($tmp1$$Register, $res$$Register, $add$$constant);
7609     __ stxr_w($tmp2$$Register, $tmp1$$Register, base);
7610     __ cbnz_w($tmp2$$Register, loop);
7611   %}
7612   ins_pipe( long_memory_op );
7613 %}
7614 #else
7615 instruct xaddI_aimmI(memoryex mem, aimmI add, iRegI res, iRegI tmp1, iRegI tmp2, flagsReg ccr) %{
7616   match(Set res (GetAndAddI mem add));
7617   effect(KILL ccr, TEMP tmp1, TEMP tmp2, TEMP res);
7618   size(20);
7619   format %{ "loop: \n\t"
7620             "LDREX    $res, $mem\n\t"
7621             "ADD      $tmp1, $res, $add\n\t"
7622             "STREX    $tmp2, $tmp1, $mem\n\t"
7623             "CMP      $tmp2, 0 \n\t"
7624             "B.ne     loop \n\t" %}
7625 
7626   ins_encode %{
7627     Label loop;
7628     __ bind(loop);
7629     __ ldrex($res$$Register,$mem$$Address);
7630     __ add($tmp1$$Register, $res$$Register, $add$$constant);
7631     __ strex($tmp2$$Register, $tmp1$$Register, $mem$$Address);
7632     __ cmp($tmp2$$Register, 0);
7633     __ b(loop, ne);
7634   %}
7635   ins_pipe( long_memory_op );
7636 %}
7637 #endif
7638 
7639 #ifdef AARCH64
7640 instruct xaddI_reg(memoryex mem, iRegI add, iRegI res, iRegI tmp1, iRegI tmp2) %{
7641   match(Set res (GetAndAddI mem add));
7642   effect(TEMP tmp1, TEMP tmp2, TEMP res);
7643   size(16);
7644   format %{ "loop:\n\t"
7645             "LDXR_w   $res, $mem\n\t"
7646             "ADD_w    $tmp1, $res, $add\n\t"
7647             "STXR_w   $tmp2, $tmp1, $mem\n\t"
7648             "CBNZ_w   $tmp2, loop" %}
7649 
7650   ins_encode %{
7651     Label loop;
7652     Register base = reg_to_register_object($mem$$base);
7653     __ bind(loop);
7654     __ ldxr_w($res$$Register, base);
7655     __ add_w($tmp1$$Register, $res$$Register, $add$$Register);
7656     __ stxr_w($tmp2$$Register, $tmp1$$Register, base);
7657     __ cbnz_w($tmp2$$Register, loop);
7658   %}
7659   ins_pipe( long_memory_op );
7660 %}
7661 #else
7662 instruct xaddI_reg(memoryex mem, iRegI add, iRegI res, iRegI tmp1, iRegI tmp2, flagsReg ccr) %{
7663   match(Set res (GetAndAddI mem add));
7664   effect(KILL ccr, TEMP tmp1, TEMP tmp2, TEMP res);
7665   size(20);
7666   format %{ "loop: \n\t"
7667             "LDREX    $res, $mem\n\t"
7668             "ADD      $tmp1, $res, $add\n\t"
7669             "STREX    $tmp2, $tmp1, $mem\n\t"
7670             "CMP      $tmp2, 0 \n\t"
7671             "B.ne     loop \n\t" %}
7672 
7673   ins_encode %{
7674     Label loop;
7675     __ bind(loop);
7676     __ ldrex($res$$Register,$mem$$Address);
7677     __ add($tmp1$$Register, $res$$Register, $add$$Register);
7678     __ strex($tmp2$$Register, $tmp1$$Register, $mem$$Address);
7679     __ cmp($tmp2$$Register, 0);
7680     __ b(loop, ne);
7681   %}
7682   ins_pipe( long_memory_op );
7683 %}
7684 #endif
7685 
7686 #ifdef AARCH64
7687 instruct xaddL_reg_no_res(memoryex mem, iRegL add, Universe dummy, iRegL tmp1, iRegI tmp2) %{
7688   predicate(n->as_LoadStore()->result_not_used());
7689   match(Set dummy (GetAndAddL mem add));
7690   effect(TEMP tmp1, TEMP tmp2);
7691   size(16);
7692   format %{ "loop:\n\t"
7693             "LDXR     $tmp1, $mem\n\t"
7694             "ADD      $tmp1, $tmp1, $add\n\t"
7695             "STXR     $tmp2, $tmp1, $mem\n\t"
7696             "CBNZ_w   $tmp2, loop" %}
7697 
7698   ins_encode %{
7699     Label loop;
7700     Register base = reg_to_register_object($mem$$base);
7701     __ bind(loop);
7702     __ ldxr($tmp1$$Register, base);
7703     __ add($tmp1$$Register, $tmp1$$Register, $add$$Register);
7704     __ stxr($tmp2$$Register, $tmp1$$Register, base);
7705     __ cbnz_w($tmp2$$Register, loop);
7706   %}
7707   ins_pipe( long_memory_op );
7708 %}
7709 #else
7710 instruct xaddL_reg_no_res(memoryex mem, iRegL add, Universe dummy, iRegLd tmp1, iRegI tmp2, flagsReg ccr) %{
7711   predicate(n->as_LoadStore()->result_not_used());
7712   match(Set dummy (GetAndAddL mem add));
7713   effect( KILL ccr, TEMP tmp1, TEMP tmp2);
7714   size(24);
7715   format %{ "loop: \n\t"
7716             "LDREXD   $tmp1, $mem\n\t"
7717             "ADDS     $tmp1.lo, $tmp1.lo, $add.lo\n\t"
7718             "ADC      $tmp1.hi, $tmp1.hi, $add.hi\n\t"
7719             "STREXD   $tmp2, $tmp1, $mem\n\t"
7720             "CMP      $tmp2, 0 \n\t"
7721             "B.ne     loop \n\t" %}
7722 
7723   ins_encode %{
7724     Label loop;
7725     __ bind(loop);
7726     __ ldrexd($tmp1$$Register, $mem$$Address);
7727     __ adds($tmp1$$Register, $tmp1$$Register, $add$$Register);
7728     __ adc($tmp1$$Register->successor(), $tmp1$$Register->successor(), $add$$Register->successor());
7729     __ strexd($tmp2$$Register, $tmp1$$Register, $mem$$Address);
7730     __ cmp($tmp2$$Register, 0);
7731     __ b(loop, ne);
7732   %}
7733   ins_pipe( long_memory_op );
7734 %}
7735 #endif
7736 
7737 #ifdef AARCH64
7738 instruct xaddL_imm_no_res(memoryex mem, aimmL add, Universe dummy, iRegL tmp1, iRegI tmp2) %{
7739   predicate(n->as_LoadStore()->result_not_used());
7740   match(Set dummy (GetAndAddL mem add));
7741   effect(TEMP tmp1, TEMP tmp2);
7742   size(16);
7743   format %{ "loop:\n\t"
7744             "LDXR     $tmp1, $mem\n\t"
7745             "ADD      $tmp1, $tmp1, $add\n\t"
7746             "STXR     $tmp2, $tmp1, $mem\n\t"
7747             "CBNZ_w   $tmp2, loop" %}
7748 
7749   ins_encode %{
7750     Label loop;
7751     Register base = reg_to_register_object($mem$$base);
7752     __ bind(loop);
7753     __ ldxr($tmp1$$Register, base);
7754     __ add($tmp1$$Register, $tmp1$$Register, $add$$constant);
7755     __ stxr($tmp2$$Register, $tmp1$$Register, base);
7756     __ cbnz_w($tmp2$$Register, loop);
7757   %}
7758   ins_pipe( long_memory_op );
7759 %}
7760 #else
7761 // TODO: try immLRot2 instead, (0, $con$$constant) becomes
7762 // (hi($con$$constant), lo($con$$constant)) becomes
7763 instruct xaddL_immRot_no_res(memoryex mem, immLlowRot add, Universe dummy, iRegLd tmp1, iRegI tmp2, flagsReg ccr) %{
7764   predicate(n->as_LoadStore()->result_not_used());
7765   match(Set dummy (GetAndAddL mem add));
7766   effect( KILL ccr, TEMP tmp1, TEMP tmp2);
7767   size(24);
7768   format %{ "loop: \n\t"
7769             "LDREXD   $tmp1, $mem\n\t"
7770             "ADDS     $tmp1.lo, $tmp1.lo, $add\n\t"
7771             "ADC      $tmp1.hi, $tmp1.hi, 0\n\t"
7772             "STREXD   $tmp2, $tmp1, $mem\n\t"
7773             "CMP      $tmp2, 0 \n\t"
7774             "B.ne     loop \n\t" %}
7775 
7776   ins_encode %{
7777     Label loop;
7778     __ bind(loop);
7779     __ ldrexd($tmp1$$Register, $mem$$Address);
7780     __ adds($tmp1$$Register, $tmp1$$Register, $add$$constant);
7781     __ adc($tmp1$$Register->successor(), $tmp1$$Register->successor(), 0);
7782     __ strexd($tmp2$$Register, $tmp1$$Register, $mem$$Address);
7783     __ cmp($tmp2$$Register, 0);
7784     __ b(loop, ne);
7785   %}
7786   ins_pipe( long_memory_op );
7787 %}
7788 #endif
7789 
7790 #ifdef AARCH64
7791 instruct xaddL_reg(memoryex mem, iRegL add, iRegL res, iRegL tmp1, iRegI tmp2) %{
7792   match(Set res (GetAndAddL mem add));
7793   effect(TEMP tmp1, TEMP tmp2, TEMP res);
7794   size(16);
7795   format %{ "loop:\n\t"
7796             "LDXR     $res, $mem\n\t"
7797             "ADD      $tmp1, $res, $add\n\t"
7798             "STXR     $tmp2, $tmp1, $mem\n\t"
7799             "CBNZ_w   $tmp2, loop" %}
7800 
7801   ins_encode %{
7802     Label loop;
7803     Register base = reg_to_register_object($mem$$base);
7804     __ bind(loop);
7805     __ ldxr($res$$Register, base);
7806     __ add($tmp1$$Register, $res$$Register, $add$$Register);
7807     __ stxr($tmp2$$Register, $tmp1$$Register, base);
7808     __ cbnz_w($tmp2$$Register, loop);
7809   %}
7810   ins_pipe( long_memory_op );
7811 %}
7812 #else
7813 instruct xaddL_reg(memoryex mem, iRegL add, iRegLd res, iRegLd tmp1, iRegI tmp2, flagsReg ccr) %{
7814   match(Set res (GetAndAddL mem add));
7815   effect( KILL ccr, TEMP tmp1, TEMP tmp2, TEMP res);
7816   size(24);
7817   format %{ "loop: \n\t"
7818             "LDREXD   $res, $mem\n\t"
7819             "ADDS     $tmp1.lo, $res.lo, $add.lo\n\t"
7820             "ADC      $tmp1.hi, $res.hi, $add.hi\n\t"
7821             "STREXD   $tmp2, $tmp1, $mem\n\t"
7822             "CMP      $tmp2, 0 \n\t"
7823             "B.ne     loop \n\t" %}
7824 
7825   ins_encode %{
7826     Label loop;
7827     __ bind(loop);
7828     __ ldrexd($res$$Register, $mem$$Address);
7829     __ adds($tmp1$$Register, $res$$Register, $add$$Register);
7830     __ adc($tmp1$$Register->successor(), $res$$Register->successor(), $add$$Register->successor());
7831     __ strexd($tmp2$$Register, $tmp1$$Register, $mem$$Address);
7832     __ cmp($tmp2$$Register, 0);
7833     __ b(loop, ne);
7834   %}
7835   ins_pipe( long_memory_op );
7836 %}
7837 #endif
7838 
7839 #ifdef AARCH64
7840 instruct xaddL_imm(memoryex mem, aimmL add, iRegL res, iRegL tmp1, iRegI tmp2) %{
7841   match(Set res (GetAndAddL mem add));
7842   effect(TEMP tmp1, TEMP tmp2, TEMP res);
7843   size(16);
7844   format %{ "loop:\n\t"
7845             "LDXR     $res, $mem\n\t"
7846             "ADD      $tmp1, $res, $add\n\t"
7847             "STXR     $tmp2, $tmp1, $mem\n\t"
7848             "CBNZ_w   $tmp2, loop" %}
7849 
7850   ins_encode %{
7851     Label loop;
7852     Register base = reg_to_register_object($mem$$base);
7853     __ bind(loop);
7854     __ ldxr($res$$Register, base);
7855     __ add($tmp1$$Register, $res$$Register, $add$$constant);
7856     __ stxr($tmp2$$Register, $tmp1$$Register, base);
7857     __ cbnz_w($tmp2$$Register, loop);
7858   %}
7859   ins_pipe( long_memory_op );
7860 %}
7861 #else
7862 // TODO: try immLRot2 instead, (0, $con$$constant) becomes
7863 // (hi($con$$constant), lo($con$$constant)) becomes
7864 instruct xaddL_immRot(memoryex mem, immLlowRot add, iRegLd res, iRegLd tmp1, iRegI tmp2, flagsReg ccr) %{
7865   match(Set res (GetAndAddL mem add));
7866   effect( KILL ccr, TEMP tmp1, TEMP tmp2, TEMP res);
7867   size(24);
7868   format %{ "loop: \n\t"
7869             "LDREXD   $res, $mem\n\t"
7870             "ADDS     $tmp1.lo, $res.lo, $add\n\t"
7871             "ADC      $tmp1.hi, $res.hi, 0\n\t"
7872             "STREXD   $tmp2, $tmp1, $mem\n\t"
7873             "CMP      $tmp2, 0 \n\t"
7874             "B.ne     loop \n\t" %}
7875 
7876   ins_encode %{
7877     Label loop;
7878     __ bind(loop);
7879     __ ldrexd($res$$Register, $mem$$Address);
7880     __ adds($tmp1$$Register, $res$$Register, $add$$constant);
7881     __ adc($tmp1$$Register->successor(), $res$$Register->successor(), 0);
7882     __ strexd($tmp2$$Register, $tmp1$$Register, $mem$$Address);
7883     __ cmp($tmp2$$Register, 0);
7884     __ b(loop, ne);
7885   %}
7886   ins_pipe( long_memory_op );
7887 %}
7888 #endif
7889 
7890 #ifdef AARCH64
7891 instruct xchgI(memoryex mem, iRegI newval, iRegI res, iRegI tmp) %{
7892   match(Set res (GetAndSetI mem newval));
7893   effect(TEMP tmp, TEMP res);
7894   size(12);
7895   format %{ "loop:\n\t"
7896             "LDXR_w   $res, $mem\n\t"
7897             "STXR_w   $tmp, $newval, $mem\n\t"
7898             "CBNZ_w   $tmp, loop" %}
7899 
7900   ins_encode %{
7901     Label loop;
7902     Register base = reg_to_register_object($mem$$base);
7903     __ bind(loop);
7904     __ ldxr_w($res$$Register, base);
7905     __ stxr_w($tmp$$Register, $newval$$Register, base);
7906     __ cbnz_w($tmp$$Register, loop);
7907   %}
7908   ins_pipe( long_memory_op );
7909 %}
7910 
7911 #ifdef XXX
7912 // Disabled until 8051805 is fixed.
7913 instruct xchgN(memoryex mem, iRegN newval, iRegN res, iRegN tmp) %{
7914   match(Set res (GetAndSetN mem newval));
7915   effect(TEMP tmp, TEMP res);
7916   size(12);
7917   format %{ "loop:\n\t"
7918             "LDXR_w   $res, $mem\n\t"
7919             "STXR_w   $tmp, $newval, $mem\n\t"
7920             "CBNZ_w   $tmp, loop" %}
7921 
7922   ins_encode %{
7923     Label loop;
7924     Register base = reg_to_register_object($mem$$base);
7925     __ bind(loop);
7926     __ ldxr_w($res$$Register, base);
7927     __ stxr_w($tmp$$Register, $newval$$Register, base);
7928     __ cbnz_w($tmp$$Register, loop);
7929   %}
7930   ins_pipe( long_memory_op );
7931 %}
7932 #endif
7933 #else
7934 instruct xchgI(memoryex mem, iRegI newval, iRegI res, iRegI tmp, flagsReg ccr) %{
7935   match(Set res (GetAndSetI mem newval));
7936   effect(KILL ccr, TEMP tmp, TEMP res);
7937   size(16);
7938   format %{ "loop: \n\t"
7939             "LDREX    $res, $mem\n\t"
7940             "STREX    $tmp, $newval, $mem\n\t"
7941             "CMP      $tmp, 0 \n\t"
7942             "B.ne     loop \n\t" %}
7943 
7944   ins_encode %{
7945     Label loop;
7946     __ bind(loop);
7947     __ ldrex($res$$Register,$mem$$Address);
7948     __ strex($tmp$$Register, $newval$$Register, $mem$$Address);
7949     __ cmp($tmp$$Register, 0);
7950     __ b(loop, ne);
7951   %}
7952   ins_pipe( long_memory_op );
7953 %}
7954 #endif
7955 
7956 #ifdef AARCH64
7957 instruct xchgL(memoryex mem, iRegL newval, iRegL res, iRegI tmp) %{
7958   match(Set res (GetAndSetL mem newval));
7959   effect(TEMP tmp, TEMP res);
7960   size(12);
7961   format %{ "loop:\n\t"
7962             "LDXR     $res, $mem\n\t"
7963             "STXR     $tmp, $newval, $mem\n\t"
7964             "CBNZ_w   $tmp, loop" %}
7965 
7966   ins_encode %{
7967     Label loop;
7968     Register base = reg_to_register_object($mem$$base);
7969     __ bind(loop);
7970     __ ldxr($res$$Register, base);
7971     __ stxr($tmp$$Register, $newval$$Register, base);
7972     __ cbnz_w($tmp$$Register, loop);
7973   %}
7974   ins_pipe( long_memory_op );
7975 %}
7976 #else
7977 instruct xchgL(memoryex mem, iRegLd newval, iRegLd res, iRegI tmp, flagsReg ccr) %{
7978   match(Set res (GetAndSetL mem newval));
7979   effect( KILL ccr, TEMP tmp, TEMP res);
7980   size(16);
7981   format %{ "loop: \n\t"
7982             "LDREXD   $res, $mem\n\t"
7983             "STREXD   $tmp, $newval, $mem\n\t"
7984             "CMP      $tmp, 0 \n\t"
7985             "B.ne     loop \n\t" %}
7986 
7987   ins_encode %{
7988     Label loop;
7989     __ bind(loop);
7990     __ ldrexd($res$$Register, $mem$$Address);
7991     __ strexd($tmp$$Register, $newval$$Register, $mem$$Address);
7992     __ cmp($tmp$$Register, 0);
7993     __ b(loop, ne);
7994   %}
7995   ins_pipe( long_memory_op );
7996 %}
7997 #endif // !AARCH64
7998 
7999 #ifdef AARCH64
8000 instruct xchgP(memoryex mem, iRegP newval, iRegP res, iRegI tmp) %{
8001   match(Set res (GetAndSetP mem newval));
8002   effect(TEMP tmp, TEMP res);
8003   size(12);
8004   format %{ "loop:\n\t"
8005             "LDREX    $res, $mem\n\t"
8006             "STREX    $tmp, $newval, $mem\n\t"
8007             "CBNZ_w   $tmp, loop" %}
8008 
8009   ins_encode %{
8010     Label loop;
8011     Register base = reg_to_register_object($mem$$base);
8012     __ bind(loop);
8013     __ ldrex($res$$Register, base);
8014     __ strex($tmp$$Register, $newval$$Register, base);
8015     __ cbnz_w($tmp$$Register, loop);
8016   %}
8017   ins_pipe( long_memory_op );
8018 %}
8019 #else
8020 instruct xchgP(memoryex mem, iRegP newval, iRegP res, iRegI tmp, flagsReg ccr) %{
8021   match(Set res (GetAndSetP mem newval));
8022   effect(KILL ccr, TEMP tmp, TEMP res);
8023   size(16);
8024   format %{ "loop: \n\t"
8025             "LDREX    $res, $mem\n\t"
8026             "STREX    $tmp, $newval, $mem\n\t"
8027             "CMP      $tmp, 0 \n\t"
8028             "B.ne     loop \n\t" %}
8029 
8030   ins_encode %{
8031     Label loop;
8032     __ bind(loop);
8033     __ ldrex($res$$Register,$mem$$Address);
8034     __ strex($tmp$$Register, $newval$$Register, $mem$$Address);
8035     __ cmp($tmp$$Register, 0);
8036     __ b(loop, ne);
8037   %}
8038   ins_pipe( long_memory_op );
8039 %}
8040 #endif // !AARCH64
8041 
8042 //---------------------
8043 // Subtraction Instructions
8044 // Register Subtraction
8045 instruct subI_reg_reg(iRegI dst, iRegI src1, iRegI src2) %{
8046   match(Set dst (SubI src1 src2));
8047 
8048   size(4);
8049   format %{ "sub_32 $dst,$src1,$src2\t! int" %}
8050   ins_encode %{
8051     __ sub_32($dst$$Register, $src1$$Register, $src2$$Register);
8052   %}
8053   ins_pipe(ialu_reg_reg);
8054 %}
8055 
8056 #ifndef AARCH64
8057 instruct subshlI_reg_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3) %{
8058   match(Set dst (SubI src1 (LShiftI src2 src3)));
8059 
8060   size(4);
8061   format %{ "SUB    $dst,$src1,$src2<<$src3" %}
8062   ins_encode %{
8063     __ sub($dst$$Register, $src1$$Register, AsmOperand($src2$$Register, lsl, $src3$$Register));
8064   %}
8065   ins_pipe(ialu_reg_reg);
8066 %}
8067 #endif
8068 
8069 instruct subshlI_reg_reg_imm(iRegI dst, iRegI src1, iRegI src2, immU5 src3) %{
8070   match(Set dst (SubI src1 (LShiftI src2 src3)));
8071 
8072   size(4);
8073   format %{ "sub_32 $dst,$src1,$src2<<$src3\t! int" %}
8074   ins_encode %{
8075     __ sub_32($dst$$Register, $src1$$Register, AsmOperand($src2$$Register, lsl, $src3$$constant));
8076   %}
8077   ins_pipe(ialu_reg_reg);
8078 %}
8079 
8080 #ifndef AARCH64
8081 instruct subsarI_reg_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3) %{
8082   match(Set dst (SubI src1 (RShiftI src2 src3)));
8083 
8084   size(4);
8085   format %{ "SUB    $dst,$src1,$src2>>$src3" %}
8086   ins_encode %{
8087     __ sub($dst$$Register, $src1$$Register, AsmOperand($src2$$Register, asr, $src3$$Register));
8088   %}
8089   ins_pipe(ialu_reg_reg);
8090 %}
8091 #endif
8092 
8093 instruct subsarI_reg_reg_imm(iRegI dst, iRegI src1, iRegI src2, immU5 src3) %{
8094   match(Set dst (SubI src1 (RShiftI src2 src3)));
8095 
8096   size(4);
8097   format %{ "sub_32 $dst,$src1,$src2>>$src3\t! int" %}
8098   ins_encode %{
8099     __ sub_32($dst$$Register, $src1$$Register, AsmOperand($src2$$Register, asr, $src3$$constant));
8100   %}
8101   ins_pipe(ialu_reg_reg);
8102 %}
8103 
8104 #ifndef AARCH64
8105 instruct subshrI_reg_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3) %{
8106   match(Set dst (SubI src1 (URShiftI src2 src3)));
8107 
8108   size(4);
8109   format %{ "SUB    $dst,$src1,$src2>>>$src3" %}
8110   ins_encode %{
8111     __ sub($dst$$Register, $src1$$Register, AsmOperand($src2$$Register, lsr, $src3$$Register));
8112   %}
8113   ins_pipe(ialu_reg_reg);
8114 %}
8115 #endif
8116 
8117 instruct subshrI_reg_reg_imm(iRegI dst, iRegI src1, iRegI src2, immU5 src3) %{
8118   match(Set dst (SubI src1 (URShiftI src2 src3)));
8119 
8120   size(4);
8121   format %{ "sub_32 $dst,$src1,$src2>>>$src3\t! int" %}
8122   ins_encode %{
8123     __ sub_32($dst$$Register, $src1$$Register, AsmOperand($src2$$Register, lsr, $src3$$constant));
8124   %}
8125   ins_pipe(ialu_reg_reg);
8126 %}
8127 
8128 #ifndef AARCH64
8129 instruct rsbshlI_reg_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3) %{
8130   match(Set dst (SubI (LShiftI src1 src2) src3));
8131 
8132   size(4);
8133   format %{ "RSB    $dst,$src3,$src1<<$src2" %}
8134   ins_encode %{
8135     __ rsb($dst$$Register, $src3$$Register, AsmOperand($src1$$Register, lsl, $src2$$Register));
8136   %}
8137   ins_pipe(ialu_reg_reg);
8138 %}
8139 
8140 instruct rsbshlI_reg_imm_reg(iRegI dst, iRegI src1, immU5 src2, iRegI src3) %{
8141   match(Set dst (SubI (LShiftI src1 src2) src3));
8142 
8143   size(4);
8144   format %{ "RSB    $dst,$src3,$src1<<$src2" %}
8145   ins_encode %{
8146     __ rsb($dst$$Register, $src3$$Register, AsmOperand($src1$$Register, lsl, $src2$$constant));
8147   %}
8148   ins_pipe(ialu_reg_reg);
8149 %}
8150 
8151 instruct rsbsarI_reg_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3) %{
8152   match(Set dst (SubI (RShiftI src1 src2) src3));
8153 
8154   size(4);
8155   format %{ "RSB    $dst,$src3,$src1>>$src2" %}
8156   ins_encode %{
8157     __ rsb($dst$$Register, $src3$$Register, AsmOperand($src1$$Register, asr, $src2$$Register));
8158   %}
8159   ins_pipe(ialu_reg_reg);
8160 %}
8161 
8162 instruct rsbsarI_reg_imm_reg(iRegI dst, iRegI src1, immU5 src2, iRegI src3) %{
8163   match(Set dst (SubI (RShiftI src1 src2) src3));
8164 
8165   size(4);
8166   format %{ "RSB    $dst,$src3,$src1>>$src2" %}
8167   ins_encode %{
8168     __ rsb($dst$$Register, $src3$$Register, AsmOperand($src1$$Register, asr, $src2$$constant));
8169   %}
8170   ins_pipe(ialu_reg_reg);
8171 %}
8172 
8173 instruct rsbshrI_reg_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3) %{
8174   match(Set dst (SubI (URShiftI src1 src2) src3));
8175 
8176   size(4);
8177   format %{ "RSB    $dst,$src3,$src1>>>$src2" %}
8178   ins_encode %{
8179     __ rsb($dst$$Register, $src3$$Register, AsmOperand($src1$$Register, lsr, $src2$$Register));
8180   %}
8181   ins_pipe(ialu_reg_reg);
8182 %}
8183 
8184 instruct rsbshrI_reg_imm_reg(iRegI dst, iRegI src1, immU5 src2, iRegI src3) %{
8185   match(Set dst (SubI (URShiftI src1 src2) src3));
8186 
8187   size(4);
8188   format %{ "RSB    $dst,$src3,$src1>>>$src2" %}
8189   ins_encode %{
8190     __ rsb($dst$$Register, $src3$$Register, AsmOperand($src1$$Register, lsr, $src2$$constant));
8191   %}
8192   ins_pipe(ialu_reg_reg);
8193 %}
8194 #endif
8195 
8196 // Immediate Subtraction
8197 instruct subI_reg_aimmI(iRegI dst, iRegI src1, aimmI src2) %{
8198   match(Set dst (SubI src1 src2));
8199 
8200   size(4);
8201   format %{ "sub_32 $dst,$src1,$src2\t! int" %}
8202   ins_encode %{
8203     __ sub_32($dst$$Register, $src1$$Register, $src2$$constant);
8204   %}
8205   ins_pipe(ialu_reg_imm);
8206 %}
8207 
8208 instruct subI_reg_immRotneg(iRegI dst, iRegI src1, aimmIneg src2) %{
8209   match(Set dst (AddI src1 src2));
8210 
8211   size(4);
8212   format %{ "sub_32 $dst,$src1,-($src2)\t! int" %}
8213   ins_encode %{
8214     __ sub_32($dst$$Register, $src1$$Register, -$src2$$constant);
8215   %}
8216   ins_pipe(ialu_reg_imm);
8217 %}
8218 
8219 #ifndef AARCH64
8220 instruct subI_immRot_reg(iRegI dst, immIRot src1, iRegI src2) %{
8221   match(Set dst (SubI src1 src2));
8222 
8223   size(4);
8224   format %{ "RSB    $dst,$src2,src1" %}
8225   ins_encode %{
8226     __ rsb($dst$$Register, $src2$$Register, $src1$$constant);
8227   %}
8228   ins_pipe(ialu_zero_reg);
8229 %}
8230 #endif
8231 
8232 // Register Subtraction
8233 #ifdef AARCH64
8234 instruct subL_reg_reg(iRegL dst, iRegL src1, iRegL src2) %{
8235   match(Set dst (SubL src1 src2));
8236 
8237   size(4);
8238   format %{ "SUB    $dst,$src1,$src2\t! long" %}
8239   ins_encode %{
8240     __ sub($dst$$Register, $src1$$Register, $src2$$Register);
8241   %}
8242   ins_pipe(ialu_reg_reg);
8243 %}
8244 #else
8245 instruct subL_reg_reg(iRegL dst, iRegL src1, iRegL src2, flagsReg icc ) %{
8246   match(Set dst (SubL src1 src2));
8247   effect (KILL icc);
8248 
8249   size(8);
8250   format %{ "SUBS   $dst.lo,$src1.lo,$src2.lo\t! long\n\t"
8251             "SBC    $dst.hi,$src1.hi,$src2.hi" %}
8252   ins_encode %{
8253     __ subs($dst$$Register, $src1$$Register, $src2$$Register);
8254     __ sbc($dst$$Register->successor(), $src1$$Register->successor(), $src2$$Register->successor());
8255   %}
8256   ins_pipe(ialu_reg_reg);
8257 %}
8258 #endif
8259 
8260 #ifdef AARCH64
8261 // Immediate Subtraction
8262 instruct subL_reg_aimm(iRegL dst, iRegL src1, aimmL src2) %{
8263   match(Set dst (SubL src1 src2));
8264 
8265   size(4);
8266   format %{ "SUB    $dst,$src1,$src2\t! long" %}
8267   ins_encode %{
8268     __ sub($dst$$Register, $src1$$Register, $src2$$constant);
8269   %}
8270   ins_pipe(ialu_reg_imm);
8271 %}
8272 
8273 instruct subL_reg_immLneg(iRegL dst, iRegL src1, aimmLneg src2) %{
8274   match(Set dst (AddL src1 src2));
8275 
8276   size(4);
8277   format %{ "SUB    $dst,$src1,-($src2)\t! long" %}
8278   ins_encode %{
8279     __ sub($dst$$Register, $src1$$Register, -$src2$$constant);
8280   %}
8281   ins_pipe(ialu_reg_imm);
8282 %}
8283 #else
8284 // TODO
8285 #endif
8286 
8287 #ifndef AARCH64
8288 // Immediate Subtraction
8289 // TODO: try immLRot2 instead, (0, $con$$constant) becomes
8290 // (hi($con$$constant), lo($con$$constant)) becomes
8291 instruct subL_reg_immRot(iRegL dst, iRegL src1, immLlowRot con, flagsReg icc) %{
8292   match(Set dst (SubL src1 con));
8293   effect (KILL icc);
8294 
8295   size(8);
8296   format %{ "SUB    $dst.lo,$src1.lo,$con\t! long\n\t"
8297             "SBC    $dst.hi,$src1.hi,0" %}
8298   ins_encode %{
8299     __ subs($dst$$Register, $src1$$Register, $con$$constant);
8300     __ sbc($dst$$Register->successor(), $src1$$Register->successor(), 0);
8301   %}
8302   ins_pipe(ialu_reg_imm);
8303 %}
8304 
8305 // Long negation
8306 instruct negL_reg_reg(iRegL dst, immL0 zero, iRegL src2, flagsReg icc) %{
8307   match(Set dst (SubL zero src2));
8308   effect (KILL icc);
8309 
8310   size(8);
8311   format %{ "RSBS   $dst.lo,$src2.lo,0\t! long\n\t"
8312             "RSC    $dst.hi,$src2.hi,0" %}
8313   ins_encode %{
8314     __ rsbs($dst$$Register, $src2$$Register, 0);
8315     __ rsc($dst$$Register->successor(), $src2$$Register->successor(), 0);
8316   %}
8317   ins_pipe(ialu_zero_reg);
8318 %}
8319 #endif // !AARCH64
8320 
8321 // Multiplication Instructions
8322 // Integer Multiplication
8323 // Register Multiplication
8324 instruct mulI_reg_reg(iRegI dst, iRegI src1, iRegI src2) %{
8325   match(Set dst (MulI src1 src2));
8326 
8327   size(4);
8328   format %{ "mul_32 $dst,$src1,$src2" %}
8329   ins_encode %{
8330     __ mul_32($dst$$Register, $src1$$Register, $src2$$Register);
8331   %}
8332   ins_pipe(imul_reg_reg);
8333 %}
8334 
8335 #ifdef AARCH64
8336 instruct mulL_reg_reg(iRegL dst, iRegL src1, iRegL src2) %{
8337   match(Set dst (MulL src1 src2));
8338   size(4);
8339   format %{ "MUL  $dst,$src1,$src2\t! long" %}
8340   ins_encode %{
8341     __ mul($dst$$Register, $src1$$Register, $src2$$Register);
8342   %}
8343   ins_pipe(imul_reg_reg);
8344 %}
8345 #else
8346 instruct mulL_lo1_hi2(iRegL dst, iRegL src1, iRegL src2) %{
8347   effect(DEF dst, USE src1, USE src2);
8348   size(4);
8349   format %{ "MUL  $dst.hi,$src1.lo,$src2.hi\t! long" %}
8350   ins_encode %{
8351     __ mul($dst$$Register->successor(), $src1$$Register, $src2$$Register->successor());
8352   %}
8353   ins_pipe(imul_reg_reg);
8354 %}
8355 
8356 instruct mulL_hi1_lo2(iRegL dst, iRegL src1, iRegL src2) %{
8357   effect(USE_DEF dst, USE src1, USE src2);
8358   size(8);
8359   format %{ "MLA  $dst.hi,$src1.hi,$src2.lo,$dst.hi\t! long\n\t"
8360             "MOV  $dst.lo, 0"%}
8361   ins_encode %{
8362     __ mla($dst$$Register->successor(), $src1$$Register->successor(), $src2$$Register, $dst$$Register->successor());
8363     __ mov($dst$$Register, 0);
8364   %}
8365   ins_pipe(imul_reg_reg);
8366 %}
8367 
8368 instruct mulL_lo1_lo2(iRegL dst, iRegL src1, iRegL src2) %{
8369   effect(USE_DEF dst, USE src1, USE src2);
8370   size(4);
8371   format %{ "UMLAL  $dst.lo,$dst.hi,$src1,$src2\t! long" %}
8372   ins_encode %{
8373     __ umlal($dst$$Register, $dst$$Register->successor(), $src1$$Register, $src2$$Register);
8374   %}
8375   ins_pipe(imul_reg_reg);
8376 %}
8377 
8378 instruct mulL_reg_reg(iRegL dst, iRegL src1, iRegL src2) %{
8379   match(Set dst (MulL src1 src2));
8380 
8381   expand %{
8382     mulL_lo1_hi2(dst, src1, src2);
8383     mulL_hi1_lo2(dst, src1, src2);
8384     mulL_lo1_lo2(dst, src1, src2);
8385   %}
8386 %}
8387 #endif // !AARCH64
8388 
8389 // Integer Division
8390 // Register Division
8391 #ifdef AARCH64
8392 instruct divI_reg_reg(iRegI dst, iRegI src1, iRegI src2) %{
8393   match(Set dst (DivI src1 src2));
8394 
8395   size(4);
8396   format %{ "SDIV    $dst,$src1,$src2\t! 32-bit" %}
8397   ins_encode %{
8398     __ sdiv_w($dst$$Register, $src1$$Register, $src2$$Register);
8399   %}
8400   ins_pipe(ialu_reg_reg); // FIXME
8401 %}
8402 #else
8403 instruct divI_reg_reg(R1RegI dst, R0RegI src1, R2RegI src2, LRRegP lr, flagsReg ccr) %{
8404   match(Set dst (DivI src1 src2));
8405   effect( KILL ccr, KILL src1, KILL src2, KILL lr);
8406   ins_cost((2+71)*DEFAULT_COST);
8407 
8408   format %{ "DIV   $dst,$src1,$src2 ! call to StubRoutines::Arm::idiv_irem_entry()" %}
8409   ins_encode %{
8410     __ call(StubRoutines::Arm::idiv_irem_entry(), relocInfo::runtime_call_type);
8411   %}
8412   ins_pipe(sdiv_reg_reg);
8413 %}
8414 #endif
8415 
8416 // Register Long Division
8417 #ifdef AARCH64
8418 instruct divL_reg_reg(iRegL dst, iRegL src1, iRegL src2) %{
8419   match(Set dst (DivL src1 src2));
8420 
8421   size(4);
8422   format %{ "SDIV    $dst,$src1,$src2" %}
8423   ins_encode %{
8424     __ sdiv($dst$$Register, $src1$$Register, $src2$$Register);
8425   %}
8426   ins_pipe(ialu_reg_reg); // FIXME
8427 %}
8428 #else
8429 instruct divL_reg_reg(R0R1RegL dst, R2R3RegL src1, R0R1RegL src2) %{
8430   match(Set dst (DivL src1 src2));
8431   effect(CALL);
8432   ins_cost(DEFAULT_COST*71);
8433   format %{ "DIVL  $src1,$src2,$dst\t! long ! call to SharedRuntime::ldiv" %}
8434   ins_encode %{
8435     address target = CAST_FROM_FN_PTR(address, SharedRuntime::ldiv);
8436     __ call(target, relocInfo::runtime_call_type);
8437   %}
8438   ins_pipe(divL_reg_reg);
8439 %}
8440 #endif
8441 
8442 // Integer Remainder
8443 // Register Remainder
8444 #ifdef AARCH64
8445 #ifdef TODO
8446 instruct msubI_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3) %{
8447   match(Set dst (SubI src1 (MulI src2 src3)));
8448 
8449   size(4);
8450   format %{ "MSUB    $dst,$src2,$src3,$src1\t! 32-bit\n\t" %}
8451   ins_encode %{
8452     __ msub_w($dst$$Register, $src2$$Register, $src3$$Register, $src1$$Register);
8453   %}
8454   ins_pipe(ialu_reg_reg); // FIXME
8455 %}
8456 #endif
8457 
8458 instruct modI_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI temp) %{
8459   match(Set dst (ModI src1 src2));
8460   effect(TEMP temp);
8461 
8462   size(8);
8463   format %{ "SDIV    $temp,$src1,$src2\t! 32-bit\n\t"
8464             "MSUB    $dst,$src2,$temp,$src1\t! 32-bit\n\t" %}
8465   ins_encode %{
8466     __ sdiv_w($temp$$Register, $src1$$Register, $src2$$Register);
8467     __ msub_w($dst$$Register, $src2$$Register, $temp$$Register, $src1$$Register);
8468   %}
8469   ins_pipe(ialu_reg_reg); // FIXME
8470 %}
8471 #else
8472 instruct modI_reg_reg(R0RegI dst, R0RegI src1, R2RegI src2, R1RegI temp, LRRegP lr, flagsReg ccr ) %{
8473   match(Set dst (ModI src1 src2));
8474   effect( KILL ccr, KILL temp, KILL src2, KILL lr);
8475 
8476   format %{ "MODI   $dst,$src1,$src2\t ! call to StubRoutines::Arm::idiv_irem_entry" %}
8477   ins_encode %{
8478     __ call(StubRoutines::Arm::idiv_irem_entry(), relocInfo::runtime_call_type);
8479   %}
8480   ins_pipe(sdiv_reg_reg);
8481 %}
8482 #endif
8483 
8484 // Register Long Remainder
8485 #ifdef AARCH64
8486 instruct modL_reg_reg(iRegL dst, iRegL src1, iRegL src2, iRegL temp) %{
8487   match(Set dst (ModL src1 src2));
8488   effect(TEMP temp);
8489 
8490   size(8);
8491   format %{ "SDIV    $temp,$src1,$src2\n\t"
8492             "MSUB    $dst,$src2,$temp,$src1" %}
8493   ins_encode %{
8494     __ sdiv($temp$$Register, $src1$$Register, $src2$$Register);
8495     __ msub($dst$$Register, $src2$$Register, $temp$$Register, $src1$$Register);
8496   %}
8497   ins_pipe(ialu_reg_reg); // FIXME
8498 %}
8499 #else
8500 instruct modL_reg_reg(R0R1RegL dst, R2R3RegL src1, R0R1RegL src2) %{
8501   match(Set dst (ModL src1 src2));
8502   effect(CALL);
8503   ins_cost(MEMORY_REF_COST); // FIXME
8504   format %{ "modL    $dst,$src1,$src2\t ! call to SharedRuntime::lrem" %}
8505   ins_encode %{
8506     address target = CAST_FROM_FN_PTR(address, SharedRuntime::lrem);
8507     __ call(target, relocInfo::runtime_call_type);
8508   %}
8509   ins_pipe(divL_reg_reg);
8510 %}
8511 #endif
8512 
8513 // Integer Shift Instructions
8514 
8515 // Register Shift Left
8516 instruct shlI_reg_reg(iRegI dst, iRegI src1, iRegI src2) %{
8517   match(Set dst (LShiftI src1 src2));
8518 
8519   size(4);
8520 #ifdef AARCH64
8521   format %{ "LSLV   $dst,$src1,$src2\t! int" %}
8522   ins_encode %{
8523     __ lslv_w($dst$$Register, $src1$$Register, $src2$$Register);
8524   %}
8525 #else
8526   format %{ "LSL  $dst,$src1,$src2 \n\t" %}
8527   ins_encode %{
8528     __ mov($dst$$Register, AsmOperand($src1$$Register, lsl, $src2$$Register));
8529   %}
8530 #endif
8531   ins_pipe(ialu_reg_reg);
8532 %}
8533 
8534 // Register Shift Left Immediate
8535 instruct shlI_reg_imm5(iRegI dst, iRegI src1, immU5 src2) %{
8536   match(Set dst (LShiftI src1 src2));
8537 
8538   size(4);
8539 #ifdef AARCH64
8540   format %{ "LSL_w  $dst,$src1,$src2\t! int" %}
8541   ins_encode %{
8542     __ _lsl($dst$$Register, $src1$$Register, $src2$$constant);
8543   %}
8544 #else
8545   format %{ "LSL    $dst,$src1,$src2\t! int" %}
8546   ins_encode %{
8547     __ logical_shift_left($dst$$Register, $src1$$Register, $src2$$constant);
8548   %}
8549 #endif
8550   ins_pipe(ialu_reg_imm);
8551 %}
8552 
8553 #ifndef AARCH64
8554 instruct shlL_reg_reg_merge_hi(iRegL dst, iRegL src1, iRegI src2) %{
8555   effect(USE_DEF dst, USE src1, USE src2);
8556   size(4);
8557   format %{"OR  $dst.hi,$dst.hi,($src1.hi << $src2)"  %}
8558   ins_encode %{
8559     __ orr($dst$$Register->successor(), $dst$$Register->successor(), AsmOperand($src1$$Register->successor(), lsl, $src2$$Register));
8560   %}
8561   ins_pipe(ialu_reg_reg);
8562 %}
8563 
8564 instruct shlL_reg_reg_merge_lo(iRegL dst, iRegL src1, iRegI src2) %{
8565   effect(USE_DEF dst, USE src1, USE src2);
8566   size(4);
8567   format %{ "LSL  $dst.lo,$src1.lo,$src2 \n\t" %}
8568   ins_encode %{
8569     __ mov($dst$$Register, AsmOperand($src1$$Register, lsl, $src2$$Register));
8570   %}
8571   ins_pipe(ialu_reg_reg);
8572 %}
8573 
8574 instruct shlL_reg_reg_overlap(iRegL dst, iRegL src1, iRegI src2, flagsReg ccr) %{
8575   effect(DEF dst, USE src1, USE src2, KILL ccr);
8576   size(16);
8577   format %{ "SUBS  $dst.hi,$src2,32 \n\t"
8578             "LSLpl $dst.hi,$src1.lo,$dst.hi \n\t"
8579             "RSBmi $dst.hi,$dst.hi,0 \n\t"
8580             "LSRmi $dst.hi,$src1.lo,$dst.hi" %}
8581 
8582   ins_encode %{
8583     // $src1$$Register and $dst$$Register->successor() can't be the same
8584     __ subs($dst$$Register->successor(), $src2$$Register, 32);
8585     __ mov($dst$$Register->successor(), AsmOperand($src1$$Register, lsl, $dst$$Register->successor()), pl);
8586     __ rsb($dst$$Register->successor(), $dst$$Register->successor(), 0, mi);
8587     __ mov($dst$$Register->successor(), AsmOperand($src1$$Register, lsr, $dst$$Register->successor()), mi);
8588   %}
8589   ins_pipe(ialu_reg_reg);
8590 %}
8591 #endif // !AARCH64
8592 
8593 instruct shlL_reg_reg(iRegL dst, iRegL src1, iRegI src2) %{
8594   match(Set dst (LShiftL src1 src2));
8595 
8596 #ifdef AARCH64
8597   size(4);
8598   format %{ "LSLV  $dst,$src1,$src2\t! long" %}
8599   ins_encode %{
8600     __ lslv($dst$$Register, $src1$$Register, $src2$$Register);
8601   %}
8602   ins_pipe(ialu_reg_reg);
8603 #else
8604   expand %{
8605     flagsReg ccr;
8606     shlL_reg_reg_overlap(dst, src1, src2, ccr);
8607     shlL_reg_reg_merge_hi(dst, src1, src2);
8608     shlL_reg_reg_merge_lo(dst, src1, src2);
8609   %}
8610 #endif
8611 %}
8612 
8613 #ifdef AARCH64
8614 instruct shlL_reg_imm6(iRegL dst, iRegL src1, immU6 src2) %{
8615   match(Set dst (LShiftL src1 src2));
8616 
8617   size(4);
8618   format %{ "LSL    $dst,$src1,$src2\t! long" %}
8619   ins_encode %{
8620     __ logical_shift_left($dst$$Register, $src1$$Register, $src2$$constant);
8621   %}
8622   ins_pipe(ialu_reg_imm);
8623 %}
8624 #else
8625 // Register Shift Left Immediate
8626 instruct shlL_reg_imm6(iRegL dst, iRegL src1, immU6Big src2) %{
8627   match(Set dst (LShiftL src1 src2));
8628 
8629   size(8);
8630   format %{ "LSL   $dst.hi,$src1.lo,$src2-32\t! or mov if $src2==32\n\t"
8631             "MOV   $dst.lo, 0" %}
8632   ins_encode %{
8633     if ($src2$$constant == 32) {
8634       __ mov($dst$$Register->successor(), $src1$$Register);
8635     } else {
8636       __ mov($dst$$Register->successor(), AsmOperand($src1$$Register, lsl, $src2$$constant-32));
8637     }
8638     __ mov($dst$$Register, 0);
8639   %}
8640   ins_pipe(ialu_reg_imm);
8641 %}
8642 
8643 instruct shlL_reg_imm5(iRegL dst, iRegL src1, immU5 src2) %{
8644   match(Set dst (LShiftL src1 src2));
8645 
8646   size(12);
8647   format %{ "LSL   $dst.hi,$src1.lo,$src2\n\t"
8648             "OR    $dst.hi, $dst.hi, $src1.lo >> 32-$src2\n\t"
8649             "LSL   $dst.lo,$src1.lo,$src2" %}
8650   ins_encode %{
8651     // The order of the following 3 instructions matters: src1.lo and
8652     // dst.hi can't overlap but src.hi and dst.hi can.
8653     __ mov($dst$$Register->successor(), AsmOperand($src1$$Register->successor(), lsl, $src2$$constant));
8654     __ orr($dst$$Register->successor(), $dst$$Register->successor(), AsmOperand($src1$$Register, lsr, 32-$src2$$constant));
8655     __ mov($dst$$Register, AsmOperand($src1$$Register, lsl, $src2$$constant));
8656   %}
8657   ins_pipe(ialu_reg_imm);
8658 %}
8659 #endif // !AARCH64
8660 
8661 // Register Arithmetic Shift Right
8662 instruct sarI_reg_reg(iRegI dst, iRegI src1, iRegI src2) %{
8663   match(Set dst (RShiftI src1 src2));
8664   size(4);
8665 #ifdef AARCH64
8666   format %{ "ASRV   $dst,$src1,$src2\t! int" %}
8667   ins_encode %{
8668     __ asrv_w($dst$$Register, $src1$$Register, $src2$$Register);
8669   %}
8670 #else
8671   format %{ "ASR    $dst,$src1,$src2\t! int" %}
8672   ins_encode %{
8673     __ mov($dst$$Register, AsmOperand($src1$$Register, asr, $src2$$Register));
8674   %}
8675 #endif
8676   ins_pipe(ialu_reg_reg);
8677 %}
8678 
8679 // Register Arithmetic Shift Right Immediate
8680 instruct sarI_reg_imm5(iRegI dst, iRegI src1, immU5 src2) %{
8681   match(Set dst (RShiftI src1 src2));
8682 
8683   size(4);
8684 #ifdef AARCH64
8685   format %{ "ASR_w  $dst,$src1,$src2" %}
8686   ins_encode %{
8687     __ _asr_w($dst$$Register, $src1$$Register, $src2$$constant);
8688   %}
8689 #else
8690   format %{ "ASR    $dst,$src1,$src2" %}
8691   ins_encode %{
8692     __ mov($dst$$Register, AsmOperand($src1$$Register, asr, $src2$$constant));
8693   %}
8694 #endif
8695   ins_pipe(ialu_reg_imm);
8696 %}
8697 
8698 #ifndef AARCH64
8699 // Register Shift Right Arithmetic Long
8700 instruct sarL_reg_reg_merge_lo(iRegL dst, iRegL src1, iRegI src2) %{
8701   effect(USE_DEF dst, USE src1, USE src2);
8702   size(4);
8703   format %{ "OR  $dst.lo,$dst.lo,($src1.lo >> $src2)"  %}
8704   ins_encode %{
8705     __ orr($dst$$Register, $dst$$Register, AsmOperand($src1$$Register, lsr, $src2$$Register));
8706   %}
8707   ins_pipe(ialu_reg_reg);
8708 %}
8709 
8710 instruct sarL_reg_reg_merge_hi(iRegL dst, iRegL src1, iRegI src2) %{
8711   effect(USE_DEF dst, USE src1, USE src2);
8712   size(4);
8713   format %{ "ASR  $dst.hi,$src1.hi,$src2 \n\t" %}
8714   ins_encode %{
8715     __ mov($dst$$Register->successor(), AsmOperand($src1$$Register->successor(), asr, $src2$$Register));
8716   %}
8717   ins_pipe(ialu_reg_reg);
8718 %}
8719 
8720 instruct sarL_reg_reg_overlap(iRegL dst, iRegL src1, iRegI src2, flagsReg ccr) %{
8721   effect(DEF dst, USE src1, USE src2, KILL ccr);
8722   size(16);
8723   format %{ "SUBS  $dst.lo,$src2,32 \n\t"
8724             "ASRpl $dst.lo,$src1.hi,$dst.lo \n\t"
8725             "RSBmi $dst.lo,$dst.lo,0 \n\t"
8726             "LSLmi $dst.lo,$src1.hi,$dst.lo" %}
8727 
8728   ins_encode %{
8729     // $src1$$Register->successor() and $dst$$Register can't be the same
8730     __ subs($dst$$Register, $src2$$Register, 32);
8731     __ mov($dst$$Register, AsmOperand($src1$$Register->successor(), asr, $dst$$Register), pl);
8732     __ rsb($dst$$Register, $dst$$Register, 0, mi);
8733     __ mov($dst$$Register, AsmOperand($src1$$Register->successor(), lsl, $dst$$Register), mi);
8734   %}
8735   ins_pipe(ialu_reg_reg);
8736 %}
8737 #endif // !AARCH64
8738 
8739 instruct sarL_reg_reg(iRegL dst, iRegL src1, iRegI src2) %{
8740   match(Set dst (RShiftL src1 src2));
8741 
8742 #ifdef AARCH64
8743   size(4);
8744   format %{ "ASRV  $dst,$src1,$src2\t! long" %}
8745   ins_encode %{
8746     __ asrv($dst$$Register, $src1$$Register, $src2$$Register);
8747   %}
8748   ins_pipe(ialu_reg_reg);
8749 #else
8750   expand %{
8751     flagsReg ccr;
8752     sarL_reg_reg_overlap(dst, src1, src2, ccr);
8753     sarL_reg_reg_merge_lo(dst, src1, src2);
8754     sarL_reg_reg_merge_hi(dst, src1, src2);
8755   %}
8756 #endif
8757 %}
8758 
8759 // Register Shift Left Immediate
8760 #ifdef AARCH64
8761 instruct sarL_reg_imm6(iRegL dst, iRegL src1, immU6 src2) %{
8762   match(Set dst (RShiftL src1 src2));
8763 
8764   size(4);
8765   format %{ "ASR    $dst,$src1,$src2\t! long" %}
8766   ins_encode %{
8767     __ _asr($dst$$Register, $src1$$Register, $src2$$constant);
8768   %}
8769   ins_pipe(ialu_reg_imm);
8770 %}
8771 #else
8772 instruct sarL_reg_imm6(iRegL dst, iRegL src1, immU6Big src2) %{
8773   match(Set dst (RShiftL src1 src2));
8774 
8775   size(8);
8776   format %{ "ASR   $dst.lo,$src1.hi,$src2-32\t! or mov if $src2==32\n\t"
8777             "ASR   $dst.hi,$src1.hi, $src2" %}
8778   ins_encode %{
8779     if ($src2$$constant == 32) {
8780       __ mov($dst$$Register, $src1$$Register->successor());
8781     } else{
8782       __ mov($dst$$Register, AsmOperand($src1$$Register->successor(), asr, $src2$$constant-32));
8783     }
8784     __ mov($dst$$Register->successor(), AsmOperand($src1$$Register->successor(), asr, 0));
8785   %}
8786 
8787   ins_pipe(ialu_reg_imm);
8788 %}
8789 
8790 instruct sarL_reg_imm5(iRegL dst, iRegL src1, immU5 src2) %{
8791   match(Set dst (RShiftL src1 src2));
8792   size(12);
8793   format %{ "LSR   $dst.lo,$src1.lo,$src2\n\t"
8794             "OR    $dst.lo, $dst.lo, $src1.hi << 32-$src2\n\t"
8795             "ASR   $dst.hi,$src1.hi,$src2" %}
8796   ins_encode %{
8797     // The order of the following 3 instructions matters: src1.lo and
8798     // dst.hi can't overlap but src.hi and dst.hi can.
8799     __ mov($dst$$Register, AsmOperand($src1$$Register, lsr, $src2$$constant));
8800     __ orr($dst$$Register, $dst$$Register, AsmOperand($src1$$Register->successor(), lsl, 32-$src2$$constant));
8801     __ mov($dst$$Register->successor(), AsmOperand($src1$$Register->successor(), asr, $src2$$constant));
8802   %}
8803   ins_pipe(ialu_reg_imm);
8804 %}
8805 #endif
8806 
8807 // Register Shift Right
8808 instruct shrI_reg_reg(iRegI dst, iRegI src1, iRegI src2) %{
8809   match(Set dst (URShiftI src1 src2));
8810   size(4);
8811 #ifdef AARCH64
8812   format %{ "LSRV   $dst,$src1,$src2\t! int" %}
8813   ins_encode %{
8814     __ lsrv_w($dst$$Register, $src1$$Register, $src2$$Register);
8815   %}
8816 #else
8817   format %{ "LSR    $dst,$src1,$src2\t! int" %}
8818   ins_encode %{
8819     __ mov($dst$$Register, AsmOperand($src1$$Register, lsr, $src2$$Register));
8820   %}
8821 #endif
8822   ins_pipe(ialu_reg_reg);
8823 %}
8824 
8825 // Register Shift Right Immediate
8826 instruct shrI_reg_imm5(iRegI dst, iRegI src1, immU5 src2) %{
8827   match(Set dst (URShiftI src1 src2));
8828 
8829   size(4);
8830 #ifdef AARCH64
8831   format %{ "LSR_w  $dst,$src1,$src2" %}
8832   ins_encode %{
8833     __ _lsr_w($dst$$Register, $src1$$Register, $src2$$constant);
8834   %}
8835 #else
8836   format %{ "LSR    $dst,$src1,$src2" %}
8837   ins_encode %{
8838     __ mov($dst$$Register, AsmOperand($src1$$Register, lsr, $src2$$constant));
8839   %}
8840 #endif
8841   ins_pipe(ialu_reg_imm);
8842 %}
8843 
8844 #ifndef AARCH64
8845 // Register Shift Right
8846 instruct shrL_reg_reg_merge_lo(iRegL dst, iRegL src1, iRegI src2) %{
8847   effect(USE_DEF dst, USE src1, USE src2);
8848   size(4);
8849   format %{ "OR   $dst.lo,$dst,($src1.lo >>> $src2)"  %}
8850   ins_encode %{
8851     __ orr($dst$$Register, $dst$$Register, AsmOperand($src1$$Register, lsr, $src2$$Register));
8852   %}
8853   ins_pipe(ialu_reg_reg);
8854 %}
8855 
8856 instruct shrL_reg_reg_merge_hi(iRegL dst, iRegL src1, iRegI src2) %{
8857   effect(USE_DEF dst, USE src1, USE src2);
8858   size(4);
8859   format %{ "LSR  $dst.hi,$src1.hi,$src2 \n\t" %}
8860   ins_encode %{
8861     __ mov($dst$$Register->successor(), AsmOperand($src1$$Register->successor(), lsr, $src2$$Register));
8862   %}
8863   ins_pipe(ialu_reg_reg);
8864 %}
8865 
8866 instruct shrL_reg_reg_overlap(iRegL dst, iRegL src1, iRegI src2, flagsReg ccr) %{
8867   effect(DEF dst, USE src1, USE src2, KILL ccr);
8868   size(16);
8869   format %{ "SUBS  $dst,$src2,32 \n\t"
8870             "LSRpl $dst,$src1.hi,$dst \n\t"
8871             "RSBmi $dst,$dst,0 \n\t"
8872             "LSLmi $dst,$src1.hi,$dst" %}
8873 
8874   ins_encode %{
8875     // $src1$$Register->successor() and $dst$$Register can't be the same
8876     __ subs($dst$$Register, $src2$$Register, 32);
8877     __ mov($dst$$Register, AsmOperand($src1$$Register->successor(), lsr, $dst$$Register), pl);
8878     __ rsb($dst$$Register, $dst$$Register, 0, mi);
8879     __ mov($dst$$Register, AsmOperand($src1$$Register->successor(), lsl, $dst$$Register), mi);
8880   %}
8881   ins_pipe(ialu_reg_reg);
8882 %}
8883 #endif // !AARCH64
8884 
8885 instruct shrL_reg_reg(iRegL dst, iRegL src1, iRegI src2) %{
8886   match(Set dst (URShiftL src1 src2));
8887 
8888 #ifdef AARCH64
8889   size(4);
8890   format %{ "LSRV  $dst,$src1,$src2\t! long" %}
8891   ins_encode %{
8892     __ lsrv($dst$$Register, $src1$$Register, $src2$$Register);
8893   %}
8894   ins_pipe(ialu_reg_reg);
8895 #else
8896   expand %{
8897     flagsReg ccr;
8898     shrL_reg_reg_overlap(dst, src1, src2, ccr);
8899     shrL_reg_reg_merge_lo(dst, src1, src2);
8900     shrL_reg_reg_merge_hi(dst, src1, src2);
8901   %}
8902 #endif
8903 %}
8904 
8905 // Register Shift Right Immediate
8906 #ifdef AARCH64
8907 instruct shrL_reg_imm6(iRegL dst, iRegL src1, immU6 src2) %{
8908   match(Set dst (URShiftL src1 src2));
8909 
8910   size(4);
8911   format %{ "LSR    $dst,$src1,$src2" %}
8912   ins_encode %{
8913     __ _lsr($dst$$Register, $src1$$Register, $src2$$constant);
8914   %}
8915   ins_pipe(ialu_reg_imm);
8916 %}
8917 #else
8918 instruct shrL_reg_imm6(iRegL dst, iRegL src1, immU6Big src2) %{
8919   match(Set dst (URShiftL src1 src2));
8920 
8921   size(8);
8922   format %{ "LSR   $dst.lo,$src1.hi,$src2-32\t! or mov if $src2==32\n\t"
8923             "MOV   $dst.hi, 0" %}
8924   ins_encode %{
8925     if ($src2$$constant == 32) {
8926       __ mov($dst$$Register, $src1$$Register->successor());
8927     } else {
8928       __ mov($dst$$Register, AsmOperand($src1$$Register->successor(), lsr, $src2$$constant-32));
8929     }
8930     __ mov($dst$$Register->successor(), 0);
8931   %}
8932 
8933   ins_pipe(ialu_reg_imm);
8934 %}
8935 
8936 instruct shrL_reg_imm5(iRegL dst, iRegL src1, immU5 src2) %{
8937   match(Set dst (URShiftL src1 src2));
8938 
8939   size(12);
8940   format %{ "LSR   $dst.lo,$src1.lo,$src2\n\t"
8941             "OR    $dst.lo, $dst.lo, $src1.hi << 32-$src2\n\t"
8942             "LSR   $dst.hi,$src1.hi,$src2" %}
8943   ins_encode %{
8944     // The order of the following 3 instructions matters: src1.lo and
8945     // dst.hi can't overlap but src.hi and dst.hi can.
8946     __ mov($dst$$Register, AsmOperand($src1$$Register, lsr, $src2$$constant));
8947     __ orr($dst$$Register, $dst$$Register, AsmOperand($src1$$Register->successor(), lsl, 32-$src2$$constant));
8948     __ mov($dst$$Register->successor(), AsmOperand($src1$$Register->successor(), lsr, $src2$$constant));
8949   %}
8950   ins_pipe(ialu_reg_imm);
8951 %}
8952 #endif // !AARCH64
8953 
8954 
8955 instruct shrP_reg_imm5(iRegX dst, iRegP src1, immU5 src2) %{
8956   match(Set dst (URShiftI (CastP2X src1) src2));
8957   size(4);
8958   format %{ "LSR    $dst,$src1,$src2\t! Cast ptr $src1 to int and shift" %}
8959   ins_encode %{
8960     __ logical_shift_right($dst$$Register, $src1$$Register, $src2$$constant);
8961   %}
8962   ins_pipe(ialu_reg_imm);
8963 %}
8964 
8965 //----------Floating Point Arithmetic Instructions-----------------------------
8966 
8967 //  Add float single precision
8968 instruct addF_reg_reg(regF dst, regF src1, regF src2) %{
8969   match(Set dst (AddF src1 src2));
8970 
8971   size(4);
8972   format %{ "FADDS  $dst,$src1,$src2" %}
8973   ins_encode %{
8974     __ add_float($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister);
8975   %}
8976 
8977   ins_pipe(faddF_reg_reg);
8978 %}
8979 
8980 //  Add float double precision
8981 instruct addD_reg_reg(regD dst, regD src1, regD src2) %{
8982   match(Set dst (AddD src1 src2));
8983 
8984   size(4);
8985   format %{ "FADDD  $dst,$src1,$src2" %}
8986   ins_encode %{
8987     __ add_double($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister);
8988   %}
8989 
8990   ins_pipe(faddD_reg_reg);
8991 %}
8992 
8993 //  Sub float single precision
8994 instruct subF_reg_reg(regF dst, regF src1, regF src2) %{
8995   match(Set dst (SubF src1 src2));
8996 
8997   size(4);
8998   format %{ "FSUBS  $dst,$src1,$src2" %}
8999   ins_encode %{
9000     __ sub_float($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister);
9001   %}
9002   ins_pipe(faddF_reg_reg);
9003 %}
9004 
9005 //  Sub float double precision
9006 instruct subD_reg_reg(regD dst, regD src1, regD src2) %{
9007   match(Set dst (SubD src1 src2));
9008 
9009   size(4);
9010   format %{ "FSUBD  $dst,$src1,$src2" %}
9011   ins_encode %{
9012     __ sub_double($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister);
9013   %}
9014   ins_pipe(faddD_reg_reg);
9015 %}
9016 
9017 //  Mul float single precision
9018 instruct mulF_reg_reg(regF dst, regF src1, regF src2) %{
9019   match(Set dst (MulF src1 src2));
9020 
9021   size(4);
9022   format %{ "FMULS  $dst,$src1,$src2" %}
9023   ins_encode %{
9024     __ mul_float($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister);
9025   %}
9026 
9027   ins_pipe(fmulF_reg_reg);
9028 %}
9029 
9030 //  Mul float double precision
9031 instruct mulD_reg_reg(regD dst, regD src1, regD src2) %{
9032   match(Set dst (MulD src1 src2));
9033 
9034   size(4);
9035   format %{ "FMULD  $dst,$src1,$src2" %}
9036   ins_encode %{
9037     __ mul_double($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister);
9038   %}
9039 
9040   ins_pipe(fmulD_reg_reg);
9041 %}
9042 
9043 //  Div float single precision
9044 instruct divF_reg_reg(regF dst, regF src1, regF src2) %{
9045   match(Set dst (DivF src1 src2));
9046 
9047   size(4);
9048   format %{ "FDIVS  $dst,$src1,$src2" %}
9049   ins_encode %{
9050     __ div_float($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister);
9051   %}
9052 
9053   ins_pipe(fdivF_reg_reg);
9054 %}
9055 
9056 //  Div float double precision
9057 instruct divD_reg_reg(regD dst, regD src1, regD src2) %{
9058   match(Set dst (DivD src1 src2));
9059 
9060   size(4);
9061   format %{ "FDIVD  $dst,$src1,$src2" %}
9062   ins_encode %{
9063     __ div_double($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister);
9064   %}
9065 
9066   ins_pipe(fdivD_reg_reg);
9067 %}
9068 
9069 //  Absolute float double precision
9070 instruct absD_reg(regD dst, regD src) %{
9071   match(Set dst (AbsD src));
9072 
9073   size(4);
9074   format %{ "FABSd  $dst,$src" %}
9075   ins_encode %{
9076     __ abs_double($dst$$FloatRegister, $src$$FloatRegister);
9077   %}
9078   ins_pipe(faddD_reg);
9079 %}
9080 
9081 //  Absolute float single precision
9082 instruct absF_reg(regF dst, regF src) %{
9083   match(Set dst (AbsF src));
9084   format %{ "FABSs  $dst,$src" %}
9085   ins_encode %{
9086     __ abs_float($dst$$FloatRegister, $src$$FloatRegister);
9087   %}
9088   ins_pipe(faddF_reg);
9089 %}
9090 
9091 instruct negF_reg(regF dst, regF src) %{
9092   match(Set dst (NegF src));
9093 
9094   size(4);
9095   format %{ "FNEGs  $dst,$src" %}
9096   ins_encode %{
9097     __ neg_float($dst$$FloatRegister, $src$$FloatRegister);
9098   %}
9099   ins_pipe(faddF_reg);
9100 %}
9101 
9102 instruct negD_reg(regD dst, regD src) %{
9103   match(Set dst (NegD src));
9104 
9105   format %{ "FNEGd  $dst,$src" %}
9106   ins_encode %{
9107     __ neg_double($dst$$FloatRegister, $src$$FloatRegister);
9108   %}
9109   ins_pipe(faddD_reg);
9110 %}
9111 
9112 //  Sqrt float double precision
9113 instruct sqrtF_reg_reg(regF dst, regF src) %{
9114   match(Set dst (ConvD2F (SqrtD (ConvF2D src))));
9115 
9116   size(4);
9117   format %{ "FSQRTS $dst,$src" %}
9118   ins_encode %{
9119     __ sqrt_float($dst$$FloatRegister, $src$$FloatRegister);
9120   %}
9121   ins_pipe(fdivF_reg_reg);
9122 %}
9123 
9124 //  Sqrt float double precision
9125 instruct sqrtD_reg_reg(regD dst, regD src) %{
9126   match(Set dst (SqrtD src));
9127 
9128   size(4);
9129   format %{ "FSQRTD $dst,$src" %}
9130   ins_encode %{
9131     __ sqrt_double($dst$$FloatRegister, $src$$FloatRegister);
9132   %}
9133   ins_pipe(fdivD_reg_reg);
9134 %}
9135 
9136 //----------Logical Instructions-----------------------------------------------
9137 // And Instructions
9138 // Register And
9139 instruct andI_reg_reg(iRegI dst, iRegI src1, iRegI src2) %{
9140   match(Set dst (AndI src1 src2));
9141 
9142   size(4);
9143   format %{ "and_32 $dst,$src1,$src2" %}
9144   ins_encode %{
9145     __ and_32($dst$$Register, $src1$$Register, $src2$$Register);
9146   %}
9147   ins_pipe(ialu_reg_reg);
9148 %}
9149 
9150 #ifndef AARCH64
9151 instruct andshlI_reg_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3) %{
9152   match(Set dst (AndI src1 (LShiftI src2 src3)));
9153 
9154   size(4);
9155   format %{ "AND    $dst,$src1,$src2<<$src3" %}
9156   ins_encode %{
9157     __ andr($dst$$Register, $src1$$Register, AsmOperand($src2$$Register, lsl, $src3$$Register));
9158   %}
9159   ins_pipe(ialu_reg_reg);
9160 %}
9161 #endif
9162 
9163 instruct andshlI_reg_reg_imm(iRegI dst, iRegI src1, iRegI src2, immU5 src3) %{
9164   match(Set dst (AndI src1 (LShiftI src2 src3)));
9165 
9166   size(4);
9167   format %{ "and_32 $dst,$src1,$src2<<$src3" %}
9168   ins_encode %{
9169     __ and_32($dst$$Register, $src1$$Register, AsmOperand($src2$$Register, lsl, $src3$$constant));
9170   %}
9171   ins_pipe(ialu_reg_reg);
9172 %}
9173 
9174 #ifndef AARCH64
9175 instruct andsarI_reg_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3) %{
9176   match(Set dst (AndI src1 (RShiftI src2 src3)));
9177 
9178   size(4);
9179   format %{ "AND    $dst,$src1,$src2>>$src3" %}
9180   ins_encode %{
9181     __ andr($dst$$Register, $src1$$Register, AsmOperand($src2$$Register, asr, $src3$$Register));
9182   %}
9183   ins_pipe(ialu_reg_reg);
9184 %}
9185 #endif
9186 
9187 instruct andsarI_reg_reg_imm(iRegI dst, iRegI src1, iRegI src2, immU5 src3) %{
9188   match(Set dst (AndI src1 (RShiftI src2 src3)));
9189 
9190   size(4);
9191   format %{ "and_32 $dst,$src1,$src2>>$src3" %}
9192   ins_encode %{
9193     __ and_32($dst$$Register, $src1$$Register, AsmOperand($src2$$Register, asr, $src3$$constant));
9194   %}
9195   ins_pipe(ialu_reg_reg);
9196 %}
9197 
9198 #ifndef AARCH64
9199 instruct andshrI_reg_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3) %{
9200   match(Set dst (AndI src1 (URShiftI src2 src3)));
9201 
9202   size(4);
9203   format %{ "AND    $dst,$src1,$src2>>>$src3" %}
9204   ins_encode %{
9205     __ andr($dst$$Register, $src1$$Register, AsmOperand($src2$$Register, lsr, $src3$$Register));
9206   %}
9207   ins_pipe(ialu_reg_reg);
9208 %}
9209 #endif
9210 
9211 instruct andshrI_reg_reg_imm(iRegI dst, iRegI src1, iRegI src2, immU5 src3) %{
9212   match(Set dst (AndI src1 (URShiftI src2 src3)));
9213 
9214   size(4);
9215   format %{ "and_32 $dst,$src1,$src2>>>$src3" %}
9216   ins_encode %{
9217     __ and_32($dst$$Register, $src1$$Register, AsmOperand($src2$$Register, lsr, $src3$$constant));
9218   %}
9219   ins_pipe(ialu_reg_reg);
9220 %}
9221 
9222 // Immediate And
9223 instruct andI_reg_limm(iRegI dst, iRegI src1, limmI src2) %{
9224   match(Set dst (AndI src1 src2));
9225 
9226   size(4);
9227   format %{ "and_32 $dst,$src1,$src2\t! int" %}
9228   ins_encode %{
9229     __ and_32($dst$$Register, $src1$$Register, $src2$$constant);
9230   %}
9231   ins_pipe(ialu_reg_imm);
9232 %}
9233 
9234 #ifndef AARCH64
9235 instruct andI_reg_limmn(iRegI dst, iRegI src1, limmIn src2) %{
9236   match(Set dst (AndI src1 src2));
9237 
9238   size(4);
9239   format %{ "bic    $dst,$src1,~$src2\t! int" %}
9240   ins_encode %{
9241     __ bic($dst$$Register, $src1$$Register, ~$src2$$constant);
9242   %}
9243   ins_pipe(ialu_reg_imm);
9244 %}
9245 #endif
9246 
9247 // Register And Long
9248 instruct andL_reg_reg(iRegL dst, iRegL src1, iRegL src2) %{
9249   match(Set dst (AndL src1 src2));
9250 
9251   ins_cost(DEFAULT_COST);
9252 #ifdef AARCH64
9253   size(4);
9254   format %{ "AND    $dst,$src1,$src2\t! long" %}
9255   ins_encode %{
9256     __ andr($dst$$Register, $src1$$Register, $src2$$Register);
9257   %}
9258 #else
9259   size(8);
9260   format %{ "AND    $dst,$src1,$src2\t! long" %}
9261   ins_encode %{
9262     __ andr($dst$$Register, $src1$$Register, $src2$$Register);
9263     __ andr($dst$$Register->successor(), $src1$$Register->successor(), $src2$$Register->successor());
9264   %}
9265 #endif
9266   ins_pipe(ialu_reg_reg);
9267 %}
9268 
9269 #ifdef AARCH64
9270 // Immediate And
9271 instruct andL_reg_limm(iRegL dst, iRegL src1, limmL src2) %{
9272   match(Set dst (AndL src1 src2));
9273 
9274   size(4);
9275   format %{ "AND    $dst,$src1,$src2\t! long" %}
9276   ins_encode %{
9277     __ andr($dst$$Register, $src1$$Register, (uintx)$src2$$constant);
9278   %}
9279   ins_pipe(ialu_reg_imm);
9280 %}
9281 #else
9282 // TODO: try immLRot2 instead, (0, $con$$constant) becomes
9283 // (hi($con$$constant), lo($con$$constant)) becomes
9284 instruct andL_reg_immRot(iRegL dst, iRegL src1, immLlowRot con) %{
9285   match(Set dst (AndL src1 con));
9286   ins_cost(DEFAULT_COST);
9287   size(8);
9288   format %{ "AND    $dst,$src1,$con\t! long" %}
9289   ins_encode %{
9290     __ andr($dst$$Register, $src1$$Register, $con$$constant);
9291     __ andr($dst$$Register->successor(), $src1$$Register->successor(), 0);
9292   %}
9293   ins_pipe(ialu_reg_imm);
9294 %}
9295 #endif
9296 
9297 // Or Instructions
9298 // Register Or
9299 instruct orI_reg_reg(iRegI dst, iRegI src1, iRegI src2) %{
9300   match(Set dst (OrI src1 src2));
9301 
9302   size(4);
9303   format %{ "orr_32 $dst,$src1,$src2\t! int" %}
9304   ins_encode %{
9305     __ orr_32($dst$$Register, $src1$$Register, $src2$$Register);
9306   %}
9307   ins_pipe(ialu_reg_reg);
9308 %}
9309 
9310 #ifndef AARCH64
9311 instruct orshlI_reg_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3) %{
9312   match(Set dst (OrI src1 (LShiftI src2 src3)));
9313 
9314   size(4);
9315   format %{ "OR    $dst,$src1,$src2<<$src3" %}
9316   ins_encode %{
9317     __ orr($dst$$Register, $src1$$Register, AsmOperand($src2$$Register, lsl, $src3$$Register));
9318   %}
9319   ins_pipe(ialu_reg_reg);
9320 %}
9321 #endif
9322 
9323 instruct orshlI_reg_reg_imm(iRegI dst, iRegI src1, iRegI src2, immU5 src3) %{
9324   match(Set dst (OrI src1 (LShiftI src2 src3)));
9325 
9326   size(4);
9327   format %{ "orr_32 $dst,$src1,$src2<<$src3" %}
9328   ins_encode %{
9329     __ orr_32($dst$$Register, $src1$$Register, AsmOperand($src2$$Register, lsl, $src3$$constant));
9330   %}
9331   ins_pipe(ialu_reg_reg);
9332 %}
9333 
9334 #ifndef AARCH64
9335 instruct orsarI_reg_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3) %{
9336   match(Set dst (OrI src1 (RShiftI src2 src3)));
9337 
9338   size(4);
9339   format %{ "OR    $dst,$src1,$src2>>$src3" %}
9340   ins_encode %{
9341     __ orr($dst$$Register, $src1$$Register, AsmOperand($src2$$Register, asr, $src3$$Register));
9342   %}
9343   ins_pipe(ialu_reg_reg);
9344 %}
9345 #endif
9346 
9347 instruct orsarI_reg_reg_imm(iRegI dst, iRegI src1, iRegI src2, immU5 src3) %{
9348   match(Set dst (OrI src1 (RShiftI src2 src3)));
9349 
9350   size(4);
9351   format %{ "orr_32 $dst,$src1,$src2>>$src3" %}
9352   ins_encode %{
9353     __ orr_32($dst$$Register, $src1$$Register, AsmOperand($src2$$Register, asr, $src3$$constant));
9354   %}
9355   ins_pipe(ialu_reg_reg);
9356 %}
9357 
9358 #ifndef AARCH64
9359 instruct orshrI_reg_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3) %{
9360   match(Set dst (OrI src1 (URShiftI src2 src3)));
9361 
9362   size(4);
9363   format %{ "OR    $dst,$src1,$src2>>>$src3" %}
9364   ins_encode %{
9365     __ orr($dst$$Register, $src1$$Register, AsmOperand($src2$$Register, lsr, $src3$$Register));
9366   %}
9367   ins_pipe(ialu_reg_reg);
9368 %}
9369 #endif
9370 
9371 instruct orshrI_reg_reg_imm(iRegI dst, iRegI src1, iRegI src2, immU5 src3) %{
9372   match(Set dst (OrI src1 (URShiftI src2 src3)));
9373 
9374   size(4);
9375   format %{ "orr_32 $dst,$src1,$src2>>>$src3" %}
9376   ins_encode %{
9377     __ orr_32($dst$$Register, $src1$$Register, AsmOperand($src2$$Register, lsr, $src3$$constant));
9378   %}
9379   ins_pipe(ialu_reg_reg);
9380 %}
9381 
9382 // Immediate Or
9383 instruct orI_reg_limm(iRegI dst, iRegI src1, limmI src2) %{
9384   match(Set dst (OrI src1 src2));
9385 
9386   size(4);
9387   format %{ "orr_32  $dst,$src1,$src2" %}
9388   ins_encode %{
9389     __ orr_32($dst$$Register, $src1$$Register, $src2$$constant);
9390   %}
9391   ins_pipe(ialu_reg_imm);
9392 %}
9393 // TODO: orn_32 with limmIn
9394 
9395 // Register Or Long
9396 instruct orL_reg_reg(iRegL dst, iRegL src1, iRegL src2) %{
9397   match(Set dst (OrL src1 src2));
9398 
9399   ins_cost(DEFAULT_COST);
9400 #ifdef AARCH64
9401   size(4);
9402   format %{ "OR     $dst,$src1,$src2\t! long" %}
9403   ins_encode %{
9404     __ orr($dst$$Register, $src1$$Register, $src2$$Register);
9405   %}
9406 #else
9407   size(8);
9408   format %{ "OR     $dst.lo,$src1.lo,$src2.lo\t! long\n\t"
9409             "OR     $dst.hi,$src1.hi,$src2.hi" %}
9410   ins_encode %{
9411     __ orr($dst$$Register, $src1$$Register, $src2$$Register);
9412     __ orr($dst$$Register->successor(), $src1$$Register->successor(), $src2$$Register->successor());
9413   %}
9414 #endif
9415   ins_pipe(ialu_reg_reg);
9416 %}
9417 
9418 #ifdef AARCH64
9419 instruct orL_reg_limm(iRegL dst, iRegL src1, limmL src2) %{
9420   match(Set dst (OrL src1 src2));
9421 
9422   size(4);
9423   format %{ "ORR    $dst,$src1,$src2\t! long" %}
9424   ins_encode %{
9425     __ orr($dst$$Register, $src1$$Register, (uintx)$src2$$constant);
9426   %}
9427   ins_pipe(ialu_reg_imm);
9428 %}
9429 #else
9430 // TODO: try immLRot2 instead, (0, $con$$constant) becomes
9431 // (hi($con$$constant), lo($con$$constant)) becomes
9432 instruct orL_reg_immRot(iRegL dst, iRegL src1, immLlowRot con) %{
9433   match(Set dst (OrL src1 con));
9434   ins_cost(DEFAULT_COST);
9435   size(8);
9436   format %{ "OR     $dst.lo,$src1.lo,$con\t! long\n\t"
9437             "OR     $dst.hi,$src1.hi,$con" %}
9438   ins_encode %{
9439     __ orr($dst$$Register, $src1$$Register, $con$$constant);
9440     __ orr($dst$$Register->successor(), $src1$$Register->successor(), 0);
9441   %}
9442   ins_pipe(ialu_reg_imm);
9443 %}
9444 #endif
9445 
9446 #ifdef TODO
9447 // Use SPRegP to match Rthread (TLS register) without spilling.
9448 // Use store_ptr_RegP to match Rthread (TLS register) without spilling.
9449 // Use sp_ptr_RegP to match Rthread (TLS register) without spilling.
9450 instruct orI_reg_castP2X(iRegI dst, iRegI src1, sp_ptr_RegP src2) %{
9451   match(Set dst (OrI src1 (CastP2X src2)));
9452   size(4);
9453   format %{ "OR     $dst,$src1,$src2" %}
9454   ins_encode %{
9455     __ orr($dst$$Register, $src1$$Register, $src2$$Register);
9456   %}
9457   ins_pipe(ialu_reg_reg);
9458 %}
9459 #endif
9460 
9461 // Xor Instructions
9462 // Register Xor
9463 instruct xorI_reg_reg(iRegI dst, iRegI src1, iRegI src2) %{
9464   match(Set dst (XorI src1 src2));
9465 
9466   size(4);
9467   format %{ "eor_32 $dst,$src1,$src2" %}
9468   ins_encode %{
9469     __ eor_32($dst$$Register, $src1$$Register, $src2$$Register);
9470   %}
9471   ins_pipe(ialu_reg_reg);
9472 %}
9473 
9474 #ifndef AARCH64
9475 instruct xorshlI_reg_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3) %{
9476   match(Set dst (XorI src1 (LShiftI src2 src3)));
9477 
9478   size(4);
9479   format %{ "XOR    $dst,$src1,$src2<<$src3" %}
9480   ins_encode %{
9481     __ eor($dst$$Register, $src1$$Register, AsmOperand($src2$$Register, lsl, $src3$$Register));
9482   %}
9483   ins_pipe(ialu_reg_reg);
9484 %}
9485 #endif
9486 
9487 instruct xorshlI_reg_reg_imm(iRegI dst, iRegI src1, iRegI src2, immU5 src3) %{
9488   match(Set dst (XorI src1 (LShiftI src2 src3)));
9489 
9490   size(4);
9491   format %{ "eor_32 $dst,$src1,$src2<<$src3" %}
9492   ins_encode %{
9493     __ eor_32($dst$$Register, $src1$$Register, AsmOperand($src2$$Register, lsl, $src3$$constant));
9494   %}
9495   ins_pipe(ialu_reg_reg);
9496 %}
9497 
9498 #ifndef AARCH64
9499 instruct xorsarI_reg_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3) %{
9500   match(Set dst (XorI src1 (RShiftI src2 src3)));
9501 
9502   size(4);
9503   format %{ "XOR    $dst,$src1,$src2>>$src3" %}
9504   ins_encode %{
9505     __ eor($dst$$Register, $src1$$Register, AsmOperand($src2$$Register, asr, $src3$$Register));
9506   %}
9507   ins_pipe(ialu_reg_reg);
9508 %}
9509 #endif
9510 
9511 instruct xorsarI_reg_reg_imm(iRegI dst, iRegI src1, iRegI src2, immU5 src3) %{
9512   match(Set dst (XorI src1 (RShiftI src2 src3)));
9513 
9514   size(4);
9515   format %{ "eor_32 $dst,$src1,$src2>>$src3" %}
9516   ins_encode %{
9517     __ eor_32($dst$$Register, $src1$$Register, AsmOperand($src2$$Register, asr, $src3$$constant));
9518   %}
9519   ins_pipe(ialu_reg_reg);
9520 %}
9521 
9522 #ifndef AARCH64
9523 instruct xorshrI_reg_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3) %{
9524   match(Set dst (XorI src1 (URShiftI src2 src3)));
9525 
9526   size(4);
9527   format %{ "XOR    $dst,$src1,$src2>>>$src3" %}
9528   ins_encode %{
9529     __ eor($dst$$Register, $src1$$Register, AsmOperand($src2$$Register, lsr, $src3$$Register));
9530   %}
9531   ins_pipe(ialu_reg_reg);
9532 %}
9533 #endif
9534 
9535 instruct xorshrI_reg_reg_imm(iRegI dst, iRegI src1, iRegI src2, immU5 src3) %{
9536   match(Set dst (XorI src1 (URShiftI src2 src3)));
9537 
9538   size(4);
9539   format %{ "eor_32 $dst,$src1,$src2>>>$src3" %}
9540   ins_encode %{
9541     __ eor_32($dst$$Register, $src1$$Register, AsmOperand($src2$$Register, lsr, $src3$$constant));
9542   %}
9543   ins_pipe(ialu_reg_reg);
9544 %}
9545 
9546 // Immediate Xor
9547 instruct xorI_reg_imm(iRegI dst, iRegI src1, limmI src2) %{
9548   match(Set dst (XorI src1 src2));
9549 
9550   size(4);
9551   format %{ "eor_32 $dst,$src1,$src2" %}
9552   ins_encode %{
9553     __ eor_32($dst$$Register, $src1$$Register, $src2$$constant);
9554   %}
9555   ins_pipe(ialu_reg_imm);
9556 %}
9557 
9558 // Register Xor Long
9559 instruct xorL_reg_reg(iRegL dst, iRegL src1, iRegL src2) %{
9560   match(Set dst (XorL src1 src2));
9561   ins_cost(DEFAULT_COST);
9562 #ifdef AARCH64
9563   size(4);
9564   format %{ "XOR     $dst,$src1,$src2\t! long" %}
9565   ins_encode %{
9566     __ eor($dst$$Register, $src1$$Register, $src2$$Register);
9567   %}
9568 #else
9569   size(8);
9570   format %{ "XOR     $dst.hi,$src1.hi,$src2.hi\t! long\n\t"
9571             "XOR     $dst.lo,$src1.lo,$src2.lo\t! long" %}
9572   ins_encode %{
9573     __ eor($dst$$Register, $src1$$Register, $src2$$Register);
9574     __ eor($dst$$Register->successor(), $src1$$Register->successor(), $src2$$Register->successor());
9575   %}
9576 #endif
9577   ins_pipe(ialu_reg_reg);
9578 %}
9579 
9580 #ifdef AARCH64
9581 instruct xorL_reg_limmL(iRegL dst, iRegL src1, limmL con) %{
9582   match(Set dst (XorL src1 con));
9583   ins_cost(DEFAULT_COST);
9584   size(4);
9585   format %{ "EOR     $dst,$src1,$con\t! long" %}
9586   ins_encode %{
9587     __ eor($dst$$Register, $src1$$Register, (uintx)$con$$constant);
9588   %}
9589   ins_pipe(ialu_reg_imm);
9590 %}
9591 #else
9592 // TODO: try immLRot2 instead, (0, $con$$constant) becomes
9593 // (hi($con$$constant), lo($con$$constant)) becomes
9594 instruct xorL_reg_immRot(iRegL dst, iRegL src1, immLlowRot con) %{
9595   match(Set dst (XorL src1 con));
9596   ins_cost(DEFAULT_COST);
9597   size(8);
9598   format %{ "XOR     $dst.hi,$src1.hi,$con\t! long\n\t"
9599             "XOR     $dst.lo,$src1.lo,0\t! long" %}
9600   ins_encode %{
9601     __ eor($dst$$Register, $src1$$Register, $con$$constant);
9602     __ eor($dst$$Register->successor(), $src1$$Register->successor(), 0);
9603   %}
9604   ins_pipe(ialu_reg_imm);
9605 %}
9606 #endif // AARCH64
9607 
9608 //----------Convert to Boolean-------------------------------------------------
9609 instruct convI2B( iRegI dst, iRegI src, flagsReg ccr ) %{
9610   match(Set dst (Conv2B src));
9611   effect(KILL ccr);
9612 #ifdef AARCH64
9613   size(8);
9614   ins_cost(DEFAULT_COST*2);
9615   format %{ "cmp_32 $src,ZR\n\t"
9616             "cset_w $dst, ne" %}
9617   ins_encode %{
9618     __ cmp_32($src$$Register, ZR);
9619     __ cset_w($dst$$Register, ne);
9620   %}
9621 #else
9622   size(12);
9623   ins_cost(DEFAULT_COST*2);
9624   format %{ "TST    $src,$src \n\t"
9625             "MOV    $dst, 0   \n\t"
9626             "MOV.ne $dst, 1" %}
9627   ins_encode %{ // FIXME: can do better?
9628     __ tst($src$$Register, $src$$Register);
9629     __ mov($dst$$Register, 0);
9630     __ mov($dst$$Register, 1, ne);
9631   %}
9632 #endif
9633   ins_pipe(ialu_reg_ialu);
9634 %}
9635 
9636 instruct convP2B( iRegI dst, iRegP src, flagsReg ccr ) %{
9637   match(Set dst (Conv2B src));
9638   effect(KILL ccr);
9639 #ifdef AARCH64
9640   size(8);
9641   ins_cost(DEFAULT_COST*2);
9642   format %{ "CMP    $src,ZR\n\t"
9643             "cset   $dst, ne" %}
9644   ins_encode %{
9645     __ cmp($src$$Register, ZR);
9646     __ cset($dst$$Register, ne);
9647   %}
9648 #else
9649   size(12);
9650   ins_cost(DEFAULT_COST*2);
9651   format %{ "TST    $src,$src \n\t"
9652             "MOV    $dst, 0   \n\t"
9653             "MOV.ne $dst, 1" %}
9654   ins_encode %{
9655     __ tst($src$$Register, $src$$Register);
9656     __ mov($dst$$Register, 0);
9657     __ mov($dst$$Register, 1, ne);
9658   %}
9659 #endif
9660   ins_pipe(ialu_reg_ialu);
9661 %}
9662 
9663 instruct cmpLTMask_reg_reg( iRegI dst, iRegI p, iRegI q, flagsReg ccr ) %{
9664   match(Set dst (CmpLTMask p q));
9665   effect( KILL ccr );
9666 #ifdef AARCH64
9667   size(8);
9668   ins_cost(DEFAULT_COST*2);
9669   format %{ "CMP_w   $p,$q\n\t"
9670             "CSETM_w $dst, lt" %}
9671   ins_encode %{
9672     __ cmp_w($p$$Register, $q$$Register);
9673     __ csetm_w($dst$$Register, lt);
9674   %}
9675 #else
9676   ins_cost(DEFAULT_COST*3);
9677   format %{ "CMP    $p,$q\n\t"
9678             "MOV    $dst, #0\n\t"
9679             "MOV.lt $dst, #-1" %}
9680   ins_encode %{
9681     __ cmp($p$$Register, $q$$Register);
9682     __ mov($dst$$Register, 0);
9683     __ mvn($dst$$Register, 0, lt);
9684   %}
9685 #endif
9686   ins_pipe(ialu_reg_reg_ialu);
9687 %}
9688 
9689 instruct cmpLTMask_reg_imm( iRegI dst, iRegI p, aimmI q, flagsReg ccr ) %{
9690   match(Set dst (CmpLTMask p q));
9691   effect( KILL ccr );
9692 #ifdef AARCH64
9693   size(8);
9694   ins_cost(DEFAULT_COST*2);
9695   format %{ "CMP_w   $p,$q\n\t"
9696             "CSETM_w $dst, lt" %}
9697   ins_encode %{
9698     __ cmp_w($p$$Register, $q$$constant);
9699     __ csetm_w($dst$$Register, lt);
9700   %}
9701 #else
9702   ins_cost(DEFAULT_COST*3);
9703   format %{ "CMP    $p,$q\n\t"
9704             "MOV    $dst, #0\n\t"
9705             "MOV.lt $dst, #-1" %}
9706   ins_encode %{
9707     __ cmp($p$$Register, $q$$constant);
9708     __ mov($dst$$Register, 0);
9709     __ mvn($dst$$Register, 0, lt);
9710   %}
9711 #endif
9712   ins_pipe(ialu_reg_reg_ialu);
9713 %}
9714 
9715 #ifdef AARCH64
9716 instruct cadd_cmpLTMask3( iRegI dst, iRegI p, iRegI q, iRegI y, iRegI x, flagsReg ccr ) %{
9717   match(Set dst (AddI (AndI (CmpLTMask p q) y) x));
9718   effect( TEMP dst, KILL ccr );
9719   size(12);
9720   ins_cost(DEFAULT_COST*3);
9721   format %{ "CMP_w  $p,$q\n\t"
9722             "ADD_w  $dst,$y,$x\n\t"
9723             "CSEL_w $dst,$dst,$x,lt" %}
9724   ins_encode %{
9725     __ cmp_w($p$$Register, $q$$Register);
9726     __ add_w($dst$$Register, $y$$Register, $x$$Register);
9727     __ csel_w($dst$$Register, $dst$$Register, $x$$Register, lt);
9728   %}
9729   ins_pipe( cadd_cmpltmask );
9730 %}
9731 #else
9732 instruct cadd_cmpLTMask3( iRegI p, iRegI q, iRegI y, iRegI z, flagsReg ccr ) %{
9733   match(Set z (AddI (AndI (CmpLTMask p q) y) z));
9734   effect( KILL ccr );
9735   ins_cost(DEFAULT_COST*2);
9736   format %{ "CMP    $p,$q\n\t"
9737             "ADD.lt $z,$y,$z" %}
9738   ins_encode %{
9739     __ cmp($p$$Register, $q$$Register);
9740     __ add($z$$Register, $y$$Register, $z$$Register, lt);
9741   %}
9742   ins_pipe( cadd_cmpltmask );
9743 %}
9744 #endif
9745 
9746 #ifdef AARCH64
9747 instruct cadd_cmpLTMask4( iRegI dst, iRegI p, aimmI q, iRegI y, iRegI x, flagsReg ccr ) %{
9748   match(Set dst (AddI (AndI (CmpLTMask p q) y) x));
9749   effect( TEMP dst, KILL ccr );
9750   size(12);
9751   ins_cost(DEFAULT_COST*3);
9752   format %{ "CMP_w  $p,$q\n\t"
9753             "ADD_w  $dst,$y,$x\n\t"
9754             "CSEL_w $dst,$dst,$x,lt" %}
9755   ins_encode %{
9756     __ cmp_w($p$$Register, $q$$constant);
9757     __ add_w($dst$$Register, $y$$Register, $x$$Register);
9758     __ csel_w($dst$$Register, $dst$$Register, $x$$Register, lt);
9759   %}
9760   ins_pipe( cadd_cmpltmask );
9761 %}
9762 #else
9763 // FIXME: remove unused "dst"
9764 instruct cadd_cmpLTMask4( iRegI dst, iRegI p, aimmI q, iRegI y, iRegI z, flagsReg ccr ) %{
9765   match(Set z (AddI (AndI (CmpLTMask p q) y) z));
9766   effect( KILL ccr );
9767   ins_cost(DEFAULT_COST*2);
9768   format %{ "CMP    $p,$q\n\t"
9769             "ADD.lt $z,$y,$z" %}
9770   ins_encode %{
9771     __ cmp($p$$Register, $q$$constant);
9772     __ add($z$$Register, $y$$Register, $z$$Register, lt);
9773   %}
9774   ins_pipe( cadd_cmpltmask );
9775 %}
9776 #endif // !AARCH64
9777 
9778 #ifdef AARCH64
9779 instruct cadd_cmpLTMask( iRegI dst, iRegI p, iRegI q, iRegI y, flagsReg ccr ) %{
9780   match(Set dst (AddI (AndI (CmpLTMask p q) y) (SubI p q)));
9781   effect( TEMP dst, KILL ccr );
9782   size(12);
9783   ins_cost(DEFAULT_COST*3);
9784   format %{ "SUBS_w $p,$p,$q\n\t"
9785             "ADD_w  $dst,$y,$p\n\t"
9786             "CSEL_w $dst,$dst,$p,lt" %}
9787   ins_encode %{
9788     __ subs_w($p$$Register, $p$$Register, $q$$Register);
9789     __ add_w($dst$$Register, $y$$Register, $p$$Register);
9790     __ csel_w($dst$$Register, $dst$$Register, $p$$Register, lt);
9791   %}
9792   ins_pipe( cadd_cmpltmask ); // FIXME
9793 %}
9794 #else
9795 instruct cadd_cmpLTMask( iRegI p, iRegI q, iRegI y, flagsReg ccr ) %{
9796   match(Set p (AddI (AndI (CmpLTMask p q) y) (SubI p q)));
9797   effect( KILL ccr );
9798   ins_cost(DEFAULT_COST*2);
9799   format %{ "SUBS   $p,$p,$q\n\t"
9800             "ADD.lt $p,$y,$p" %}
9801   ins_encode %{
9802     __ subs($p$$Register, $p$$Register, $q$$Register);
9803     __ add($p$$Register, $y$$Register, $p$$Register, lt);
9804   %}
9805   ins_pipe( cadd_cmpltmask );
9806 %}
9807 #endif
9808 
9809 //----------Arithmetic Conversion Instructions---------------------------------
9810 // The conversions operations are all Alpha sorted.  Please keep it that way!
9811 
9812 instruct convD2F_reg(regF dst, regD src) %{
9813   match(Set dst (ConvD2F src));
9814   size(4);
9815   format %{ "FCVTSD  $dst,$src" %}
9816   ins_encode %{
9817     __ convert_d2f($dst$$FloatRegister, $src$$FloatRegister);
9818   %}
9819   ins_pipe(fcvtD2F);
9820 %}
9821 
9822 // Convert a double to an int in a float register.
9823 // If the double is a NAN, stuff a zero in instead.
9824 
9825 #ifdef AARCH64
9826 instruct convD2I_reg_reg(iRegI dst, regD src) %{
9827   match(Set dst (ConvD2I src));
9828   ins_cost(DEFAULT_COST*2 + MEMORY_REF_COST*2 + BRANCH_COST); // FIXME
9829   format %{ "FCVTZS_wd $dst, $src" %}
9830   ins_encode %{
9831     __ fcvtzs_wd($dst$$Register, $src$$FloatRegister);
9832   %}
9833   ins_pipe(fcvtD2I);
9834 %}
9835 
9836 instruct convD2L_reg_reg(iRegL dst, regD src) %{
9837   match(Set dst (ConvD2L src));
9838   ins_cost(DEFAULT_COST*2 + MEMORY_REF_COST*2 + BRANCH_COST); // FIXME
9839   format %{ "FCVTZS_xd $dst, $src" %}
9840   ins_encode %{
9841     __ fcvtzs_xd($dst$$Register, $src$$FloatRegister);
9842   %}
9843   ins_pipe(fcvtD2L);
9844 %}
9845 #else
9846 instruct convD2I_reg_reg(iRegI dst, regD src, regF tmp) %{
9847   match(Set dst (ConvD2I src));
9848   effect( TEMP tmp );
9849   ins_cost(DEFAULT_COST*2 + MEMORY_REF_COST*2 + BRANCH_COST); // FIXME
9850   format %{ "FTOSIZD  $tmp,$src\n\t"
9851             "FMRS     $dst, $tmp" %}
9852   ins_encode %{
9853     __ ftosizd($tmp$$FloatRegister, $src$$FloatRegister);
9854     __ fmrs($dst$$Register, $tmp$$FloatRegister);
9855   %}
9856   ins_pipe(fcvtD2I);
9857 %}
9858 #endif
9859 
9860 // Convert a double to a long in a double register.
9861 // If the double is a NAN, stuff a zero in instead.
9862 
9863 #ifndef AARCH64
9864 // Double to Long conversion
9865 instruct convD2L_reg(R0R1RegL dst, regD src) %{
9866   match(Set dst (ConvD2L src));
9867   effect(CALL);
9868   ins_cost(MEMORY_REF_COST); // FIXME
9869   format %{ "convD2L    $dst,$src\t ! call to SharedRuntime::d2l" %}
9870   ins_encode %{
9871 #ifndef __ABI_HARD__
9872     __ fmrrd($dst$$Register, $dst$$Register->successor(), $src$$FloatRegister);
9873 #else
9874     if ($src$$FloatRegister != D0) {
9875       __ mov_double(D0, $src$$FloatRegister);
9876     }
9877 #endif
9878     address target = CAST_FROM_FN_PTR(address, SharedRuntime::d2l);
9879     __ call(target, relocInfo::runtime_call_type);
9880   %}
9881   ins_pipe(fcvtD2L);
9882 %}
9883 #endif
9884 
9885 instruct convF2D_reg(regD dst, regF src) %{
9886   match(Set dst (ConvF2D src));
9887   size(4);
9888   format %{ "FCVTDS  $dst,$src" %}
9889   ins_encode %{
9890     __ convert_f2d($dst$$FloatRegister, $src$$FloatRegister);
9891   %}
9892   ins_pipe(fcvtF2D);
9893 %}
9894 
9895 #ifdef AARCH64
9896 instruct convF2I_reg_reg(iRegI dst, regF src) %{
9897   match(Set dst (ConvF2I src));
9898   ins_cost(DEFAULT_COST*2 + MEMORY_REF_COST*2 + BRANCH_COST); // FIXME
9899   size(4);
9900   format %{ "FCVTZS_ws $dst, $src" %}
9901   ins_encode %{
9902     __ fcvtzs_ws($dst$$Register, $src$$FloatRegister);
9903   %}
9904   ins_pipe(fcvtF2I);
9905 %}
9906 
9907 instruct convF2L_reg_reg(iRegL dst, regF src) %{
9908   match(Set dst (ConvF2L src));
9909   ins_cost(DEFAULT_COST*2 + MEMORY_REF_COST*2 + BRANCH_COST); // FIXME
9910   size(4);
9911   format %{ "FCVTZS_xs $dst, $src" %}
9912   ins_encode %{
9913     __ fcvtzs_xs($dst$$Register, $src$$FloatRegister);
9914   %}
9915   ins_pipe(fcvtF2L);
9916 %}
9917 #else
9918 instruct convF2I_reg_reg(iRegI dst, regF src, regF tmp) %{
9919   match(Set dst (ConvF2I src));
9920   effect( TEMP tmp );
9921   ins_cost(DEFAULT_COST*2 + MEMORY_REF_COST*2 + BRANCH_COST); // FIXME
9922   size(8);
9923   format %{ "FTOSIZS  $tmp,$src\n\t"
9924             "FMRS     $dst, $tmp" %}
9925   ins_encode %{
9926     __ ftosizs($tmp$$FloatRegister, $src$$FloatRegister);
9927     __ fmrs($dst$$Register, $tmp$$FloatRegister);
9928   %}
9929   ins_pipe(fcvtF2I);
9930 %}
9931 
9932 // Float to Long conversion
9933 instruct convF2L_reg(R0R1RegL dst, regF src, R0RegI arg1) %{
9934   match(Set dst (ConvF2L src));
9935   ins_cost(DEFAULT_COST*2 + MEMORY_REF_COST*2 + BRANCH_COST); // FIXME
9936   effect(CALL);
9937   format %{ "convF2L  $dst,$src\t! call to SharedRuntime::f2l" %}
9938   ins_encode %{
9939 #ifndef __ABI_HARD__
9940     __ fmrs($arg1$$Register, $src$$FloatRegister);
9941 #else
9942     if($src$$FloatRegister != S0) {
9943       __ mov_float(S0, $src$$FloatRegister);
9944     }
9945 #endif
9946     address target = CAST_FROM_FN_PTR(address, SharedRuntime::f2l);
9947     __ call(target, relocInfo::runtime_call_type);
9948   %}
9949   ins_pipe(fcvtF2L);
9950 %}
9951 #endif
9952 
9953 #ifdef AARCH64
9954 instruct convI2D_reg_reg(iRegI src, regD dst) %{
9955   match(Set dst (ConvI2D src));
9956   ins_cost(DEFAULT_COST + MEMORY_REF_COST); // FIXME
9957   size(4);
9958   format %{ "SCVTF_dw $dst,$src" %}
9959   ins_encode %{
9960       __ scvtf_dw($dst$$FloatRegister, $src$$Register);
9961   %}
9962   ins_pipe(fcvtI2D);
9963 %}
9964 #else
9965 instruct convI2D_reg_reg(iRegI src, regD_low dst) %{
9966   match(Set dst (ConvI2D src));
9967   ins_cost(DEFAULT_COST + MEMORY_REF_COST); // FIXME
9968   size(8);
9969   format %{ "FMSR     $dst,$src \n\t"
9970             "FSITOD   $dst $dst"%}
9971   ins_encode %{
9972       __ fmsr($dst$$FloatRegister, $src$$Register);
9973       __ fsitod($dst$$FloatRegister, $dst$$FloatRegister);
9974   %}
9975   ins_pipe(fcvtI2D);
9976 %}
9977 #endif
9978 
9979 instruct convI2F_reg_reg( regF dst, iRegI src ) %{
9980   match(Set dst (ConvI2F src));
9981   ins_cost(DEFAULT_COST + MEMORY_REF_COST); // FIXME
9982 #ifdef AARCH64
9983   size(4);
9984   format %{ "SCVTF_sw $dst,$src" %}
9985   ins_encode %{
9986       __ scvtf_sw($dst$$FloatRegister, $src$$Register);
9987   %}
9988 #else
9989   size(8);
9990   format %{ "FMSR     $dst,$src \n\t"
9991             "FSITOS   $dst, $dst"%}
9992   ins_encode %{
9993       __ fmsr($dst$$FloatRegister, $src$$Register);
9994       __ fsitos($dst$$FloatRegister, $dst$$FloatRegister);
9995   %}
9996 #endif
9997   ins_pipe(fcvtI2F);
9998 %}
9999 
10000 instruct convI2L_reg(iRegL dst, iRegI src) %{
10001   match(Set dst (ConvI2L src));
10002 #ifdef AARCH64
10003   size(4);
10004   format %{ "SXTW   $dst,$src\t! int->long" %}
10005   ins_encode %{
10006     __ sxtw($dst$$Register, $src$$Register);
10007   %}
10008 #else
10009   size(8);
10010   format %{ "MOV    $dst.lo, $src \n\t"
10011             "ASR    $dst.hi,$src,31\t! int->long" %}
10012   ins_encode %{
10013     __ mov($dst$$Register, $src$$Register);
10014     __ mov($dst$$Register->successor(), AsmOperand($src$$Register, asr, 31));
10015   %}
10016 #endif
10017   ins_pipe(ialu_reg_reg);
10018 %}
10019 
10020 // Zero-extend convert int to long
10021 instruct convI2L_reg_zex(iRegL dst, iRegI src, immL_32bits mask ) %{
10022   match(Set dst (AndL (ConvI2L src) mask) );
10023 #ifdef AARCH64
10024   size(4);
10025   format %{ "mov_w  $dst,$src\t! zero-extend int to long"  %}
10026   ins_encode %{
10027     __ mov_w($dst$$Register, $src$$Register);
10028   %}
10029 #else
10030   size(8);
10031   format %{ "MOV    $dst.lo,$src.lo\t! zero-extend int to long\n\t"
10032             "MOV    $dst.hi, 0"%}
10033   ins_encode %{
10034     __ mov($dst$$Register, $src$$Register);
10035     __ mov($dst$$Register->successor(), 0);
10036   %}
10037 #endif
10038   ins_pipe(ialu_reg_reg);
10039 %}
10040 
10041 // Zero-extend long
10042 instruct zerox_long(iRegL dst, iRegL src, immL_32bits mask ) %{
10043   match(Set dst (AndL src mask) );
10044 #ifdef AARCH64
10045   size(4);
10046   format %{ "mov_w  $dst,$src\t! zero-extend long"  %}
10047   ins_encode %{
10048     __ mov_w($dst$$Register, $src$$Register);
10049   %}
10050 #else
10051   size(8);
10052   format %{ "MOV    $dst.lo,$src.lo\t! zero-extend long\n\t"
10053             "MOV    $dst.hi, 0"%}
10054   ins_encode %{
10055     __ mov($dst$$Register, $src$$Register);
10056     __ mov($dst$$Register->successor(), 0);
10057   %}
10058 #endif
10059   ins_pipe(ialu_reg_reg);
10060 %}
10061 
10062 instruct MoveF2I_reg_reg(iRegI dst, regF src) %{
10063   match(Set dst (MoveF2I src));
10064   effect(DEF dst, USE src);
10065   ins_cost(MEMORY_REF_COST); // FIXME
10066 
10067   size(4);
10068   format %{ "FMRS   $dst,$src\t! MoveF2I" %}
10069   ins_encode %{
10070     __ fmrs($dst$$Register, $src$$FloatRegister);
10071   %}
10072   ins_pipe(iload_mem); // FIXME
10073 %}
10074 
10075 instruct MoveI2F_reg_reg(regF dst, iRegI src) %{
10076   match(Set dst (MoveI2F src));
10077   ins_cost(MEMORY_REF_COST); // FIXME
10078 
10079   size(4);
10080   format %{ "FMSR   $dst,$src\t! MoveI2F" %}
10081   ins_encode %{
10082     __ fmsr($dst$$FloatRegister, $src$$Register);
10083   %}
10084   ins_pipe(iload_mem); // FIXME
10085 %}
10086 
10087 instruct MoveD2L_reg_reg(iRegL dst, regD src) %{
10088   match(Set dst (MoveD2L src));
10089   effect(DEF dst, USE src);
10090   ins_cost(MEMORY_REF_COST); // FIXME
10091 
10092   size(4);
10093 #ifdef AARCH64
10094   format %{ "FMOV_xd  $dst,$src\t! MoveD2L" %}
10095   ins_encode %{
10096     __ fmov_xd($dst$$Register, $src$$FloatRegister);
10097   %}
10098 #else
10099   format %{ "FMRRD    $dst,$src\t! MoveD2L" %}
10100   ins_encode %{
10101     __ fmrrd($dst$$Register, $dst$$Register->successor(), $src$$FloatRegister);
10102   %}
10103 #endif
10104   ins_pipe(iload_mem); // FIXME
10105 %}
10106 
10107 instruct MoveL2D_reg_reg(regD dst, iRegL src) %{
10108   match(Set dst (MoveL2D src));
10109   effect(DEF dst, USE src);
10110   ins_cost(MEMORY_REF_COST); // FIXME
10111 
10112   size(4);
10113 #ifdef AARCH64
10114   format %{ "FMOV_dx $dst,$src\t! MoveL2D" %}
10115   ins_encode %{
10116     __ fmov_dx($dst$$FloatRegister, $src$$Register);
10117   %}
10118 #else
10119   format %{ "FMDRR   $dst,$src\t! MoveL2D" %}
10120   ins_encode %{
10121     __ fmdrr($dst$$FloatRegister, $src$$Register, $src$$Register->successor());
10122   %}
10123 #endif
10124   ins_pipe(ialu_reg_reg); // FIXME
10125 %}
10126 
10127 //-----------
10128 // Long to Double conversion
10129 
10130 #ifdef AARCH64
10131 instruct convL2D(regD dst, iRegL src) %{
10132   match(Set dst (ConvL2D src));
10133   ins_cost(DEFAULT_COST*2 + MEMORY_REF_COST*2 + BRANCH_COST); // FIXME
10134   size(4);
10135   format %{ "SCVTF_dx $dst, $src" %}
10136   ins_encode %{
10137     __ scvtf_dx($dst$$FloatRegister, $src$$Register);
10138   %}
10139   ins_pipe(fcvtL2D);
10140 %}
10141 
10142 instruct convL2F(regF dst, iRegL src) %{
10143   match(Set dst (ConvL2F src));
10144   ins_cost(DEFAULT_COST*2 + MEMORY_REF_COST*2 + BRANCH_COST); // FIXME
10145   size(4);
10146   format %{ "SCVTF_sx $dst, $src" %}
10147   ins_encode %{
10148     __ scvtf_sx($dst$$FloatRegister, $src$$Register);
10149   %}
10150   ins_pipe(fcvtL2F);
10151 %}
10152 #else
10153 // Magic constant, 0x43300000
10154 instruct loadConI_x43300000(iRegI dst) %{
10155   effect(DEF dst);
10156   size(8);
10157   format %{ "MOV_SLOW  $dst,0x43300000\t! 2^52" %}
10158   ins_encode %{
10159     __ mov_slow($dst$$Register, 0x43300000);
10160   %}
10161   ins_pipe(ialu_none);
10162 %}
10163 
10164 // Magic constant, 0x41f00000
10165 instruct loadConI_x41f00000(iRegI dst) %{
10166   effect(DEF dst);
10167   size(8);
10168   format %{ "MOV_SLOW  $dst, 0x41f00000\t! 2^32" %}
10169   ins_encode %{
10170     __ mov_slow($dst$$Register, 0x41f00000);
10171   %}
10172   ins_pipe(ialu_none);
10173 %}
10174 
10175 instruct loadConI_x0(iRegI dst) %{
10176   effect(DEF dst);
10177   size(4);
10178   format %{ "MOV  $dst, 0x0\t! 0" %}
10179   ins_encode %{
10180     __ mov($dst$$Register, 0);
10181   %}
10182   ins_pipe(ialu_none);
10183 %}
10184 
10185 // Construct a double from two float halves
10186 instruct regDHi_regDLo_to_regD(regD_low dst, regD_low src1, regD_low src2) %{
10187   effect(DEF dst, USE src1, USE src2);
10188   size(8);
10189   format %{ "FCPYS  $dst.hi,$src1.hi\n\t"
10190             "FCPYS  $dst.lo,$src2.lo" %}
10191   ins_encode %{
10192     __ fcpys($dst$$FloatRegister->successor(), $src1$$FloatRegister->successor());
10193     __ fcpys($dst$$FloatRegister, $src2$$FloatRegister);
10194   %}
10195   ins_pipe(faddD_reg_reg);
10196 %}
10197 
10198 #ifndef AARCH64
10199 // Convert integer in high half of a double register (in the lower half of
10200 // the double register file) to double
10201 instruct convI2D_regDHi_regD(regD dst, regD_low src) %{
10202   effect(DEF dst, USE src);
10203   size(4);
10204   format %{ "FSITOD  $dst,$src" %}
10205   ins_encode %{
10206     __ fsitod($dst$$FloatRegister, $src$$FloatRegister->successor());
10207   %}
10208   ins_pipe(fcvtLHi2D);
10209 %}
10210 #endif
10211 
10212 // Add float double precision
10213 instruct addD_regD_regD(regD dst, regD src1, regD src2) %{
10214   effect(DEF dst, USE src1, USE src2);
10215   size(4);
10216   format %{ "FADDD  $dst,$src1,$src2" %}
10217   ins_encode %{
10218     __ add_double($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister);
10219   %}
10220   ins_pipe(faddD_reg_reg);
10221 %}
10222 
10223 // Sub float double precision
10224 instruct subD_regD_regD(regD dst, regD src1, regD src2) %{
10225   effect(DEF dst, USE src1, USE src2);
10226   size(4);
10227   format %{ "FSUBD  $dst,$src1,$src2" %}
10228   ins_encode %{
10229     __ sub_double($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister);
10230   %}
10231   ins_pipe(faddD_reg_reg);
10232 %}
10233 
10234 // Mul float double precision
10235 instruct mulD_regD_regD(regD dst, regD src1, regD src2) %{
10236   effect(DEF dst, USE src1, USE src2);
10237   size(4);
10238   format %{ "FMULD  $dst,$src1,$src2" %}
10239   ins_encode %{
10240     __ mul_double($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister);
10241   %}
10242   ins_pipe(fmulD_reg_reg);
10243 %}
10244 
10245 instruct regL_to_regD(regD dst, iRegL src) %{
10246   // No match rule to avoid chain rule match.
10247   effect(DEF dst, USE src);
10248   ins_cost(MEMORY_REF_COST);
10249   size(4);
10250   format %{ "FMDRR   $dst,$src\t! regL to regD" %}
10251   ins_encode %{
10252     __ fmdrr($dst$$FloatRegister, $src$$Register, $src$$Register->successor());
10253   %}
10254   ins_pipe(ialu_reg_reg); // FIXME
10255 %}
10256 
10257 instruct regI_regI_to_regD(regD dst, iRegI src1, iRegI src2) %{
10258   // No match rule to avoid chain rule match.
10259   effect(DEF dst, USE src1, USE src2);
10260   ins_cost(MEMORY_REF_COST);
10261   size(4);
10262   format %{ "FMDRR   $dst,$src1,$src2\t! regI,regI to regD" %}
10263   ins_encode %{
10264     __ fmdrr($dst$$FloatRegister, $src1$$Register, $src2$$Register);
10265   %}
10266   ins_pipe(ialu_reg_reg); // FIXME
10267 %}
10268 
10269 instruct convL2D_reg_slow_fxtof(regD dst, iRegL src) %{
10270   match(Set dst (ConvL2D src));
10271   ins_cost(DEFAULT_COST*8 + MEMORY_REF_COST*6); // FIXME
10272 
10273   expand %{
10274     regD_low   tmpsrc;
10275     iRegI      ix43300000;
10276     iRegI      ix41f00000;
10277     iRegI      ix0;
10278     regD_low   dx43300000;
10279     regD       dx41f00000;
10280     regD       tmp1;
10281     regD_low   tmp2;
10282     regD       tmp3;
10283     regD       tmp4;
10284 
10285     regL_to_regD(tmpsrc, src);
10286 
10287     loadConI_x43300000(ix43300000);
10288     loadConI_x41f00000(ix41f00000);
10289     loadConI_x0(ix0);
10290 
10291     regI_regI_to_regD(dx43300000, ix0, ix43300000);
10292     regI_regI_to_regD(dx41f00000, ix0, ix41f00000);
10293 
10294     convI2D_regDHi_regD(tmp1, tmpsrc);
10295     regDHi_regDLo_to_regD(tmp2, dx43300000, tmpsrc);
10296     subD_regD_regD(tmp3, tmp2, dx43300000);
10297     mulD_regD_regD(tmp4, tmp1, dx41f00000);
10298     addD_regD_regD(dst, tmp3, tmp4);
10299   %}
10300 %}
10301 #endif // !AARCH64
10302 
10303 instruct convL2I_reg(iRegI dst, iRegL src) %{
10304   match(Set dst (ConvL2I src));
10305   size(4);
10306 #ifdef AARCH64
10307   format %{ "MOV_w  $dst,$src\t! long->int" %}
10308   ins_encode %{
10309     __ mov_w($dst$$Register, $src$$Register);
10310   %}
10311 #else
10312   format %{ "MOV    $dst,$src.lo\t! long->int" %}
10313   ins_encode %{
10314     __ mov($dst$$Register, $src$$Register);
10315   %}
10316 #endif
10317   ins_pipe(ialu_move_reg_I_to_L);
10318 %}
10319 
10320 #ifndef AARCH64
10321 // Register Shift Right Immediate
10322 instruct shrL_reg_imm6_L2I(iRegI dst, iRegL src, immI_32_63 cnt) %{
10323   match(Set dst (ConvL2I (RShiftL src cnt)));
10324   size(4);
10325   format %{ "ASR    $dst,$src.hi,($cnt - 32)\t! long->int or mov if $cnt==32" %}
10326   ins_encode %{
10327     if ($cnt$$constant == 32) {
10328       __ mov($dst$$Register, $src$$Register->successor());
10329     } else {
10330       __ mov($dst$$Register, AsmOperand($src$$Register->successor(), asr, $cnt$$constant - 32));
10331     }
10332   %}
10333   ins_pipe(ialu_reg_imm);
10334 %}
10335 #endif
10336 
10337 
10338 //----------Control Flow Instructions------------------------------------------
10339 // Compare Instructions
10340 // Compare Integers
10341 instruct compI_iReg(flagsReg icc, iRegI op1, iRegI op2) %{
10342   match(Set icc (CmpI op1 op2));
10343   effect( DEF icc, USE op1, USE op2 );
10344 
10345   size(4);
10346   format %{ "cmp_32 $op1,$op2\t! int" %}
10347   ins_encode %{
10348     __ cmp_32($op1$$Register, $op2$$Register);
10349   %}
10350   ins_pipe(ialu_cconly_reg_reg);
10351 %}
10352 
10353 #ifdef _LP64
10354 // Compare compressed pointers
10355 instruct compN_reg2(flagsRegU icc, iRegN op1, iRegN op2) %{
10356   match(Set icc (CmpN op1 op2));
10357   effect( DEF icc, USE op1, USE op2 );
10358 
10359   size(4);
10360   format %{ "cmp_32 $op1,$op2\t! int" %}
10361   ins_encode %{
10362     __ cmp_32($op1$$Register, $op2$$Register);
10363   %}
10364   ins_pipe(ialu_cconly_reg_reg);
10365 %}
10366 #endif
10367 
10368 instruct compU_iReg(flagsRegU icc, iRegI op1, iRegI op2) %{
10369   match(Set icc (CmpU op1 op2));
10370 
10371   size(4);
10372   format %{ "cmp_32 $op1,$op2\t! unsigned int" %}
10373   ins_encode %{
10374     __ cmp_32($op1$$Register, $op2$$Register);
10375   %}
10376   ins_pipe(ialu_cconly_reg_reg);
10377 %}
10378 
10379 instruct compI_iReg_immneg(flagsReg icc, iRegI op1, aimmIneg op2) %{
10380   match(Set icc (CmpI op1 op2));
10381   effect( DEF icc, USE op1 );
10382 
10383   size(4);
10384   format %{ "cmn_32 $op1,-$op2\t! int" %}
10385   ins_encode %{
10386     __ cmn_32($op1$$Register, -$op2$$constant);
10387   %}
10388   ins_pipe(ialu_cconly_reg_imm);
10389 %}
10390 
10391 instruct compI_iReg_imm(flagsReg icc, iRegI op1, aimmI op2) %{
10392   match(Set icc (CmpI op1 op2));
10393   effect( DEF icc, USE op1 );
10394 
10395   size(4);
10396   format %{ "cmp_32 $op1,$op2\t! int" %}
10397   ins_encode %{
10398     __ cmp_32($op1$$Register, $op2$$constant);
10399   %}
10400   ins_pipe(ialu_cconly_reg_imm);
10401 %}
10402 
10403 instruct testI_reg_reg( flagsReg_EQNELTGE icc, iRegI op1, iRegI op2, immI0 zero ) %{
10404   match(Set icc (CmpI (AndI op1 op2) zero));
10405   size(4);
10406   format %{ "tst_32 $op2,$op1" %}
10407 
10408   ins_encode %{
10409     __ tst_32($op1$$Register, $op2$$Register);
10410   %}
10411   ins_pipe(ialu_cconly_reg_reg_zero);
10412 %}
10413 
10414 #ifndef AARCH64
10415 instruct testshlI_reg_reg_reg( flagsReg_EQNELTGE icc, iRegI op1, iRegI op2, iRegI op3, immI0 zero ) %{
10416   match(Set icc (CmpI (AndI op1 (LShiftI op2 op3)) zero));
10417   size(4);
10418   format %{ "TST   $op2,$op1<<$op3" %}
10419 
10420   ins_encode %{
10421     __ tst($op1$$Register, AsmOperand($op2$$Register, lsl, $op3$$Register));
10422   %}
10423   ins_pipe(ialu_cconly_reg_reg_zero);
10424 %}
10425 #endif
10426 
10427 instruct testshlI_reg_reg_imm( flagsReg_EQNELTGE icc, iRegI op1, iRegI op2, immU5 op3, immI0 zero ) %{
10428   match(Set icc (CmpI (AndI op1 (LShiftI op2 op3)) zero));
10429   size(4);
10430   format %{ "tst_32 $op2,$op1<<$op3" %}
10431 
10432   ins_encode %{
10433     __ tst_32($op1$$Register, AsmOperand($op2$$Register, lsl, $op3$$constant));
10434   %}
10435   ins_pipe(ialu_cconly_reg_reg_zero);
10436 %}
10437 
10438 #ifndef AARCH64
10439 instruct testsarI_reg_reg_reg( flagsReg_EQNELTGE icc, iRegI op1, iRegI op2, iRegI op3, immI0 zero ) %{
10440   match(Set icc (CmpI (AndI op1 (RShiftI op2 op3)) zero));
10441   size(4);
10442   format %{ "TST   $op2,$op1<<$op3" %}
10443 
10444   ins_encode %{
10445     __ tst($op1$$Register, AsmOperand($op2$$Register, asr, $op3$$Register));
10446   %}
10447   ins_pipe(ialu_cconly_reg_reg_zero);
10448 %}
10449 #endif
10450 
10451 instruct testsarI_reg_reg_imm( flagsReg_EQNELTGE icc, iRegI op1, iRegI op2, immU5 op3, immI0 zero ) %{
10452   match(Set icc (CmpI (AndI op1 (RShiftI op2 op3)) zero));
10453   size(4);
10454   format %{ "tst_32 $op2,$op1<<$op3" %}
10455 
10456   ins_encode %{
10457     __ tst_32($op1$$Register, AsmOperand($op2$$Register, asr, $op3$$constant));
10458   %}
10459   ins_pipe(ialu_cconly_reg_reg_zero);
10460 %}
10461 
10462 #ifndef AARCH64
10463 instruct testshrI_reg_reg_reg( flagsReg_EQNELTGE icc, iRegI op1, iRegI op2, iRegI op3, immI0 zero ) %{
10464   match(Set icc (CmpI (AndI op1 (URShiftI op2 op3)) zero));
10465   size(4);
10466   format %{ "TST   $op2,$op1<<$op3" %}
10467 
10468   ins_encode %{
10469     __ tst($op1$$Register, AsmOperand($op2$$Register, lsr, $op3$$Register));
10470   %}
10471   ins_pipe(ialu_cconly_reg_reg_zero);
10472 %}
10473 #endif
10474 
10475 instruct testshrI_reg_reg_imm( flagsReg_EQNELTGE icc, iRegI op1, iRegI op2, immU5 op3, immI0 zero ) %{
10476   match(Set icc (CmpI (AndI op1 (URShiftI op2 op3)) zero));
10477   size(4);
10478   format %{ "tst_32 $op2,$op1<<$op3" %}
10479 
10480   ins_encode %{
10481     __ tst_32($op1$$Register, AsmOperand($op2$$Register, lsr, $op3$$constant));
10482   %}
10483   ins_pipe(ialu_cconly_reg_reg_zero);
10484 %}
10485 
10486 instruct testI_reg_imm( flagsReg_EQNELTGE icc, iRegI op1, limmI op2, immI0 zero ) %{
10487   match(Set icc (CmpI (AndI op1 op2) zero));
10488   size(4);
10489   format %{ "tst_32 $op2,$op1" %}
10490 
10491   ins_encode %{
10492     __ tst_32($op1$$Register, $op2$$constant);
10493   %}
10494   ins_pipe(ialu_cconly_reg_imm_zero);
10495 %}
10496 
10497 #ifdef AARCH64
10498 instruct compL_reg_reg(flagsReg xcc, iRegL op1, iRegL op2)
10499 %{
10500   match(Set xcc (CmpL op1 op2));
10501   effect( DEF xcc, USE op1, USE op2 );
10502 
10503   size(4);
10504   format %{ "CMP     $op1,$op2\t! long" %}
10505   ins_encode %{
10506     __ cmp($op1$$Register, $op2$$Register);
10507   %}
10508   ins_pipe(ialu_cconly_reg_reg);
10509 %}
10510 
10511 instruct compUL_iReg(flagsRegU xcc, iRegL op1, iRegL op2) %{
10512   match(Set xcc (CmpUL op1 op2));
10513 
10514   size(4);
10515   format %{ "CMP     $op1,$op2\t! unsigned long" %}
10516   ins_encode %{
10517     __ cmp($op1$$Register, $op2$$Register);
10518   %}
10519   ins_pipe(ialu_cconly_reg_reg);
10520 %}
10521 #else
10522 instruct compL_reg_reg_LTGE(flagsRegL_LTGE xcc, iRegL op1, iRegL op2, iRegL tmp) %{
10523   match(Set xcc (CmpL op1 op2));
10524   effect( DEF xcc, USE op1, USE op2, TEMP tmp );
10525 
10526   size(8);
10527   format %{ "SUBS    $tmp,$op1.low,$op2.low\t\t! long\n\t"
10528             "SBCS    $tmp,$op1.hi,$op2.hi" %}
10529   ins_encode %{
10530     __ subs($tmp$$Register, $op1$$Register, $op2$$Register);
10531     __ sbcs($tmp$$Register->successor(), $op1$$Register->successor(), $op2$$Register->successor());
10532   %}
10533   ins_pipe(ialu_cconly_reg_reg);
10534 %}
10535 
10536 instruct compUL_reg_reg_LTGE(flagsRegUL_LTGE xcc, iRegL op1, iRegL op2, iRegL tmp) %{
10537   match(Set xcc (CmpUL op1 op2));
10538   effect(DEF xcc, USE op1, USE op2, TEMP tmp);
10539 
10540   size(8);
10541   format %{ "SUBS    $tmp,$op1.low,$op2.low\t\t! unsigned long\n\t"
10542             "SBCS    $tmp,$op1.hi,$op2.hi" %}
10543   ins_encode %{
10544     __ subs($tmp$$Register, $op1$$Register, $op2$$Register);
10545     __ sbcs($tmp$$Register->successor(), $op1$$Register->successor(), $op2$$Register->successor());
10546   %}
10547   ins_pipe(ialu_cconly_reg_reg);
10548 %}
10549 #endif
10550 
10551 #ifdef AARCH64
10552 instruct compL_reg_con(flagsReg xcc, iRegL op1, aimmL con) %{
10553   match(Set xcc (CmpL op1 con));
10554   effect( DEF xcc, USE op1, USE con );
10555 
10556   size(8);
10557   format %{ "CMP     $op1,$con\t\t! long"  %}
10558   ins_encode %{
10559     __ cmp($op1$$Register, $con$$constant);
10560   %}
10561 
10562   ins_pipe(ialu_cconly_reg_imm);
10563 %}
10564 
10565 instruct compUL_reg_con(flagsRegU xcc, iRegL op1, aimmL con) %{
10566   match(Set xcc (CmpUL op1 con));
10567   effect(DEF xcc, USE op1, USE con);
10568 
10569   size(8);
10570   format %{ "CMP     $op1,$con\t\t! unsigned long"  %}
10571   ins_encode %{
10572     __ cmp($op1$$Register, $con$$constant);
10573   %}
10574 
10575   ins_pipe(ialu_cconly_reg_imm);
10576 %}
10577 #else
10578 instruct compL_reg_reg_EQNE(flagsRegL_EQNE xcc, iRegL op1, iRegL op2) %{
10579   match(Set xcc (CmpL op1 op2));
10580   effect( DEF xcc, USE op1, USE op2 );
10581 
10582   size(8);
10583   format %{ "TEQ    $op1.hi,$op2.hi\t\t! long\n\t"
10584             "TEQ.eq $op1.lo,$op2.lo" %}
10585   ins_encode %{
10586     __ teq($op1$$Register->successor(), $op2$$Register->successor());
10587     __ teq($op1$$Register, $op2$$Register, eq);
10588   %}
10589   ins_pipe(ialu_cconly_reg_reg);
10590 %}
10591 
10592 instruct compL_reg_reg_LEGT(flagsRegL_LEGT xcc, iRegL op1, iRegL op2, iRegL tmp) %{
10593   match(Set xcc (CmpL op1 op2));
10594   effect( DEF xcc, USE op1, USE op2, TEMP tmp );
10595 
10596   size(8);
10597   format %{ "SUBS    $tmp,$op2.low,$op1.low\t\t! long\n\t"
10598             "SBCS    $tmp,$op2.hi,$op1.hi" %}
10599   ins_encode %{
10600     __ subs($tmp$$Register, $op2$$Register, $op1$$Register);
10601     __ sbcs($tmp$$Register->successor(), $op2$$Register->successor(), $op1$$Register->successor());
10602   %}
10603   ins_pipe(ialu_cconly_reg_reg);
10604 %}
10605 
10606 // TODO: try immLRot2 instead, (0, $con$$constant) becomes
10607 // (hi($con$$constant), lo($con$$constant)) becomes
10608 instruct compL_reg_con_LTGE(flagsRegL_LTGE xcc, iRegL op1, immLlowRot con, iRegL tmp) %{
10609   match(Set xcc (CmpL op1 con));
10610   effect( DEF xcc, USE op1, USE con, TEMP tmp );
10611 
10612   size(8);
10613   format %{ "SUBS    $tmp,$op1.low,$con\t\t! long\n\t"
10614             "SBCS    $tmp,$op1.hi,0" %}
10615   ins_encode %{
10616     __ subs($tmp$$Register, $op1$$Register, $con$$constant);
10617     __ sbcs($tmp$$Register->successor(), $op1$$Register->successor(), 0);
10618   %}
10619 
10620   ins_pipe(ialu_cconly_reg_reg);
10621 %}
10622 
10623 // TODO: try immLRot2 instead, (0, $con$$constant) becomes
10624 // (hi($con$$constant), lo($con$$constant)) becomes
10625 instruct compL_reg_con_EQNE(flagsRegL_EQNE xcc, iRegL op1, immLlowRot con) %{
10626   match(Set xcc (CmpL op1 con));
10627   effect( DEF xcc, USE op1, USE con );
10628 
10629   size(8);
10630   format %{ "TEQ    $op1.hi,0\t\t! long\n\t"
10631             "TEQ.eq $op1.lo,$con" %}
10632   ins_encode %{
10633     __ teq($op1$$Register->successor(), 0);
10634     __ teq($op1$$Register, $con$$constant, eq);
10635   %}
10636 
10637   ins_pipe(ialu_cconly_reg_reg);
10638 %}
10639 
10640 // TODO: try immLRot2 instead, (0, $con$$constant) becomes
10641 // (hi($con$$constant), lo($con$$constant)) becomes
10642 instruct compL_reg_con_LEGT(flagsRegL_LEGT xcc, iRegL op1, immLlowRot con, iRegL tmp) %{
10643   match(Set xcc (CmpL op1 con));
10644   effect( DEF xcc, USE op1, USE con, TEMP tmp );
10645 
10646   size(8);
10647   format %{ "RSBS    $tmp,$op1.low,$con\t\t! long\n\t"
10648             "RSCS    $tmp,$op1.hi,0" %}
10649   ins_encode %{
10650     __ rsbs($tmp$$Register, $op1$$Register, $con$$constant);
10651     __ rscs($tmp$$Register->successor(), $op1$$Register->successor(), 0);
10652   %}
10653 
10654   ins_pipe(ialu_cconly_reg_reg);
10655 %}
10656 
10657 instruct compUL_reg_reg_EQNE(flagsRegUL_EQNE xcc, iRegL op1, iRegL op2) %{
10658   match(Set xcc (CmpUL op1 op2));
10659   effect(DEF xcc, USE op1, USE op2);
10660 
10661   size(8);
10662   format %{ "TEQ    $op1.hi,$op2.hi\t\t! unsigned long\n\t"
10663             "TEQ.eq $op1.lo,$op2.lo" %}
10664   ins_encode %{
10665     __ teq($op1$$Register->successor(), $op2$$Register->successor());
10666     __ teq($op1$$Register, $op2$$Register, eq);
10667   %}
10668   ins_pipe(ialu_cconly_reg_reg);
10669 %}
10670 
10671 instruct compUL_reg_reg_LEGT(flagsRegUL_LEGT xcc, iRegL op1, iRegL op2, iRegL tmp) %{
10672   match(Set xcc (CmpUL op1 op2));
10673   effect(DEF xcc, USE op1, USE op2, TEMP tmp);
10674 
10675   size(8);
10676   format %{ "SUBS    $tmp,$op2.low,$op1.low\t\t! unsigned long\n\t"
10677             "SBCS    $tmp,$op2.hi,$op1.hi" %}
10678   ins_encode %{
10679     __ subs($tmp$$Register, $op2$$Register, $op1$$Register);
10680     __ sbcs($tmp$$Register->successor(), $op2$$Register->successor(), $op1$$Register->successor());
10681   %}
10682   ins_pipe(ialu_cconly_reg_reg);
10683 %}
10684 
10685 // TODO: try immLRot2 instead, (0, $con$$constant) becomes
10686 // (hi($con$$constant), lo($con$$constant)) becomes
10687 instruct compUL_reg_con_LTGE(flagsRegUL_LTGE xcc, iRegL op1, immLlowRot con, iRegL tmp) %{
10688   match(Set xcc (CmpUL op1 con));
10689   effect(DEF xcc, USE op1, USE con, TEMP tmp);
10690 
10691   size(8);
10692   format %{ "SUBS    $tmp,$op1.low,$con\t\t! unsigned long\n\t"
10693             "SBCS    $tmp,$op1.hi,0" %}
10694   ins_encode %{
10695     __ subs($tmp$$Register, $op1$$Register, $con$$constant);
10696     __ sbcs($tmp$$Register->successor(), $op1$$Register->successor(), 0);
10697   %}
10698 
10699   ins_pipe(ialu_cconly_reg_reg);
10700 %}
10701 
10702 // TODO: try immLRot2 instead, (0, $con$$constant) becomes
10703 // (hi($con$$constant), lo($con$$constant)) becomes
10704 instruct compUL_reg_con_EQNE(flagsRegUL_EQNE xcc, iRegL op1, immLlowRot con) %{
10705   match(Set xcc (CmpUL op1 con));
10706   effect(DEF xcc, USE op1, USE con);
10707 
10708   size(8);
10709   format %{ "TEQ    $op1.hi,0\t\t! unsigned long\n\t"
10710             "TEQ.eq $op1.lo,$con" %}
10711   ins_encode %{
10712     __ teq($op1$$Register->successor(), 0);
10713     __ teq($op1$$Register, $con$$constant, eq);
10714   %}
10715 
10716   ins_pipe(ialu_cconly_reg_reg);
10717 %}
10718 
10719 // TODO: try immLRot2 instead, (0, $con$$constant) becomes
10720 // (hi($con$$constant), lo($con$$constant)) becomes
10721 instruct compUL_reg_con_LEGT(flagsRegUL_LEGT xcc, iRegL op1, immLlowRot con, iRegL tmp) %{
10722   match(Set xcc (CmpUL op1 con));
10723   effect(DEF xcc, USE op1, USE con, TEMP tmp);
10724 
10725   size(8);
10726   format %{ "RSBS    $tmp,$op1.low,$con\t\t! unsigned long\n\t"
10727             "RSCS    $tmp,$op1.hi,0" %}
10728   ins_encode %{
10729     __ rsbs($tmp$$Register, $op1$$Register, $con$$constant);
10730     __ rscs($tmp$$Register->successor(), $op1$$Register->successor(), 0);
10731   %}
10732 
10733   ins_pipe(ialu_cconly_reg_reg);
10734 %}
10735 #endif
10736 
10737 /* instruct testL_reg_reg(flagsRegL xcc, iRegL op1, iRegL op2, immL0 zero) %{ */
10738 /*   match(Set xcc (CmpL (AndL op1 op2) zero)); */
10739 /*   ins_encode %{ */
10740 /*     __ stop("testL_reg_reg unimplemented"); */
10741 /*   %} */
10742 /*   ins_pipe(ialu_cconly_reg_reg); */
10743 /* %} */
10744 
10745 /* // useful for checking the alignment of a pointer: */
10746 /* instruct testL_reg_con(flagsRegL xcc, iRegL op1, immLlowRot con, immL0 zero) %{ */
10747 /*   match(Set xcc (CmpL (AndL op1 con) zero)); */
10748 /*   ins_encode %{ */
10749 /*     __ stop("testL_reg_con unimplemented"); */
10750 /*   %} */
10751 /*   ins_pipe(ialu_cconly_reg_reg); */
10752 /* %} */
10753 
10754 instruct compU_iReg_imm(flagsRegU icc, iRegI op1, aimmU31 op2 ) %{
10755   match(Set icc (CmpU op1 op2));
10756 
10757   size(4);
10758   format %{ "cmp_32 $op1,$op2\t! unsigned" %}
10759   ins_encode %{
10760     __ cmp_32($op1$$Register, $op2$$constant);
10761   %}
10762   ins_pipe(ialu_cconly_reg_imm);
10763 %}
10764 
10765 // Compare Pointers
10766 instruct compP_iRegP(flagsRegP pcc, iRegP op1, iRegP op2 ) %{
10767   match(Set pcc (CmpP op1 op2));
10768 
10769   size(4);
10770   format %{ "CMP    $op1,$op2\t! ptr" %}
10771   ins_encode %{
10772     __ cmp($op1$$Register, $op2$$Register);
10773   %}
10774   ins_pipe(ialu_cconly_reg_reg);
10775 %}
10776 
10777 instruct compP_iRegP_imm(flagsRegP pcc, iRegP op1, aimmP op2 ) %{
10778   match(Set pcc (CmpP op1 op2));
10779 
10780   size(4);
10781   format %{ "CMP    $op1,$op2\t! ptr" %}
10782   ins_encode %{
10783     assert($op2$$constant == 0 || _opnds[2]->constant_reloc() == relocInfo::none, "reloc in cmp?");
10784     __ cmp($op1$$Register, $op2$$constant);
10785   %}
10786   ins_pipe(ialu_cconly_reg_imm);
10787 %}
10788 
10789 //----------Max and Min--------------------------------------------------------
10790 // Min Instructions
10791 // Conditional move for min
10792 instruct cmovI_reg_lt( iRegI op2, iRegI op1, flagsReg icc ) %{
10793   effect( USE_DEF op2, USE op1, USE icc );
10794 
10795   size(4);
10796   format %{ "MOV.lt  $op2,$op1\t! min" %}
10797   ins_encode %{
10798     __ mov($op2$$Register, $op1$$Register, lt);
10799   %}
10800   ins_pipe(ialu_reg_flags);
10801 %}
10802 
10803 // Min Register with Register.
10804 instruct minI_eReg(iRegI op1, iRegI op2) %{
10805   match(Set op2 (MinI op1 op2));
10806   ins_cost(DEFAULT_COST*2);
10807   expand %{
10808     flagsReg icc;
10809     compI_iReg(icc,op1,op2);
10810     cmovI_reg_lt(op2,op1,icc);
10811   %}
10812 %}
10813 
10814 // Max Instructions
10815 // Conditional move for max
10816 instruct cmovI_reg_gt( iRegI op2, iRegI op1, flagsReg icc ) %{
10817   effect( USE_DEF op2, USE op1, USE icc );
10818   format %{ "MOV.gt  $op2,$op1\t! max" %}
10819   ins_encode %{
10820     __ mov($op2$$Register, $op1$$Register, gt);
10821   %}
10822   ins_pipe(ialu_reg_flags);
10823 %}
10824 
10825 // Max Register with Register
10826 instruct maxI_eReg(iRegI op1, iRegI op2) %{
10827   match(Set op2 (MaxI op1 op2));
10828   ins_cost(DEFAULT_COST*2);
10829   expand %{
10830     flagsReg icc;
10831     compI_iReg(icc,op1,op2);
10832     cmovI_reg_gt(op2,op1,icc);
10833   %}
10834 %}
10835 
10836 
10837 //----------Float Compares----------------------------------------------------
10838 // Compare floating, generate condition code
10839 instruct cmpF_cc(flagsRegF fcc, flagsReg icc, regF src1, regF src2) %{
10840   match(Set icc (CmpF src1 src2));
10841   effect(KILL fcc);
10842 
10843 #ifdef AARCH64
10844   size(4);
10845   format %{ "FCMP_s  $src1,$src2" %}
10846   ins_encode %{
10847     __ fcmp_s($src1$$FloatRegister, $src2$$FloatRegister);
10848   %}
10849 #else
10850   size(8);
10851   format %{ "FCMPs  $src1,$src2\n\t"
10852             "FMSTAT" %}
10853   ins_encode %{
10854     __ fcmps($src1$$FloatRegister, $src2$$FloatRegister);
10855     __ fmstat();
10856   %}
10857 #endif
10858   ins_pipe(faddF_fcc_reg_reg_zero);
10859 %}
10860 
10861 instruct cmpF0_cc(flagsRegF fcc, flagsReg icc, regF src1, immF0 src2) %{
10862   match(Set icc (CmpF src1 src2));
10863   effect(KILL fcc);
10864 
10865 #ifdef AARCH64
10866   size(4);
10867   format %{ "FCMP0_s $src1" %}
10868   ins_encode %{
10869     __ fcmp0_s($src1$$FloatRegister);
10870   %}
10871 #else
10872   size(8);
10873   format %{ "FCMPs  $src1,$src2\n\t"
10874             "FMSTAT" %}
10875   ins_encode %{
10876     __ fcmpzs($src1$$FloatRegister);
10877     __ fmstat();
10878   %}
10879 #endif
10880   ins_pipe(faddF_fcc_reg_reg_zero);
10881 %}
10882 
10883 instruct cmpD_cc(flagsRegF fcc, flagsReg icc, regD src1, regD src2) %{
10884   match(Set icc (CmpD src1 src2));
10885   effect(KILL fcc);
10886 
10887 #ifdef AARCH64
10888   size(4);
10889   format %{ "FCMP_d $src1,$src2" %}
10890   ins_encode %{
10891     __ fcmp_d($src1$$FloatRegister, $src2$$FloatRegister);
10892   %}
10893 #else
10894   size(8);
10895   format %{ "FCMPd  $src1,$src2 \n\t"
10896             "FMSTAT" %}
10897   ins_encode %{
10898     __ fcmpd($src1$$FloatRegister, $src2$$FloatRegister);
10899     __ fmstat();
10900   %}
10901 #endif
10902   ins_pipe(faddD_fcc_reg_reg_zero);
10903 %}
10904 
10905 instruct cmpD0_cc(flagsRegF fcc, flagsReg icc, regD src1, immD0 src2) %{
10906   match(Set icc (CmpD src1 src2));
10907   effect(KILL fcc);
10908 
10909 #ifdef AARCH64
10910   size(8);
10911   format %{ "FCMP0_d $src1" %}
10912   ins_encode %{
10913     __ fcmp0_d($src1$$FloatRegister);
10914   %}
10915 #else
10916   size(8);
10917   format %{ "FCMPZd  $src1,$src2 \n\t"
10918             "FMSTAT" %}
10919   ins_encode %{
10920     __ fcmpzd($src1$$FloatRegister);
10921     __ fmstat();
10922   %}
10923 #endif
10924   ins_pipe(faddD_fcc_reg_reg_zero);
10925 %}
10926 
10927 #ifdef AARCH64
10928 // Compare floating, generate -1,0,1
10929 instruct cmpF_reg(iRegI dst, regF src1, regF src2, flagsReg icc) %{
10930   match(Set dst (CmpF3 src1 src2));
10931   // effect(KILL fcc); // nobody cares if flagsRegF is killed
10932   effect(KILL icc);
10933   ins_cost(DEFAULT_COST*3); // FIXME
10934   size(12);
10935   format %{ "FCMP_s $src1,$src2\n\t"
10936             "CSET   $dst, gt\n\t"
10937             "CSINV  $dst, $dst, ZR, ge" %}
10938   ins_encode %{
10939     Register dst = $dst$$Register;
10940     __ fcmp_s($src1$$FloatRegister, $src2$$FloatRegister);
10941     __ cset(dst, gt);            // 1 if '>', else 0
10942     __ csinv(dst, dst, ZR, ge);  // previous value if '>=', else -1
10943   %}
10944   ins_pipe( floating_cmp ); // FIXME
10945 %}
10946 
10947 // Compare floating, generate -1,0,1
10948 instruct cmpD_reg(iRegI dst, regD src1, regD src2, flagsReg icc) %{
10949   match(Set dst (CmpD3 src1 src2));
10950   // effect(KILL fcc); // nobody cares if flagsRegF is killed
10951   effect(KILL icc);
10952   ins_cost(DEFAULT_COST*3); // FIXME
10953   size(12);
10954   format %{ "FCMP_d $src1,$src2\n\t"
10955             "CSET   $dst, gt\n\t"
10956             "CSINV  $dst, $dst, ZR, ge" %}
10957   ins_encode %{
10958     Register dst = $dst$$Register;
10959     __ fcmp_d($src1$$FloatRegister, $src2$$FloatRegister);
10960     __ cset(dst, gt);            // 1 if '>', else 0
10961     __ csinv(dst, dst, ZR, ge);  // previous value if '>=', else -1
10962   %}
10963   ins_pipe( floating_cmp ); // FIXME
10964 %}
10965 
10966 // Compare floating, generate -1,0,1
10967 instruct cmpF0_reg(iRegI dst, regF src1, immF0 src2, flagsReg icc) %{
10968   match(Set dst (CmpF3 src1 src2));
10969   // effect(KILL fcc); // nobody cares if flagsRegF is killed
10970   effect(KILL icc);
10971   ins_cost(DEFAULT_COST*3); // FIXME
10972   size(12);
10973   format %{ "FCMP0_s $src1\n\t"
10974             "CSET   $dst, gt\n\t"
10975             "CSINV  $dst, $dst, ZR, ge" %}
10976   ins_encode %{
10977     Register dst = $dst$$Register;
10978     __ fcmp0_s($src1$$FloatRegister);
10979     __ cset(dst, gt);            // 1 if '>', else 0
10980     __ csinv(dst, dst, ZR, ge);  // previous value if '>=', else -1
10981   %}
10982   ins_pipe( floating_cmp ); // FIXME
10983 %}
10984 
10985 // Compare floating, generate -1,0,1
10986 instruct cmpD0_reg(iRegI dst, regD src1, immD0 src2, flagsReg icc) %{
10987   match(Set dst (CmpD3 src1 src2));
10988   // effect(KILL fcc); // nobody cares if flagsRegF is killed
10989   effect(KILL icc);
10990   ins_cost(DEFAULT_COST*3); // FIXME
10991   size(12);
10992   format %{ "FCMP0_d $src1\n\t"
10993             "CSET   $dst, gt\n\t"
10994             "CSINV  $dst, $dst, ZR, ge" %}
10995   ins_encode %{
10996     Register dst = $dst$$Register;
10997     __ fcmp0_d($src1$$FloatRegister);
10998     __ cset(dst, gt);            // 1 if '>', else 0
10999     __ csinv(dst, dst, ZR, ge);  // previous value if '>=', else -1
11000   %}
11001   ins_pipe( floating_cmp ); // FIXME
11002 %}
11003 #else
11004 // Compare floating, generate -1,0,1
11005 instruct cmpF_reg(iRegI dst, regF src1, regF src2, flagsRegF fcc) %{
11006   match(Set dst (CmpF3 src1 src2));
11007   effect(KILL fcc);
11008   ins_cost(DEFAULT_COST*3+BRANCH_COST*3); // FIXME
11009   size(20);
11010   // same number of instructions as code using conditional moves but
11011   // doesn't kill integer condition register
11012   format %{ "FCMPs  $dst,$src1,$src2 \n\t"
11013             "VMRS   $dst, FPSCR \n\t"
11014             "OR     $dst, $dst, 0x08000000 \n\t"
11015             "EOR    $dst, $dst, $dst << 3 \n\t"
11016             "MOV    $dst, $dst >> 30" %}
11017   ins_encode %{
11018     __ fcmps($src1$$FloatRegister, $src2$$FloatRegister);
11019     __ floating_cmp($dst$$Register);
11020   %}
11021   ins_pipe( floating_cmp );
11022 %}
11023 
11024 instruct cmpF0_reg(iRegI dst, regF src1, immF0 src2, flagsRegF fcc) %{
11025   match(Set dst (CmpF3 src1 src2));
11026   effect(KILL fcc);
11027   ins_cost(DEFAULT_COST*3+BRANCH_COST*3); // FIXME
11028   size(20);
11029   // same number of instructions as code using conditional moves but
11030   // doesn't kill integer condition register
11031   format %{ "FCMPZs $dst,$src1,$src2 \n\t"
11032             "VMRS   $dst, FPSCR \n\t"
11033             "OR     $dst, $dst, 0x08000000 \n\t"
11034             "EOR    $dst, $dst, $dst << 3 \n\t"
11035             "MOV    $dst, $dst >> 30" %}
11036   ins_encode %{
11037     __ fcmpzs($src1$$FloatRegister);
11038     __ floating_cmp($dst$$Register);
11039   %}
11040   ins_pipe( floating_cmp );
11041 %}
11042 
11043 instruct cmpD_reg(iRegI dst, regD src1, regD src2, flagsRegF fcc) %{
11044   match(Set dst (CmpD3 src1 src2));
11045   effect(KILL fcc);
11046   ins_cost(DEFAULT_COST*3+BRANCH_COST*3); // FIXME
11047   size(20);
11048   // same number of instructions as code using conditional moves but
11049   // doesn't kill integer condition register
11050   format %{ "FCMPd  $dst,$src1,$src2 \n\t"
11051             "VMRS   $dst, FPSCR \n\t"
11052             "OR     $dst, $dst, 0x08000000 \n\t"
11053             "EOR    $dst, $dst, $dst << 3 \n\t"
11054             "MOV    $dst, $dst >> 30" %}
11055   ins_encode %{
11056     __ fcmpd($src1$$FloatRegister, $src2$$FloatRegister);
11057     __ floating_cmp($dst$$Register);
11058   %}
11059   ins_pipe( floating_cmp );
11060 %}
11061 
11062 instruct cmpD0_reg(iRegI dst, regD src1, immD0 src2, flagsRegF fcc) %{
11063   match(Set dst (CmpD3 src1 src2));
11064   effect(KILL fcc);
11065   ins_cost(DEFAULT_COST*3+BRANCH_COST*3); // FIXME
11066   size(20);
11067   // same number of instructions as code using conditional moves but
11068   // doesn't kill integer condition register
11069   format %{ "FCMPZd $dst,$src1,$src2 \n\t"
11070             "VMRS   $dst, FPSCR \n\t"
11071             "OR     $dst, $dst, 0x08000000 \n\t"
11072             "EOR    $dst, $dst, $dst << 3 \n\t"
11073             "MOV    $dst, $dst >> 30" %}
11074   ins_encode %{
11075     __ fcmpzd($src1$$FloatRegister);
11076     __ floating_cmp($dst$$Register);
11077   %}
11078   ins_pipe( floating_cmp );
11079 %}
11080 #endif // !AARCH64
11081 
11082 //----------Branches---------------------------------------------------------
11083 // Jump
11084 // (compare 'operand indIndex' and 'instruct addP_reg_reg' above)
11085 // FIXME
11086 instruct jumpXtnd(iRegX switch_val, iRegP tmp) %{
11087   match(Jump switch_val);
11088   effect(TEMP tmp);
11089   ins_cost(350);
11090   format %{  "ADD    $tmp, $constanttablebase, $switch_val\n\t"
11091              "LDR    $tmp,[$tmp + $constantoffset]\n\t"
11092              "BX     $tmp" %}
11093   size(20);
11094   ins_encode %{
11095     Register table_reg;
11096     Register label_reg = $tmp$$Register;
11097     if (constant_offset() == 0) {
11098       table_reg = $constanttablebase;
11099       __ ldr(label_reg, Address(table_reg, $switch_val$$Register));
11100     } else {
11101       table_reg = $tmp$$Register;
11102       int offset = $constantoffset;
11103       if (is_memoryP(offset)) {
11104         __ add(table_reg, $constanttablebase, $switch_val$$Register);
11105         __ ldr(label_reg, Address(table_reg, offset));
11106       } else {
11107         __ mov_slow(table_reg, $constantoffset);
11108         __ add(table_reg, $constanttablebase, table_reg);
11109         __ ldr(label_reg, Address(table_reg, $switch_val$$Register));
11110       }
11111     }
11112     __ jump(label_reg); // ldr + b better than ldr to PC for branch predictor?
11113     //    __ ldr(PC, Address($table$$Register, $switch_val$$Register));
11114   %}
11115   ins_pipe(ialu_reg_reg);
11116 %}
11117 
11118 // // Direct Branch.
11119 instruct branch(label labl) %{
11120   match(Goto);
11121   effect(USE labl);
11122 
11123   size(4);
11124   ins_cost(BRANCH_COST);
11125   format %{ "B     $labl" %}
11126   ins_encode %{
11127     __ b(*($labl$$label));
11128   %}
11129   ins_pipe(br);
11130 %}
11131 
11132 // Conditional Direct Branch
11133 instruct branchCon(cmpOp cmp, flagsReg icc, label labl) %{
11134   match(If cmp icc);
11135   effect(USE labl);
11136 
11137   size(4);
11138   ins_cost(BRANCH_COST);
11139   format %{ "B$cmp   $icc,$labl" %}
11140   ins_encode %{
11141     __ b(*($labl$$label), (AsmCondition)($cmp$$cmpcode));
11142   %}
11143   ins_pipe(br_cc);
11144 %}
11145 
11146 #ifdef ARM
11147 instruct branchCon_EQNELTGE(cmpOp0 cmp, flagsReg_EQNELTGE icc, label labl) %{
11148   match(If cmp icc);
11149   effect(USE labl);
11150   predicate( _kids[0]->_leaf->as_Bool()->_test._test == BoolTest::eq || _kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ne || _kids[0]->_leaf->as_Bool()->_test._test == BoolTest::lt || _kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ge);
11151 
11152   size(4);
11153   ins_cost(BRANCH_COST);
11154   format %{ "B$cmp   $icc,$labl" %}
11155   ins_encode %{
11156     __ b(*($labl$$label), (AsmCondition)($cmp$$cmpcode));
11157   %}
11158   ins_pipe(br_cc);
11159 %}
11160 #endif
11161 
11162 #ifdef AARCH64
11163 instruct cbzI(cmpOp cmp, iRegI op1, immI0 op2, label labl) %{
11164   match(If cmp (CmpI op1 op2));
11165   effect(USE labl);
11166   predicate(_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::eq ||
11167             _kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ne);
11168   size(4);
11169   ins_cost(BRANCH_COST);
11170   format %{ "CB{N}Z $op1, $labl\t! int $cmp" %}
11171   ins_encode %{
11172     if ($cmp$$cmpcode == eq) {
11173       __ cbz_w($op1$$Register, *($labl$$label));
11174     } else {
11175       __ cbnz_w($op1$$Register, *($labl$$label));
11176     }
11177   %}
11178   ins_pipe(br_cc); // FIXME
11179 %}
11180 
11181 instruct cbzP(cmpOpP cmp, iRegP op1, immP0 op2, label labl) %{
11182   match(If cmp (CmpP op1 op2));
11183   effect(USE labl);
11184   predicate(_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::eq ||
11185             _kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ne);
11186   size(4);
11187   ins_cost(BRANCH_COST);
11188   format %{ "CB{N}Z $op1, $labl\t! ptr $cmp" %}
11189   ins_encode %{
11190     if ($cmp$$cmpcode == eq) {
11191       __ cbz($op1$$Register, *($labl$$label));
11192     } else {
11193       __ cbnz($op1$$Register, *($labl$$label));
11194     }
11195   %}
11196   ins_pipe(br_cc); // FIXME
11197 %}
11198 
11199 instruct cbzL(cmpOpL cmp, iRegL op1, immL0 op2, label labl) %{
11200   match(If cmp (CmpL op1 op2));
11201   effect(USE labl);
11202   predicate(_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::eq ||
11203             _kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ne);
11204   size(4);
11205   ins_cost(BRANCH_COST);
11206   format %{ "CB{N}Z $op1, $labl\t! long $cmp" %}
11207   ins_encode %{
11208     if ($cmp$$cmpcode == eq) {
11209       __ cbz($op1$$Register, *($labl$$label));
11210     } else {
11211       __ cbnz($op1$$Register, *($labl$$label));
11212     }
11213   %}
11214   ins_pipe(br_cc); // FIXME
11215 %}
11216 #endif
11217 
11218 instruct branchConU(cmpOpU cmp, flagsRegU icc, label labl) %{
11219   match(If cmp icc);
11220   effect(USE labl);
11221 
11222   size(4);
11223   ins_cost(BRANCH_COST);
11224   format %{ "B$cmp  $icc,$labl" %}
11225   ins_encode %{
11226     __ b(*($labl$$label), (AsmCondition)($cmp$$cmpcode));
11227   %}
11228   ins_pipe(br_cc);
11229 %}
11230 
11231 instruct branchConP(cmpOpP cmp, flagsRegP pcc, label labl) %{
11232   match(If cmp pcc);
11233   effect(USE labl);
11234 
11235   size(4);
11236   ins_cost(BRANCH_COST);
11237   format %{ "B$cmp  $pcc,$labl" %}
11238   ins_encode %{
11239     __ b(*($labl$$label), (AsmCondition)($cmp$$cmpcode));
11240   %}
11241   ins_pipe(br_cc);
11242 %}
11243 
11244 #ifndef AARCH64
11245 instruct branchConL_LTGE(cmpOpL cmp, flagsRegL_LTGE xcc, label labl) %{
11246   match(If cmp xcc);
11247   effect(USE labl);
11248   predicate( _kids[0]->_leaf->as_Bool()->_test._test == BoolTest::lt || _kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ge );
11249 
11250   size(4);
11251   ins_cost(BRANCH_COST);
11252   format %{ "B$cmp  $xcc,$labl" %}
11253   ins_encode %{
11254     __ b(*($labl$$label), (AsmCondition)($cmp$$cmpcode));
11255   %}
11256   ins_pipe(br_cc);
11257 %}
11258 
11259 instruct branchConL_EQNE(cmpOpL cmp, flagsRegL_EQNE xcc, label labl) %{
11260   match(If cmp xcc);
11261   effect(USE labl);
11262   predicate( _kids[0]->_leaf->as_Bool()->_test._test == BoolTest::eq || _kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ne );
11263 
11264   size(4);
11265   ins_cost(BRANCH_COST);
11266   format %{ "B$cmp  $xcc,$labl" %}
11267   ins_encode %{
11268     __ b(*($labl$$label), (AsmCondition)($cmp$$cmpcode));
11269   %}
11270   ins_pipe(br_cc);
11271 %}
11272 
11273 instruct branchConL_LEGT(cmpOpL_commute cmp, flagsRegL_LEGT xcc, label labl) %{
11274   match(If cmp xcc);
11275   effect(USE labl);
11276   predicate( _kids[0]->_leaf->as_Bool()->_test._test == BoolTest::gt || _kids[0]->_leaf->as_Bool()->_test._test == BoolTest::le );
11277 
11278   size(4);
11279   ins_cost(BRANCH_COST);
11280   format %{ "B$cmp  $xcc,$labl" %}
11281   ins_encode %{
11282     __ b(*($labl$$label), (AsmCondition)($cmp$$cmpcode));
11283   %}
11284   ins_pipe(br_cc);
11285 %}
11286 
11287 instruct branchConUL_LTGE(cmpOpUL cmp, flagsRegUL_LTGE xcc, label labl) %{
11288   match(If cmp xcc);
11289   effect(USE labl);
11290   predicate(_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::lt || _kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ge);
11291 
11292   size(4);
11293   ins_cost(BRANCH_COST);
11294   format %{ "B$cmp  $xcc,$labl" %}
11295   ins_encode %{
11296     __ b(*($labl$$label), (AsmCondition)($cmp$$cmpcode));
11297   %}
11298   ins_pipe(br_cc);
11299 %}
11300 
11301 instruct branchConUL_EQNE(cmpOpUL cmp, flagsRegUL_EQNE xcc, label labl) %{
11302   match(If cmp xcc);
11303   effect(USE labl);
11304   predicate(_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::eq || _kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ne);
11305 
11306   size(4);
11307   ins_cost(BRANCH_COST);
11308   format %{ "B$cmp  $xcc,$labl" %}
11309   ins_encode %{
11310     __ b(*($labl$$label), (AsmCondition)($cmp$$cmpcode));
11311   %}
11312   ins_pipe(br_cc);
11313 %}
11314 
11315 instruct branchConUL_LEGT(cmpOpUL_commute cmp, flagsRegUL_LEGT xcc, label labl) %{
11316   match(If cmp xcc);
11317   effect(USE labl);
11318   predicate(_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::gt || _kids[0]->_leaf->as_Bool()->_test._test == BoolTest::le);
11319 
11320   size(4);
11321   ins_cost(BRANCH_COST);
11322   format %{ "B$cmp  $xcc,$labl" %}
11323   ins_encode %{
11324     __ b(*($labl$$label), (AsmCondition)($cmp$$cmpcode));
11325   %}
11326   ins_pipe(br_cc);
11327 %}
11328 #endif
11329 
11330 instruct branchLoopEnd(cmpOp cmp, flagsReg icc, label labl) %{
11331   match(CountedLoopEnd cmp icc);
11332   effect(USE labl);
11333 
11334   size(4);
11335   ins_cost(BRANCH_COST);
11336   format %{ "B$cmp   $icc,$labl\t! Loop end" %}
11337   ins_encode %{
11338     __ b(*($labl$$label), (AsmCondition)($cmp$$cmpcode));
11339   %}
11340   ins_pipe(br_cc);
11341 %}
11342 
11343 // instruct branchLoopEndU(cmpOpU cmp, flagsRegU icc, label labl) %{
11344 //   match(CountedLoopEnd cmp icc);
11345 //   ins_pipe(br_cc);
11346 // %}
11347 
11348 // ============================================================================
11349 // Long Compare
11350 //
11351 // Currently we hold longs in 2 registers.  Comparing such values efficiently
11352 // is tricky.  The flavor of compare used depends on whether we are testing
11353 // for LT, LE, or EQ.  For a simple LT test we can check just the sign bit.
11354 // The GE test is the negated LT test.  The LE test can be had by commuting
11355 // the operands (yielding a GE test) and then negating; negate again for the
11356 // GT test.  The EQ test is done by ORcc'ing the high and low halves, and the
11357 // NE test is negated from that.
11358 
11359 // Due to a shortcoming in the ADLC, it mixes up expressions like:
11360 // (foo (CmpI (CmpL X Y) 0)) and (bar (CmpI (CmpL X 0L) 0)).  Note the
11361 // difference between 'Y' and '0L'.  The tree-matches for the CmpI sections
11362 // are collapsed internally in the ADLC's dfa-gen code.  The match for
11363 // (CmpI (CmpL X Y) 0) is silently replaced with (CmpI (CmpL X 0L) 0) and the
11364 // foo match ends up with the wrong leaf.  One fix is to not match both
11365 // reg-reg and reg-zero forms of long-compare.  This is unfortunate because
11366 // both forms beat the trinary form of long-compare and both are very useful
11367 // on Intel which has so few registers.
11368 
11369 // instruct branchCon_long(cmpOp cmp, flagsRegL xcc, label labl) %{
11370 //   match(If cmp xcc);
11371 //   ins_pipe(br_cc);
11372 // %}
11373 
11374 // Manifest a CmpL3 result in an integer register.  Very painful.
11375 // This is the test to avoid.
11376 #ifdef AARCH64
11377 instruct cmpL3_reg_reg(iRegI dst, iRegL src1, iRegL src2, flagsReg ccr) %{
11378   match(Set dst (CmpL3 src1 src2));
11379   // effect(KILL fcc); // nobody cares if flagsRegF is killed
11380   effect(KILL ccr);
11381   ins_cost(DEFAULT_COST*3); // FIXME
11382   size(12);
11383   format %{ "CMP    $src1,$src2\n\t"
11384             "CSET   $dst, gt\n\t"
11385             "CSINV  $dst, $dst, ZR, ge" %}
11386   ins_encode %{
11387     Register dst = $dst$$Register;
11388     __ cmp($src1$$Register, $src2$$Register);
11389     __ cset(dst, gt);            // 1 if '>', else 0
11390     __ csinv(dst, dst, ZR, ge);  // previous value if '>=', else -1
11391   %}
11392   ins_pipe( ialu_cconly_reg_reg ); // FIXME
11393 %}
11394 // TODO cmpL3_reg_imm
11395 #else
11396 instruct cmpL3_reg_reg(iRegI dst, iRegL src1, iRegL src2, flagsReg ccr ) %{
11397   match(Set dst (CmpL3 src1 src2) );
11398   effect( KILL ccr );
11399   ins_cost(6*DEFAULT_COST); // FIXME
11400   size(32);
11401   format %{
11402       "CMP    $src1.hi, $src2.hi\t\t! long\n"
11403     "\tMOV.gt $dst, 1\n"
11404     "\tmvn.lt $dst, 0\n"
11405     "\tB.ne   done\n"
11406     "\tSUBS   $dst, $src1.lo, $src2.lo\n"
11407     "\tMOV.hi $dst, 1\n"
11408     "\tmvn.lo $dst, 0\n"
11409     "done:"     %}
11410   ins_encode %{
11411     Label done;
11412     __ cmp($src1$$Register->successor(), $src2$$Register->successor());
11413     __ mov($dst$$Register, 1, gt);
11414     __ mvn($dst$$Register, 0, lt);
11415     __ b(done, ne);
11416     __ subs($dst$$Register, $src1$$Register, $src2$$Register);
11417     __ mov($dst$$Register, 1, hi);
11418     __ mvn($dst$$Register, 0, lo);
11419     __ bind(done);
11420   %}
11421   ins_pipe(cmpL_reg);
11422 %}
11423 #endif
11424 
11425 #ifndef AARCH64
11426 // Conditional move
11427 instruct cmovLL_reg_LTGE(cmpOpL cmp, flagsRegL_LTGE xcc, iRegL dst, iRegL src) %{
11428   match(Set dst (CMoveL (Binary cmp xcc) (Binary dst src)));
11429   predicate(_kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::lt || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ge );
11430 
11431   ins_cost(150);
11432   size(8);
11433   format %{ "MOV$cmp  $dst.lo,$src.lo\t! long\n\t"
11434             "MOV$cmp  $dst,$src.hi" %}
11435   ins_encode %{
11436     __ mov($dst$$Register, $src$$Register, (AsmCondition)($cmp$$cmpcode));
11437     __ mov($dst$$Register->successor(), $src$$Register->successor(), (AsmCondition)($cmp$$cmpcode));
11438   %}
11439   ins_pipe(ialu_reg);
11440 %}
11441 
11442 instruct cmovLL_reg_EQNE(cmpOpL cmp, flagsRegL_EQNE xcc, iRegL dst, iRegL src) %{
11443   match(Set dst (CMoveL (Binary cmp xcc) (Binary dst src)));
11444   predicate(_kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::eq || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ne );
11445 
11446   ins_cost(150);
11447   size(8);
11448   format %{ "MOV$cmp  $dst.lo,$src.lo\t! long\n\t"
11449             "MOV$cmp  $dst,$src.hi" %}
11450   ins_encode %{
11451     __ mov($dst$$Register, $src$$Register, (AsmCondition)($cmp$$cmpcode));
11452     __ mov($dst$$Register->successor(), $src$$Register->successor(), (AsmCondition)($cmp$$cmpcode));
11453   %}
11454   ins_pipe(ialu_reg);
11455 %}
11456 
11457 instruct cmovLL_reg_LEGT(cmpOpL_commute cmp, flagsRegL_LEGT xcc, iRegL dst, iRegL src) %{
11458   match(Set dst (CMoveL (Binary cmp xcc) (Binary dst src)));
11459   predicate(_kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::le || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::gt );
11460 
11461   ins_cost(150);
11462   size(8);
11463   format %{ "MOV$cmp  $dst.lo,$src.lo\t! long\n\t"
11464             "MOV$cmp  $dst,$src.hi" %}
11465   ins_encode %{
11466     __ mov($dst$$Register, $src$$Register, (AsmCondition)($cmp$$cmpcode));
11467     __ mov($dst$$Register->successor(), $src$$Register->successor(), (AsmCondition)($cmp$$cmpcode));
11468   %}
11469   ins_pipe(ialu_reg);
11470 %}
11471 
11472 instruct cmovLL_imm_LTGE(cmpOpL cmp, flagsRegL_LTGE xcc, iRegL dst, immL0 src) %{
11473   match(Set dst (CMoveL (Binary cmp xcc) (Binary dst src)));
11474   predicate(_kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::lt || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ge );
11475   ins_cost(140);
11476   size(8);
11477   format %{ "MOV$cmp  $dst.lo,0\t! long\n\t"
11478             "MOV$cmp  $dst,0" %}
11479   ins_encode %{
11480     __ mov($dst$$Register, 0, (AsmCondition)($cmp$$cmpcode));
11481     __ mov($dst$$Register->successor(), 0, (AsmCondition)($cmp$$cmpcode));
11482   %}
11483   ins_pipe(ialu_imm);
11484 %}
11485 
11486 instruct cmovLL_imm_EQNE(cmpOpL cmp, flagsRegL_EQNE xcc, iRegL dst, immL0 src) %{
11487   match(Set dst (CMoveL (Binary cmp xcc) (Binary dst src)));
11488   predicate(_kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::eq || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ne );
11489   ins_cost(140);
11490   size(8);
11491   format %{ "MOV$cmp  $dst.lo,0\t! long\n\t"
11492             "MOV$cmp  $dst,0" %}
11493   ins_encode %{
11494     __ mov($dst$$Register, 0, (AsmCondition)($cmp$$cmpcode));
11495     __ mov($dst$$Register->successor(), 0, (AsmCondition)($cmp$$cmpcode));
11496   %}
11497   ins_pipe(ialu_imm);
11498 %}
11499 
11500 instruct cmovLL_imm_LEGT(cmpOpL_commute cmp, flagsRegL_LEGT xcc, iRegL dst, immL0 src) %{
11501   match(Set dst (CMoveL (Binary cmp xcc) (Binary dst src)));
11502   predicate(_kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::le || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::gt );
11503   ins_cost(140);
11504   size(8);
11505   format %{ "MOV$cmp  $dst.lo,0\t! long\n\t"
11506             "MOV$cmp  $dst,0" %}
11507   ins_encode %{
11508     __ mov($dst$$Register, 0, (AsmCondition)($cmp$$cmpcode));
11509     __ mov($dst$$Register->successor(), 0, (AsmCondition)($cmp$$cmpcode));
11510   %}
11511   ins_pipe(ialu_imm);
11512 %}
11513 #endif // !AARCH64
11514 
11515 #ifndef AARCH64
11516 instruct cmovIL_reg_LTGE(cmpOpL cmp, flagsRegL_LTGE xcc, iRegI dst, iRegI src) %{
11517   match(Set dst (CMoveI (Binary cmp xcc) (Binary dst src)));
11518   predicate(_kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::lt || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ge );
11519 
11520   ins_cost(150);
11521   size(4);
11522   format %{ "MOV$cmp  $dst,$src" %}
11523   ins_encode %{
11524     __ mov($dst$$Register, $src$$Register, (AsmCondition)($cmp$$cmpcode));
11525   %}
11526   ins_pipe(ialu_reg);
11527 %}
11528 
11529 instruct cmovIL_reg_EQNE(cmpOpL cmp, flagsRegL_EQNE xcc, iRegI dst, iRegI src) %{
11530   match(Set dst (CMoveI (Binary cmp xcc) (Binary dst src)));
11531   predicate(_kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::eq || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ne );
11532 
11533   ins_cost(150);
11534   size(4);
11535   format %{ "MOV$cmp  $dst,$src" %}
11536   ins_encode %{
11537     __ mov($dst$$Register, $src$$Register, (AsmCondition)($cmp$$cmpcode));
11538   %}
11539   ins_pipe(ialu_reg);
11540 %}
11541 
11542 instruct cmovIL_reg_LEGT(cmpOpL_commute cmp, flagsRegL_LEGT xcc, iRegI dst, iRegI src) %{
11543   match(Set dst (CMoveI (Binary cmp xcc) (Binary dst src)));
11544   predicate(_kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::le || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::gt );
11545 
11546   ins_cost(150);
11547   size(4);
11548   format %{ "MOV$cmp  $dst,$src" %}
11549   ins_encode %{
11550     __ mov($dst$$Register, $src$$Register, (AsmCondition)($cmp$$cmpcode));
11551   %}
11552   ins_pipe(ialu_reg);
11553 %}
11554 #endif // !AARCH64
11555 
11556 #ifndef AARCH64
11557 instruct cmovIL_imm_LTGE(cmpOpL cmp, flagsRegL_LTGE xcc, iRegI dst, immI16 src) %{
11558   match(Set dst (CMoveI (Binary cmp xcc) (Binary dst src)));
11559   predicate(_kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::lt || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ge );
11560 
11561   ins_cost(140);
11562   format %{ "MOVW$cmp  $dst,$src" %}
11563   ins_encode %{
11564     __ movw($dst$$Register, $src$$constant, (AsmCondition)($cmp$$cmpcode));
11565   %}
11566   ins_pipe(ialu_imm);
11567 %}
11568 
11569 instruct cmovIL_imm_EQNE(cmpOpL cmp, flagsRegL_EQNE xcc, iRegI dst, immI16 src) %{
11570   match(Set dst (CMoveI (Binary cmp xcc) (Binary dst src)));
11571   predicate(_kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::eq || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ne );
11572 
11573   ins_cost(140);
11574   format %{ "MOVW$cmp  $dst,$src" %}
11575   ins_encode %{
11576     __ movw($dst$$Register, $src$$constant, (AsmCondition)($cmp$$cmpcode));
11577   %}
11578   ins_pipe(ialu_imm);
11579 %}
11580 
11581 instruct cmovIL_imm_LEGT(cmpOpL_commute cmp, flagsRegL_LEGT xcc, iRegI dst, immI16 src) %{
11582   match(Set dst (CMoveI (Binary cmp xcc) (Binary dst src)));
11583   predicate(_kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::le || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::gt );
11584 
11585   ins_cost(140);
11586   format %{ "MOVW$cmp  $dst,$src" %}
11587   ins_encode %{
11588     __ movw($dst$$Register, $src$$constant, (AsmCondition)($cmp$$cmpcode));
11589   %}
11590   ins_pipe(ialu_imm);
11591 %}
11592 
11593 instruct cmovPL_reg_LTGE(cmpOpL cmp, flagsRegL_LTGE xcc, iRegP dst, iRegP src) %{
11594   match(Set dst (CMoveP (Binary cmp xcc) (Binary dst src)));
11595   predicate(_kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::lt || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ge );
11596 
11597   ins_cost(150);
11598   size(4);
11599   format %{ "MOV$cmp  $dst,$src" %}
11600   ins_encode %{
11601     __ mov($dst$$Register, $src$$Register, (AsmCondition)($cmp$$cmpcode));
11602   %}
11603   ins_pipe(ialu_reg);
11604 %}
11605 
11606 instruct cmovPL_reg_EQNE(cmpOpL cmp, flagsRegL_EQNE xcc, iRegP dst, iRegP src) %{
11607   match(Set dst (CMoveP (Binary cmp xcc) (Binary dst src)));
11608   predicate(_kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::eq || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ne );
11609 
11610   ins_cost(150);
11611   size(4);
11612   format %{ "MOV$cmp  $dst,$src" %}
11613   ins_encode %{
11614     __ mov($dst$$Register, $src$$Register, (AsmCondition)($cmp$$cmpcode));
11615   %}
11616   ins_pipe(ialu_reg);
11617 %}
11618 
11619 instruct cmovPL_reg_LEGT(cmpOpL_commute cmp, flagsRegL_LEGT xcc, iRegP dst, iRegP src) %{
11620   match(Set dst (CMoveP (Binary cmp xcc) (Binary dst src)));
11621   predicate(_kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::le || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::gt );
11622 
11623   ins_cost(150);
11624   size(4);
11625   format %{ "MOV$cmp  $dst,$src" %}
11626   ins_encode %{
11627     __ mov($dst$$Register, $src$$Register, (AsmCondition)($cmp$$cmpcode));
11628   %}
11629   ins_pipe(ialu_reg);
11630 %}
11631 
11632 instruct cmovPL_imm_LTGE(cmpOpL cmp, flagsRegL_LTGE xcc, iRegP dst, immP0 src) %{
11633   match(Set dst (CMoveP (Binary cmp xcc) (Binary dst src)));
11634   predicate(_kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::lt || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ge );
11635 
11636   ins_cost(140);
11637   format %{ "MOVW$cmp  $dst,$src" %}
11638   ins_encode %{
11639     __ movw($dst$$Register, $src$$constant, (AsmCondition)($cmp$$cmpcode));
11640   %}
11641   ins_pipe(ialu_imm);
11642 %}
11643 
11644 instruct cmovPL_imm_EQNE(cmpOpL cmp, flagsRegL_EQNE xcc, iRegP dst, immP0 src) %{
11645   match(Set dst (CMoveP (Binary cmp xcc) (Binary dst src)));
11646   predicate(_kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::eq || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ne );
11647 
11648   ins_cost(140);
11649   format %{ "MOVW$cmp  $dst,$src" %}
11650   ins_encode %{
11651     __ movw($dst$$Register, $src$$constant, (AsmCondition)($cmp$$cmpcode));
11652   %}
11653   ins_pipe(ialu_imm);
11654 %}
11655 
11656 instruct cmovPL_imm_LEGT(cmpOpL_commute cmp, flagsRegL_LEGT xcc, iRegP dst, immP0 src) %{
11657   match(Set dst (CMoveP (Binary cmp xcc) (Binary dst src)));
11658   predicate(_kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::le || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::gt );
11659 
11660   ins_cost(140);
11661   format %{ "MOVW$cmp  $dst,$src" %}
11662   ins_encode %{
11663     __ movw($dst$$Register, $src$$constant, (AsmCondition)($cmp$$cmpcode));
11664   %}
11665   ins_pipe(ialu_imm);
11666 %}
11667 
11668 instruct cmovFL_reg_LTGE(cmpOpL cmp, flagsRegL_LTGE xcc, regF dst, regF src) %{
11669   match(Set dst (CMoveF (Binary cmp xcc) (Binary dst src)));
11670   predicate(_kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::lt || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ge );
11671   ins_cost(150);
11672   size(4);
11673   format %{ "FCPYS$cmp $dst,$src" %}
11674   ins_encode %{
11675     __ fcpys($dst$$FloatRegister, $src$$FloatRegister, (AsmCondition)($cmp$$cmpcode));
11676   %}
11677   ins_pipe(int_conditional_float_move);
11678 %}
11679 
11680 instruct cmovFL_reg_EQNE(cmpOpL cmp, flagsRegL_EQNE xcc, regF dst, regF src) %{
11681   match(Set dst (CMoveF (Binary cmp xcc) (Binary dst src)));
11682   predicate(_kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::eq || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ne );
11683   ins_cost(150);
11684   size(4);
11685   format %{ "FCPYS$cmp $dst,$src" %}
11686   ins_encode %{
11687     __ fcpys($dst$$FloatRegister, $src$$FloatRegister, (AsmCondition)($cmp$$cmpcode));
11688   %}
11689   ins_pipe(int_conditional_float_move);
11690 %}
11691 
11692 instruct cmovFL_reg_LEGT(cmpOpL_commute cmp, flagsRegL_LEGT xcc, regF dst, regF src) %{
11693   match(Set dst (CMoveF (Binary cmp xcc) (Binary dst src)));
11694   predicate(_kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::le || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::gt );
11695   ins_cost(150);
11696   size(4);
11697   format %{ "FCPYS$cmp $dst,$src" %}
11698   ins_encode %{
11699     __ fcpys($dst$$FloatRegister, $src$$FloatRegister, (AsmCondition)($cmp$$cmpcode));
11700   %}
11701   ins_pipe(int_conditional_float_move);
11702 %}
11703 
11704 instruct cmovDL_reg_LTGE(cmpOpL cmp, flagsRegL_LTGE xcc, regD dst, regD src) %{
11705   match(Set dst (CMoveD (Binary cmp xcc) (Binary dst src)));
11706   predicate(_kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::lt || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ge );
11707 
11708   ins_cost(150);
11709   size(4);
11710   format %{ "FCPYD$cmp $dst,$src" %}
11711   ins_encode %{
11712     __ fcpyd($dst$$FloatRegister, $src$$FloatRegister, (AsmCondition)($cmp$$cmpcode));
11713   %}
11714   ins_pipe(int_conditional_float_move);
11715 %}
11716 
11717 instruct cmovDL_reg_EQNE(cmpOpL cmp, flagsRegL_EQNE xcc, regD dst, regD src) %{
11718   match(Set dst (CMoveD (Binary cmp xcc) (Binary dst src)));
11719   predicate(_kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::eq || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ne );
11720 
11721   ins_cost(150);
11722   size(4);
11723   format %{ "FCPYD$cmp $dst,$src" %}
11724   ins_encode %{
11725     __ fcpyd($dst$$FloatRegister, $src$$FloatRegister, (AsmCondition)($cmp$$cmpcode));
11726   %}
11727   ins_pipe(int_conditional_float_move);
11728 %}
11729 
11730 instruct cmovDL_reg_LEGT(cmpOpL_commute cmp, flagsRegL_LEGT xcc, regD dst, regD src) %{
11731   match(Set dst (CMoveD (Binary cmp xcc) (Binary dst src)));
11732   predicate(_kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::le || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::gt );
11733 
11734   ins_cost(150);
11735   size(4);
11736   format %{ "FCPYD$cmp $dst,$src" %}
11737   ins_encode %{
11738     __ fcpyd($dst$$FloatRegister, $src$$FloatRegister, (AsmCondition)($cmp$$cmpcode));
11739   %}
11740   ins_pipe(int_conditional_float_move);
11741 %}
11742 #endif // !AARCH64
11743 
11744 // ============================================================================
11745 // Safepoint Instruction
11746 #ifdef AARCH64
11747 instruct safePoint_poll(iRegP poll, flagsReg icc, RtempRegP tmp) %{
11748   match(SafePoint poll);
11749   // The handler stub kills Rtemp
11750   effect(USE poll, KILL tmp, KILL icc);
11751 
11752   size(4);
11753   format %{ "LDR   ZR,[$poll]\t! Safepoint: poll for GC" %}
11754   ins_encode %{
11755     __ relocate(relocInfo::poll_type);
11756     __ ldr(ZR, Address($poll$$Register));
11757   %}
11758   ins_pipe(loadPollP);
11759 %}
11760 #else
11761 // rather than KILL R12, it would be better to use any reg as
11762 // TEMP. Can't do that at this point because it crashes the compiler
11763 instruct safePoint_poll(iRegP poll, R12RegI tmp, flagsReg icc) %{
11764   match(SafePoint poll);
11765   effect(USE poll, KILL tmp, KILL icc);
11766 
11767   size(4);
11768   format %{ "LDR   $tmp,[$poll]\t! Safepoint: poll for GC" %}
11769   ins_encode %{
11770     __ relocate(relocInfo::poll_type);
11771     __ ldr($tmp$$Register, Address($poll$$Register));
11772   %}
11773   ins_pipe(loadPollP);
11774 %}
11775 #endif
11776 
11777 
11778 // ============================================================================
11779 // Call Instructions
11780 // Call Java Static Instruction
11781 instruct CallStaticJavaDirect( method meth ) %{
11782   match(CallStaticJava);
11783   predicate(! ((CallStaticJavaNode*)n)->is_method_handle_invoke());
11784   effect(USE meth);
11785 
11786   ins_cost(CALL_COST);
11787   format %{ "CALL,static ==> " %}
11788   ins_encode( Java_Static_Call( meth ), call_epilog );
11789   ins_pipe(simple_call);
11790 %}
11791 
11792 // Call Java Static Instruction (method handle version)
11793 instruct CallStaticJavaHandle( method meth ) %{
11794   match(CallStaticJava);
11795   predicate(((CallStaticJavaNode*)n)->is_method_handle_invoke());
11796   effect(USE meth);
11797   // FP is saved by all callees (for interpreter stack correction).
11798   // We use it here for a similar purpose, in {preserve,restore}_FP.
11799 
11800   ins_cost(CALL_COST);
11801   format %{ "CALL,static/MethodHandle ==> " %}
11802   ins_encode( preserve_SP, Java_Static_Call( meth ), restore_SP, call_epilog );
11803   ins_pipe(simple_call);
11804 %}
11805 
11806 // Call Java Dynamic Instruction
11807 instruct CallDynamicJavaDirect( method meth ) %{
11808   match(CallDynamicJava);
11809   effect(USE meth);
11810 
11811   ins_cost(CALL_COST);
11812   format %{ "MOV_OOP    (empty),R_R8\n\t"
11813             "CALL,dynamic  ; NOP ==> " %}
11814   ins_encode( Java_Dynamic_Call( meth ), call_epilog );
11815   ins_pipe(call);
11816 %}
11817 
11818 // Call Runtime Instruction
11819 instruct CallRuntimeDirect(method meth) %{
11820   match(CallRuntime);
11821   effect(USE meth);
11822   ins_cost(CALL_COST);
11823   format %{ "CALL,runtime" %}
11824 #ifdef AARCH64
11825   ins_encode( save_last_PC, Java_To_Runtime( meth ),
11826               call_epilog );
11827 #else
11828   ins_encode( Java_To_Runtime( meth ),
11829               call_epilog );
11830 #endif
11831   ins_pipe(simple_call);
11832 %}
11833 
11834 // Call runtime without safepoint - same as CallRuntime
11835 instruct CallLeafDirect(method meth) %{
11836   match(CallLeaf);
11837   effect(USE meth);
11838   ins_cost(CALL_COST);
11839   format %{ "CALL,runtime leaf" %}
11840   // TODO: ned save_last_PC here?
11841   ins_encode( Java_To_Runtime( meth ),
11842               call_epilog );
11843   ins_pipe(simple_call);
11844 %}
11845 
11846 // Call runtime without safepoint - same as CallLeaf
11847 instruct CallLeafNoFPDirect(method meth) %{
11848   match(CallLeafNoFP);
11849   effect(USE meth);
11850   ins_cost(CALL_COST);
11851   format %{ "CALL,runtime leaf nofp" %}
11852   // TODO: ned save_last_PC here?
11853   ins_encode( Java_To_Runtime( meth ),
11854               call_epilog );
11855   ins_pipe(simple_call);
11856 %}
11857 
11858 // Tail Call; Jump from runtime stub to Java code.
11859 // Also known as an 'interprocedural jump'.
11860 // Target of jump will eventually return to caller.
11861 // TailJump below removes the return address.
11862 instruct TailCalljmpInd(IPRegP jump_target, inline_cache_regP method_oop) %{
11863   match(TailCall jump_target method_oop );
11864 
11865   ins_cost(CALL_COST);
11866   format %{ "MOV    Rexception_pc, LR\n\t"
11867             "jump   $jump_target  \t! $method_oop holds method oop" %}
11868   ins_encode %{
11869     __ mov(Rexception_pc, LR);   // this is used only to call
11870                                  // StubRoutines::forward_exception_entry()
11871                                  // which expects PC of exception in
11872                                  // R5. FIXME?
11873     __ jump($jump_target$$Register);
11874   %}
11875   ins_pipe(tail_call);
11876 %}
11877 
11878 
11879 // Return Instruction
11880 instruct Ret() %{
11881   match(Return);
11882 
11883   format %{ "ret LR" %}
11884 
11885   ins_encode %{
11886     __ ret(LR);
11887   %}
11888 
11889   ins_pipe(br);
11890 %}
11891 
11892 
11893 // Tail Jump; remove the return address; jump to target.
11894 // TailCall above leaves the return address around.
11895 // TailJump is used in only one place, the rethrow_Java stub (fancy_jump=2).
11896 // ex_oop (Exception Oop) is needed in %o0 at the jump. As there would be a
11897 // "restore" before this instruction (in Epilogue), we need to materialize it
11898 // in %i0.
11899 instruct tailjmpInd(IPRegP jump_target, RExceptionRegP ex_oop) %{
11900   match( TailJump jump_target ex_oop );
11901   ins_cost(CALL_COST);
11902   format %{ "MOV    Rexception_pc, LR\n\t"
11903             "jump   $jump_target \t! $ex_oop holds exc. oop" %}
11904   ins_encode %{
11905     __ mov(Rexception_pc, LR);
11906     __ jump($jump_target$$Register);
11907   %}
11908   ins_pipe(tail_call);
11909 %}
11910 
11911 // Create exception oop: created by stack-crawling runtime code.
11912 // Created exception is now available to this handler, and is setup
11913 // just prior to jumping to this handler.  No code emitted.
11914 instruct CreateException( RExceptionRegP ex_oop )
11915 %{
11916   match(Set ex_oop (CreateEx));
11917   ins_cost(0);
11918 
11919   size(0);
11920   // use the following format syntax
11921   format %{ "! exception oop is in Rexception_obj; no code emitted" %}
11922   ins_encode();
11923   ins_pipe(empty);
11924 %}
11925 
11926 
11927 // Rethrow exception:
11928 // The exception oop will come in the first argument position.
11929 // Then JUMP (not call) to the rethrow stub code.
11930 instruct RethrowException()
11931 %{
11932   match(Rethrow);
11933   ins_cost(CALL_COST);
11934 
11935   // use the following format syntax
11936   format %{ "b    rethrow_stub" %}
11937   ins_encode %{
11938     Register scratch = R1_tmp;
11939     assert_different_registers(scratch, c_rarg0, LR);
11940     __ jump(OptoRuntime::rethrow_stub(), relocInfo::runtime_call_type, scratch);
11941   %}
11942   ins_pipe(tail_call);
11943 %}
11944 
11945 
11946 // Die now
11947 instruct ShouldNotReachHere( )
11948 %{
11949   match(Halt);
11950   ins_cost(CALL_COST);
11951 
11952   size(4);
11953   // Use the following format syntax
11954   format %{ "ShouldNotReachHere" %}
11955   ins_encode %{
11956 #ifdef AARCH64
11957     __ dpcs1(0xdead);
11958 #else
11959     __ udf(0xdead);
11960 #endif
11961   %}
11962   ins_pipe(tail_call);
11963 %}
11964 
11965 // ============================================================================
11966 // The 2nd slow-half of a subtype check.  Scan the subklass's 2ndary superklass
11967 // array for an instance of the superklass.  Set a hidden internal cache on a
11968 // hit (cache is checked with exposed code in gen_subtype_check()).  Return
11969 // not zero for a miss or zero for a hit.  The encoding ALSO sets flags.
11970 instruct partialSubtypeCheck( R0RegP index, R1RegP sub, R2RegP super, flagsRegP pcc, LRRegP lr ) %{
11971   match(Set index (PartialSubtypeCheck sub super));
11972   effect( KILL pcc, KILL lr );
11973   ins_cost(DEFAULT_COST*10);
11974   format %{ "CALL   PartialSubtypeCheck" %}
11975   ins_encode %{
11976     __ call(StubRoutines::Arm::partial_subtype_check(), relocInfo::runtime_call_type);
11977   %}
11978   ins_pipe(partial_subtype_check_pipe);
11979 %}
11980 
11981 /* instruct partialSubtypeCheck_vs_zero( flagsRegP pcc, o1RegP sub, o2RegP super, immP0 zero, o0RegP idx, o7RegP o7 ) %{ */
11982 /*   match(Set pcc (CmpP (PartialSubtypeCheck sub super) zero)); */
11983 /*   ins_pipe(partial_subtype_check_pipe); */
11984 /* %} */
11985 
11986 
11987 // ============================================================================
11988 // inlined locking and unlocking
11989 
11990 #ifdef AARCH64
11991 instruct cmpFastLock(flagsRegP pcc, iRegP object, iRegP box, iRegP scratch2, iRegP scratch, iRegP scratch3 )
11992 #else
11993 instruct cmpFastLock(flagsRegP pcc, iRegP object, iRegP box, iRegP scratch2, iRegP scratch )
11994 #endif
11995 %{
11996   match(Set pcc (FastLock object box));
11997 
11998 #ifdef AARCH64
11999   effect(TEMP scratch, TEMP scratch2, TEMP scratch3);
12000 #else
12001   effect(TEMP scratch, TEMP scratch2);
12002 #endif
12003   ins_cost(100);
12004 
12005 #ifdef AARCH64
12006   format %{ "FASTLOCK  $object, $box; KILL $scratch, $scratch2, $scratch3" %}
12007   ins_encode %{
12008     __ fast_lock($object$$Register, $box$$Register, $scratch$$Register, $scratch2$$Register, $scratch3$$Register);
12009   %}
12010 #else
12011   format %{ "FASTLOCK  $object, $box; KILL $scratch, $scratch2" %}
12012   ins_encode %{
12013     __ fast_lock($object$$Register, $box$$Register, $scratch$$Register, $scratch2$$Register);
12014   %}
12015 #endif
12016   ins_pipe(long_memory_op);
12017 %}
12018 
12019 
12020 #ifdef AARCH64
12021 instruct cmpFastUnlock(flagsRegP pcc, iRegP object, iRegP box, iRegP scratch2, iRegP scratch, iRegP scratch3 ) %{
12022   match(Set pcc (FastUnlock object box));
12023   effect(TEMP scratch, TEMP scratch2, TEMP scratch3);
12024   ins_cost(100);
12025 
12026   format %{ "FASTUNLOCK  $object, $box; KILL $scratch, $scratch2, $scratch3" %}
12027   ins_encode %{
12028     __ fast_unlock($object$$Register, $box$$Register, $scratch$$Register, $scratch2$$Register, $scratch3$$Register);
12029   %}
12030   ins_pipe(long_memory_op);
12031 %}
12032 #else
12033 instruct cmpFastUnlock(flagsRegP pcc, iRegP object, iRegP box, iRegP scratch2, iRegP scratch ) %{
12034   match(Set pcc (FastUnlock object box));
12035   effect(TEMP scratch, TEMP scratch2);
12036   ins_cost(100);
12037 
12038   format %{ "FASTUNLOCK  $object, $box; KILL $scratch, $scratch2" %}
12039   ins_encode %{
12040     __ fast_unlock($object$$Register, $box$$Register, $scratch$$Register, $scratch2$$Register);
12041   %}
12042   ins_pipe(long_memory_op);
12043 %}
12044 #endif
12045 
12046 #ifdef AARCH64
12047 // TODO: add version that takes immI cnt?
12048 instruct clear_array(iRegX cnt, iRegP base, iRegP ptr, iRegX temp, Universe dummy, flagsReg cpsr) %{
12049   match(Set dummy (ClearArray cnt base));
12050   effect(TEMP temp, TEMP ptr, KILL cpsr);
12051   ins_cost(300);
12052   format %{
12053       "        MOV    $temp,$cnt\n"
12054       "        ADD    $ptr,$base,$cnt\n"
12055       "        SUBS   $temp,$temp,16\t! Count down dword pair in bytes\n"
12056       "        B.lt   done16\n"
12057       "loop:   STP    ZR,ZR,[$ptr,-16]!\n"
12058       "        SUBS   $temp,$temp,16\t! Count down dword pair in bytes\n"
12059       "        B.ge   loop\t! Clearing loop\n"
12060       "done16: ADDS   $temp,$temp,8\t! Room for 1 more long?\n"
12061       "        B.lt   done\n"
12062       "        STR    ZR,[$base+$temp]\n"
12063       "done:"
12064   %}
12065   ins_encode %{
12066     // TODO: preload?
12067     __ mov($temp$$Register, $cnt$$Register);
12068     __ add($ptr$$Register, $base$$Register, $cnt$$Register);
12069     Label loop, done, done16;
12070     __ subs($temp$$Register, $temp$$Register, 16);
12071     __ b(done16, lt);
12072     __ bind(loop);
12073     __ stp(ZR, ZR, Address($ptr$$Register, -16, pre_indexed));
12074     __ subs($temp$$Register, $temp$$Register, 16);
12075     __ b(loop, ge);
12076     __ bind(done16);
12077     __ adds($temp$$Register, $temp$$Register, 8);
12078     __ b(done, lt);
12079     // $temp should be 0 here
12080     __ str(ZR, Address($base$$Register, $temp$$Register));
12081     __ bind(done);
12082   %}
12083   ins_pipe(long_memory_op);
12084 %}
12085 #else
12086 // Count and Base registers are fixed because the allocator cannot
12087 // kill unknown registers.  The encodings are generic.
12088 instruct clear_array(iRegX cnt, iRegP base, iRegI temp, iRegX zero, Universe dummy, flagsReg cpsr) %{
12089   match(Set dummy (ClearArray cnt base));
12090   effect(TEMP temp, TEMP zero, KILL cpsr);
12091   ins_cost(300);
12092   format %{ "MOV    $zero,0\n"
12093       "        MOV    $temp,$cnt\n"
12094       "loop:   SUBS   $temp,$temp,4\t! Count down a dword of bytes\n"
12095       "        STR.ge $zero,[$base+$temp]\t! delay slot"
12096       "        B.gt   loop\t\t! Clearing loop\n" %}
12097   ins_encode %{
12098     __ mov($zero$$Register, 0);
12099     __ mov($temp$$Register, $cnt$$Register);
12100     Label(loop);
12101     __ bind(loop);
12102     __ subs($temp$$Register, $temp$$Register, 4);
12103     __ str($zero$$Register, Address($base$$Register, $temp$$Register), ge);
12104     __ b(loop, gt);
12105   %}
12106   ins_pipe(long_memory_op);
12107 %}
12108 #endif
12109 
12110 #ifdef XXX
12111 // FIXME: Why R0/R1/R2/R3?
12112 instruct string_compare(R0RegP str1, R1RegP str2, R2RegI cnt1, R3RegI cnt2, iRegI result,
12113                         iRegI tmp1, iRegI tmp2, flagsReg ccr) %{
12114   predicate(!CompactStrings);
12115   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
12116   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL ccr, TEMP tmp1, TEMP tmp2);
12117   ins_cost(300);
12118   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   // TEMP $tmp1, $tmp2" %}
12119   ins_encode( enc_String_Compare(str1, str2, cnt1, cnt2, result, tmp1, tmp2) );
12120 
12121   ins_pipe(long_memory_op);
12122 %}
12123 
12124 // FIXME: Why R0/R1/R2?
12125 instruct string_equals(R0RegP str1, R1RegP str2, R2RegI cnt, iRegI result, iRegI tmp1, iRegI tmp2,
12126                        flagsReg ccr) %{
12127   predicate(!CompactStrings);
12128   match(Set result (StrEquals (Binary str1 str2) cnt));
12129   effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt, TEMP tmp1, TEMP tmp2, TEMP result, KILL ccr);
12130 
12131   ins_cost(300);
12132   format %{ "String Equals $str1,$str2,$cnt -> $result   // TEMP $tmp1, $tmp2" %}
12133   ins_encode( enc_String_Equals(str1, str2, cnt, result, tmp1, tmp2) );
12134   ins_pipe(long_memory_op);
12135 %}
12136 
12137 // FIXME: Why R0/R1?
12138 instruct array_equals(R0RegP ary1, R1RegP ary2, iRegI tmp1, iRegI tmp2, iRegI tmp3, iRegI result,
12139                       flagsReg ccr) %{
12140   predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
12141   match(Set result (AryEq ary1 ary2));
12142   effect(USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP result, KILL ccr);
12143 
12144   ins_cost(300);
12145   format %{ "Array Equals $ary1,$ary2 -> $result   // TEMP $tmp1,$tmp2,$tmp3" %}
12146   ins_encode( enc_Array_Equals(ary1, ary2, tmp1, tmp2, tmp3, result));
12147   ins_pipe(long_memory_op);
12148 %}
12149 #endif
12150 
12151 //---------- Zeros Count Instructions ------------------------------------------
12152 
12153 instruct countLeadingZerosI(iRegI dst, iRegI src) %{
12154   match(Set dst (CountLeadingZerosI src));
12155   size(4);
12156   format %{ "CLZ_32 $dst,$src" %}
12157   ins_encode %{
12158     __ clz_32($dst$$Register, $src$$Register);
12159   %}
12160   ins_pipe(ialu_reg);
12161 %}
12162 
12163 #ifdef AARCH64
12164 instruct countLeadingZerosL(iRegI dst, iRegL src) %{
12165   match(Set dst (CountLeadingZerosL src));
12166   size(4);
12167   format %{ "CLZ $dst,$src" %}
12168   ins_encode %{
12169     __ clz($dst$$Register, $src$$Register);
12170   %}
12171   ins_pipe(ialu_reg);
12172 %}
12173 #else
12174 instruct countLeadingZerosL(iRegI dst, iRegL src, iRegI tmp, flagsReg ccr) %{
12175   match(Set dst (CountLeadingZerosL src));
12176   effect(TEMP tmp, TEMP dst, KILL ccr);
12177   size(16);
12178   format %{ "CLZ    $dst,$src.hi\n\t"
12179             "TEQ    $dst,32\n\t"
12180             "CLZ.eq $tmp,$src.lo\n\t"
12181             "ADD.eq $dst, $dst, $tmp\n\t" %}
12182   ins_encode %{
12183     __ clz($dst$$Register, $src$$Register->successor());
12184     __ teq($dst$$Register, 32);
12185     __ clz($tmp$$Register, $src$$Register, eq);
12186     __ add($dst$$Register, $dst$$Register, $tmp$$Register, eq);
12187   %}
12188   ins_pipe(ialu_reg);
12189 %}
12190 #endif
12191 
12192 instruct countTrailingZerosI(iRegI dst, iRegI src, iRegI tmp) %{
12193   match(Set dst (CountTrailingZerosI src));
12194   effect(TEMP tmp);
12195   size(8);
12196   format %{ "RBIT_32 $tmp, $src\n\t"
12197             "CLZ_32  $dst,$tmp" %}
12198   ins_encode %{
12199     __ rbit_32($tmp$$Register, $src$$Register);
12200     __ clz_32($dst$$Register, $tmp$$Register);
12201   %}
12202   ins_pipe(ialu_reg);
12203 %}
12204 
12205 #ifdef AARCH64
12206 instruct countTrailingZerosL(iRegI dst, iRegL src, iRegL tmp) %{
12207   match(Set dst (CountTrailingZerosL src));
12208   effect(TEMP tmp);
12209   size(8);
12210   format %{ "RBIT $tmp, $src\n\t"
12211             "CLZ  $dst,$tmp" %}
12212   ins_encode %{
12213     __ rbit($tmp$$Register, $src$$Register);
12214     __ clz($dst$$Register, $tmp$$Register);
12215   %}
12216   ins_pipe(ialu_reg);
12217 %}
12218 #else
12219 instruct countTrailingZerosL(iRegI dst, iRegL src, iRegI tmp, flagsReg ccr) %{
12220   match(Set dst (CountTrailingZerosL src));
12221   effect(TEMP tmp, TEMP dst, KILL ccr);
12222   size(24);
12223   format %{ "RBIT   $tmp,$src.lo\n\t"
12224             "CLZ    $dst,$tmp\n\t"
12225             "TEQ    $dst,32\n\t"
12226             "RBIT   $tmp,$src.hi\n\t"
12227             "CLZ.eq $tmp,$tmp\n\t"
12228             "ADD.eq $dst,$dst,$tmp\n\t" %}
12229   ins_encode %{
12230     __ rbit($tmp$$Register, $src$$Register);
12231     __ clz($dst$$Register, $tmp$$Register);
12232     __ teq($dst$$Register, 32);
12233     __ rbit($tmp$$Register, $src$$Register->successor());
12234     __ clz($tmp$$Register, $tmp$$Register, eq);
12235     __ add($dst$$Register, $dst$$Register, $tmp$$Register, eq);
12236   %}
12237   ins_pipe(ialu_reg);
12238 %}
12239 #endif
12240 
12241 
12242 //---------- Population Count Instructions -------------------------------------
12243 
12244 #ifdef AARCH64
12245 instruct popCountI(iRegI dst, iRegI src, regD_low tmp) %{
12246   predicate(UsePopCountInstruction);
12247   match(Set dst (PopCountI src));
12248   effect(TEMP tmp);
12249   size(20);
12250 
12251   format %{ "MOV_W      $dst,$src\n\t"
12252             "FMOV_dx    $tmp,$dst\n\t"
12253             "VCNT       $tmp.8B,$tmp.8B\n\t"
12254             "ADDV       $tmp.B,$tmp.8B\n\t"
12255             "FMRS       $dst,$tmp" %}
12256 
12257   ins_encode %{
12258     __ mov_w($dst$$Register, $src$$Register);
12259     __ fmov_dx($tmp$$FloatRegister, $dst$$Register);
12260     int quad = 0;
12261     int cnt_size = 0; // VELEM_SIZE_8
12262     __ vcnt($tmp$$FloatRegister, $tmp$$FloatRegister, quad, cnt_size);
12263     int add_size = 0; // VELEM_SIZE_8
12264     __ addv($tmp$$FloatRegister, $tmp$$FloatRegister, quad, add_size);
12265     __ fmrs($dst$$Register, $tmp$$FloatRegister);
12266   %}
12267   ins_pipe(ialu_reg); // FIXME
12268 %}
12269 #else
12270 instruct popCountI(iRegI dst, iRegI src, regD_low tmp) %{
12271   predicate(UsePopCountInstruction);
12272   match(Set dst (PopCountI src));
12273   effect(TEMP tmp);
12274 
12275   format %{ "FMSR       $tmp,$src\n\t"
12276             "VCNT.8     $tmp,$tmp\n\t"
12277             "VPADDL.U8  $tmp,$tmp\n\t"
12278             "VPADDL.U16 $tmp,$tmp\n\t"
12279             "FMRS       $dst,$tmp" %}
12280   size(20);
12281 
12282   ins_encode %{
12283     __ fmsr($tmp$$FloatRegister, $src$$Register);
12284     __ vcnt($tmp$$FloatRegister, $tmp$$FloatRegister);
12285     __ vpaddl($tmp$$FloatRegister, $tmp$$FloatRegister, 8, 0);
12286     __ vpaddl($tmp$$FloatRegister, $tmp$$FloatRegister, 16, 0);
12287     __ fmrs($dst$$Register, $tmp$$FloatRegister);
12288   %}
12289   ins_pipe(ialu_reg); // FIXME
12290 %}
12291 #endif
12292 
12293 #ifdef AARCH64
12294 instruct popCountL(iRegI dst, iRegL src, regD tmp) %{
12295   predicate(UsePopCountInstruction);
12296   match(Set dst (PopCountL src));
12297   effect(TEMP tmp);
12298   size(16);
12299 
12300   format %{ "FMOV_dx    $tmp,$src\n\t"
12301             "VCNT       $tmp.8B,$tmp.8B\n\t"
12302             "ADDV       $tmp.B,$tmp.8B\n\t"
12303             "FMOV_ws    $dst,$tmp" %}
12304 
12305   ins_encode %{
12306     __ fmov_dx($tmp$$FloatRegister, $src$$Register);
12307     int quad = 0;
12308     int cnt_size = 0;
12309     __ vcnt($tmp$$FloatRegister, $tmp$$FloatRegister, quad, cnt_size);
12310     int add_size = 0;
12311     __ addv($tmp$$FloatRegister, $tmp$$FloatRegister, quad, add_size);
12312     __ fmov_ws($dst$$Register, $tmp$$FloatRegister);
12313   %}
12314   ins_pipe(ialu_reg); // FIXME
12315 %}
12316 #else
12317 // Note: Long.bitCount(long) returns an int.
12318 instruct popCountL(iRegI dst, iRegL src, regD_low tmp) %{
12319   predicate(UsePopCountInstruction);
12320   match(Set dst (PopCountL src));
12321   effect(TEMP tmp);
12322 
12323   format %{ "FMDRR       $tmp,$src.lo,$src.hi\n\t"
12324             "VCNT.8      $tmp,$tmp\n\t"
12325             "VPADDL.U8   $tmp,$tmp\n\t"
12326             "VPADDL.U16  $tmp,$tmp\n\t"
12327             "VPADDL.U32  $tmp,$tmp\n\t"
12328             "FMRS        $dst,$tmp" %}
12329 
12330   size(32);
12331 
12332   ins_encode %{
12333     __ fmdrr($tmp$$FloatRegister, $src$$Register, $src$$Register->successor());
12334     __ vcnt($tmp$$FloatRegister, $tmp$$FloatRegister);
12335     __ vpaddl($tmp$$FloatRegister, $tmp$$FloatRegister, 8, 0);
12336     __ vpaddl($tmp$$FloatRegister, $tmp$$FloatRegister, 16, 0);
12337     __ vpaddl($tmp$$FloatRegister, $tmp$$FloatRegister, 32, 0);
12338     __ fmrs($dst$$Register, $tmp$$FloatRegister);
12339   %}
12340   ins_pipe(ialu_reg);
12341 %}
12342 #endif
12343 
12344 
12345 // ============================================================================
12346 //------------Bytes reverse--------------------------------------------------
12347 
12348 instruct bytes_reverse_int(iRegI dst, iRegI src) %{
12349   match(Set dst (ReverseBytesI src));
12350 
12351   size(4);
12352   format %{ "REV32 $dst,$src" %}
12353   ins_encode %{
12354 #ifdef AARCH64
12355     __ rev_w($dst$$Register, $src$$Register);
12356     // high 32 bits zeroed, not sign extended
12357 #else
12358     __ rev($dst$$Register, $src$$Register);
12359 #endif
12360   %}
12361   ins_pipe( iload_mem ); // FIXME
12362 %}
12363 
12364 instruct bytes_reverse_long(iRegL dst, iRegL src) %{
12365   match(Set dst (ReverseBytesL src));
12366 #ifdef AARCH64
12367 //size(4);
12368   format %{ "REV $dst,$src"  %}
12369   ins_encode %{
12370     __ rev($dst$$Register, $src$$Register);
12371   %}
12372   ins_pipe(ialu_reg_reg); // FIXME
12373 #else
12374   effect(TEMP dst);
12375   size(8);
12376   format %{ "REV $dst.lo,$src.lo\n\t"
12377             "REV $dst.hi,$src.hi" %}
12378   ins_encode %{
12379     __ rev($dst$$Register, $src$$Register->successor());
12380     __ rev($dst$$Register->successor(), $src$$Register);
12381   %}
12382   ins_pipe( iload_mem ); // FIXME
12383 #endif
12384 %}
12385 
12386 instruct bytes_reverse_unsigned_short(iRegI dst, iRegI src) %{
12387   match(Set dst (ReverseBytesUS src));
12388 #ifdef AARCH64
12389   size(4);
12390   format %{ "REV16_W $dst,$src" %}
12391   ins_encode %{
12392     __ rev16_w($dst$$Register, $src$$Register);
12393     // high 32 bits zeroed
12394   %}
12395 #else
12396   size(4);
12397   format %{ "REV16 $dst,$src" %}
12398   ins_encode %{
12399     __ rev16($dst$$Register, $src$$Register);
12400   %}
12401 #endif
12402   ins_pipe( iload_mem ); // FIXME
12403 %}
12404 
12405 instruct bytes_reverse_short(iRegI dst, iRegI src) %{
12406   match(Set dst (ReverseBytesS src));
12407 #ifdef AARCH64
12408   size(8);
12409   format %{ "REV16_W $dst,$src\n\t"
12410             "SIGN_EXT16 $dst" %}
12411   ins_encode %{
12412     __ rev16_w($dst$$Register, $src$$Register);
12413     __ sign_extend($dst$$Register, $dst$$Register, 16);
12414   %}
12415 #else
12416   size(4);
12417   format %{ "REVSH $dst,$src" %}
12418   ins_encode %{
12419     __ revsh($dst$$Register, $src$$Register);
12420   %}
12421 #endif
12422   ins_pipe( iload_mem ); // FIXME
12423 %}
12424 
12425 
12426 // ====================VECTOR INSTRUCTIONS=====================================
12427 
12428 // Load Aligned Packed values into a Double Register
12429 instruct loadV8(vecD dst, memoryD mem) %{
12430   predicate(n->as_LoadVector()->memory_size() == 8);
12431   match(Set dst (LoadVector mem));
12432   ins_cost(MEMORY_REF_COST);
12433   size(4);
12434   format %{ "FLDD   $mem,$dst\t! load vector (8 bytes)" %}
12435   ins_encode %{
12436     __ ldr_double($dst$$FloatRegister, $mem$$Address);
12437   %}
12438   ins_pipe(floadD_mem);
12439 %}
12440 
12441 // Load Aligned Packed values into a Double Register Pair
12442 instruct loadV16(vecX dst, memoryvld mem) %{
12443   predicate(n->as_LoadVector()->memory_size() == 16);
12444   match(Set dst (LoadVector mem));
12445   ins_cost(MEMORY_REF_COST);
12446   size(4);
12447   format %{ "VLD1   $mem,$dst.Q\t! load vector (16 bytes)" %}
12448   ins_encode %{
12449     __ vld1($dst$$FloatRegister, $mem$$Address, MacroAssembler::VELEM_SIZE_16, 128);
12450   %}
12451   ins_pipe(floadD_mem); // FIXME
12452 %}
12453 
12454 // Store Vector in Double register to memory
12455 instruct storeV8(memoryD mem, vecD src) %{
12456   predicate(n->as_StoreVector()->memory_size() == 8);
12457   match(Set mem (StoreVector mem src));
12458   ins_cost(MEMORY_REF_COST);
12459   size(4);
12460   format %{ "FSTD   $src,$mem\t! store vector (8 bytes)" %}
12461   ins_encode %{
12462     __ str_double($src$$FloatRegister, $mem$$Address);
12463   %}
12464   ins_pipe(fstoreD_mem_reg);
12465 %}
12466 
12467 // Store Vector in Double Register Pair to memory
12468 instruct storeV16(memoryvld mem, vecX src) %{
12469   predicate(n->as_StoreVector()->memory_size() == 16);
12470   match(Set mem (StoreVector mem src));
12471   ins_cost(MEMORY_REF_COST);
12472   size(4);
12473   format %{ "VST1   $src,$mem\t! store vector (16 bytes)" %}
12474   ins_encode %{
12475     __ vst1($src$$FloatRegister, $mem$$Address, MacroAssembler::VELEM_SIZE_16, 128);
12476   %}
12477   ins_pipe(fstoreD_mem_reg); // FIXME
12478 %}
12479 
12480 #ifndef AARCH64
12481 // Replicate scalar to packed byte values in Double register
12482 instruct Repl8B_reg(vecD dst, iRegI src, iRegI tmp) %{
12483   predicate(n->as_Vector()->length() == 8);
12484   match(Set dst (ReplicateB src));
12485   ins_cost(DEFAULT_COST*4);
12486   effect(TEMP tmp);
12487   size(16);
12488 
12489   // FIXME: could use PKH instruction instead?
12490   format %{ "LSL      $tmp, $src, 24 \n\t"
12491             "OR       $tmp, $tmp, ($tmp >> 8) \n\t"
12492             "OR       $tmp, $tmp, ($tmp >> 16) \n\t"
12493             "FMDRR    $dst,$tmp,$tmp\t" %}
12494   ins_encode %{
12495     __ mov($tmp$$Register, AsmOperand($src$$Register, lsl, 24));
12496     __ orr($tmp$$Register, $tmp$$Register, AsmOperand($tmp$$Register, lsr, 8));
12497     __ orr($tmp$$Register, $tmp$$Register, AsmOperand($tmp$$Register, lsr, 16));
12498     __ fmdrr($dst$$FloatRegister, $tmp$$Register, $tmp$$Register);
12499   %}
12500   ins_pipe(ialu_reg); // FIXME
12501 %}
12502 #endif /* !AARCH64 */
12503 
12504 // Replicate scalar to packed byte values in Double register
12505 instruct Repl8B_reg_simd(vecD dst, iRegI src) %{
12506   predicate(n->as_Vector()->length_in_bytes() == 8 && VM_Version::has_simd());
12507   match(Set dst (ReplicateB src));
12508   size(4);
12509 
12510   format %{ "VDUP.8 $dst,$src\t" %}
12511   ins_encode %{
12512     bool quad = false;
12513     __ vdupI($dst$$FloatRegister, $src$$Register,
12514              MacroAssembler::VELEM_SIZE_8, quad);
12515   %}
12516   ins_pipe(ialu_reg); // FIXME
12517 %}
12518 
12519 // Replicate scalar to packed byte values in Double register pair
12520 instruct Repl16B_reg(vecX dst, iRegI src) %{
12521   predicate(n->as_Vector()->length_in_bytes() == 16);
12522   match(Set dst (ReplicateB src));
12523   size(4);
12524 
12525   format %{ "VDUP.8 $dst.Q,$src\t" %}
12526   ins_encode %{
12527     bool quad = true;
12528     __ vdupI($dst$$FloatRegister, $src$$Register,
12529              MacroAssembler::VELEM_SIZE_8, quad);
12530   %}
12531   ins_pipe(ialu_reg); // FIXME
12532 %}
12533 
12534 #ifndef AARCH64
12535 // Replicate scalar constant to packed byte values in Double register
12536 instruct Repl8B_immI(vecD dst, immI src, iRegI tmp) %{
12537   predicate(n->as_Vector()->length() == 8);
12538   match(Set dst (ReplicateB src));
12539   ins_cost(DEFAULT_COST*2);
12540   effect(TEMP tmp);
12541   size(12);
12542 
12543   format %{ "MOV      $tmp, Repl4($src))\n\t"
12544             "FMDRR    $dst,$tmp,$tmp\t" %}
12545   ins_encode( LdReplImmI(src, dst, tmp, (4), (1)) );
12546   ins_pipe(loadConFD); // FIXME
12547 %}
12548 #endif /* !AARCH64 */
12549 
12550 // Replicate scalar constant to packed byte values in Double register
12551 // TODO: support negative constants with MVNI?
12552 instruct Repl8B_immU8(vecD dst, immU8 src) %{
12553   predicate(n->as_Vector()->length_in_bytes() == 8 && VM_Version::has_simd());
12554   match(Set dst (ReplicateB src));
12555   size(4);
12556 
12557   format %{ "VMOV.U8  $dst,$src" %}
12558   ins_encode %{
12559     bool quad = false;
12560     __ vmovI($dst$$FloatRegister, $src$$constant,
12561              MacroAssembler::VELEM_SIZE_8, quad);
12562   %}
12563   ins_pipe(loadConFD); // FIXME
12564 %}
12565 
12566 // Replicate scalar constant to packed byte values in Double register pair
12567 instruct Repl16B_immU8(vecX dst, immU8 src) %{
12568   predicate(n->as_Vector()->length_in_bytes() == 16 && VM_Version::has_simd());
12569   match(Set dst (ReplicateB src));
12570   size(4);
12571 
12572   format %{ "VMOV.U8  $dst.Q,$src" %}
12573   ins_encode %{
12574     bool quad = true;
12575     __ vmovI($dst$$FloatRegister, $src$$constant,
12576              MacroAssembler::VELEM_SIZE_8, quad);
12577   %}
12578   ins_pipe(loadConFD); // FIXME
12579 %}
12580 
12581 #ifndef AARCH64
12582 // Replicate scalar to packed short/char values into Double register
12583 instruct Repl4S_reg(vecD dst, iRegI src, iRegI tmp) %{
12584   predicate(n->as_Vector()->length() == 4);
12585   match(Set dst (ReplicateS src));
12586   ins_cost(DEFAULT_COST*3);
12587   effect(TEMP tmp);
12588   size(12);
12589 
12590   // FIXME: could use PKH instruction instead?
12591   format %{ "LSL      $tmp, $src, 16 \n\t"
12592             "OR       $tmp, $tmp, ($tmp >> 16) \n\t"
12593             "FMDRR    $dst,$tmp,$tmp\t" %}
12594   ins_encode %{
12595     __ mov($tmp$$Register, AsmOperand($src$$Register, lsl, 16));
12596     __ orr($tmp$$Register, $tmp$$Register, AsmOperand($tmp$$Register, lsr, 16));
12597     __ fmdrr($dst$$FloatRegister, $tmp$$Register, $tmp$$Register);
12598   %}
12599   ins_pipe(ialu_reg); // FIXME
12600 %}
12601 #endif /* !AARCH64 */
12602 
12603 // Replicate scalar to packed byte values in Double register
12604 instruct Repl4S_reg_simd(vecD dst, iRegI src) %{
12605   predicate(n->as_Vector()->length_in_bytes() == 8 && VM_Version::has_simd());
12606   match(Set dst (ReplicateS src));
12607   size(4);
12608 
12609   format %{ "VDUP.16 $dst,$src\t" %}
12610   ins_encode %{
12611     bool quad = false;
12612     __ vdupI($dst$$FloatRegister, $src$$Register,
12613              MacroAssembler::VELEM_SIZE_16, quad);
12614   %}
12615   ins_pipe(ialu_reg); // FIXME
12616 %}
12617 
12618 // Replicate scalar to packed byte values in Double register pair
12619 instruct Repl8S_reg(vecX dst, iRegI src) %{
12620   predicate(n->as_Vector()->length_in_bytes() == 16 && VM_Version::has_simd());
12621   match(Set dst (ReplicateS src));
12622   size(4);
12623 
12624   format %{ "VDUP.16 $dst.Q,$src\t" %}
12625   ins_encode %{
12626     bool quad = true;
12627     __ vdupI($dst$$FloatRegister, $src$$Register,
12628              MacroAssembler::VELEM_SIZE_16, quad);
12629   %}
12630   ins_pipe(ialu_reg); // FIXME
12631 %}
12632 
12633 
12634 #ifndef AARCH64
12635 // Replicate scalar constant to packed short/char values in Double register
12636 instruct Repl4S_immI(vecD dst, immI src, iRegP tmp) %{
12637   predicate(n->as_Vector()->length() == 4);
12638   match(Set dst (ReplicateS src));
12639   effect(TEMP tmp);
12640   size(12);
12641   ins_cost(DEFAULT_COST*4); // FIXME
12642 
12643   format %{ "MOV      $tmp, Repl2($src))\n\t"
12644             "FMDRR    $dst,$tmp,$tmp\t" %}
12645   ins_encode( LdReplImmI(src, dst, tmp, (2), (2)) );
12646   ins_pipe(loadConFD); // FIXME
12647 %}
12648 #endif /* !AARCH64 */
12649 
12650 // Replicate scalar constant to packed byte values in Double register
12651 instruct Repl4S_immU8(vecD dst, immU8 src) %{
12652   predicate(n->as_Vector()->length_in_bytes() == 8 && VM_Version::has_simd());
12653   match(Set dst (ReplicateS src));
12654   size(4);
12655 
12656   format %{ "VMOV.U16  $dst,$src" %}
12657   ins_encode %{
12658     bool quad = false;
12659     __ vmovI($dst$$FloatRegister, $src$$constant,
12660              MacroAssembler::VELEM_SIZE_16, quad);
12661   %}
12662   ins_pipe(loadConFD); // FIXME
12663 %}
12664 
12665 // Replicate scalar constant to packed byte values in Double register pair
12666 instruct Repl8S_immU8(vecX dst, immU8 src) %{
12667   predicate(n->as_Vector()->length_in_bytes() == 16 && VM_Version::has_simd());
12668   match(Set dst (ReplicateS src));
12669   size(4);
12670 
12671   format %{ "VMOV.U16  $dst.Q,$src" %}
12672   ins_encode %{
12673     bool quad = true;
12674     __ vmovI($dst$$FloatRegister, $src$$constant,
12675              MacroAssembler::VELEM_SIZE_16, quad);
12676   %}
12677   ins_pipe(loadConFD); // FIXME
12678 %}
12679 
12680 #ifndef AARCH64
12681 // Replicate scalar to packed int values in Double register
12682 instruct Repl2I_reg(vecD dst, iRegI src) %{
12683   predicate(n->as_Vector()->length() == 2);
12684   match(Set dst (ReplicateI src));
12685   size(4);
12686 
12687   format %{ "FMDRR    $dst,$src,$src\t" %}
12688   ins_encode %{
12689     __ fmdrr($dst$$FloatRegister, $src$$Register, $src$$Register);
12690   %}
12691   ins_pipe(ialu_reg); // FIXME
12692 %}
12693 
12694 // Replicate scalar to packed int values in Double register pair
12695 instruct Repl4I_reg(vecX dst, iRegI src) %{
12696   predicate(n->as_Vector()->length() == 4);
12697   match(Set dst (ReplicateI src));
12698   ins_cost(DEFAULT_COST*2);
12699   size(8);
12700 
12701   format %{ "FMDRR    $dst.lo,$src,$src\n\t"
12702             "FMDRR    $dst.hi,$src,$src" %}
12703 
12704   ins_encode %{
12705     __ fmdrr($dst$$FloatRegister, $src$$Register, $src$$Register);
12706     __ fmdrr($dst$$FloatRegister->successor()->successor(),
12707              $src$$Register, $src$$Register);
12708   %}
12709   ins_pipe(ialu_reg); // FIXME
12710 %}
12711 #endif /* !AARCH64 */
12712 
12713 // Replicate scalar to packed int values in Double register
12714 instruct Repl2I_reg_simd(vecD dst, iRegI src) %{
12715   predicate(n->as_Vector()->length_in_bytes() == 8 && VM_Version::has_simd());
12716   match(Set dst (ReplicateI src));
12717   size(4);
12718 
12719   format %{ "VDUP.32 $dst.D,$src\t" %}
12720   ins_encode %{
12721     bool quad = false;
12722     __ vdupI($dst$$FloatRegister, $src$$Register,
12723              MacroAssembler::VELEM_SIZE_32, quad);
12724   %}
12725   ins_pipe(ialu_reg); // FIXME
12726 %}
12727 
12728 // Replicate scalar to packed int values in Double register pair
12729 instruct Repl4I_reg_simd(vecX dst, iRegI src) %{
12730   predicate(n->as_Vector()->length_in_bytes() == 16 && VM_Version::has_simd());
12731   match(Set dst (ReplicateI src));
12732   size(4);
12733 
12734   format %{ "VDUP.32 $dst.Q,$src\t" %}
12735   ins_encode %{
12736     bool quad = true;
12737     __ vdupI($dst$$FloatRegister, $src$$Register,
12738              MacroAssembler::VELEM_SIZE_32, quad);
12739   %}
12740   ins_pipe(ialu_reg); // FIXME
12741 %}
12742 
12743 
12744 #ifndef AARCH64
12745 // Replicate scalar zero constant to packed int values in Double register
12746 instruct Repl2I_immI(vecD dst, immI src, iRegI tmp) %{
12747   predicate(n->as_Vector()->length() == 2);
12748   match(Set dst (ReplicateI src));
12749   effect(TEMP tmp);
12750   size(12);
12751   ins_cost(DEFAULT_COST*4); // FIXME
12752 
12753   format %{ "MOV      $tmp, Repl1($src))\n\t"
12754             "FMDRR    $dst,$tmp,$tmp\t" %}
12755   ins_encode( LdReplImmI(src, dst, tmp, (1), (4)) );
12756   ins_pipe(loadConFD); // FIXME
12757 %}
12758 #endif /* !AARCH64 */
12759 
12760 // Replicate scalar constant to packed byte values in Double register
12761 instruct Repl2I_immU8(vecD dst, immU8 src) %{
12762   predicate(n->as_Vector()->length_in_bytes() == 8 && VM_Version::has_simd());
12763   match(Set dst (ReplicateI src));
12764   size(4);
12765 
12766   format %{ "VMOV.I32  $dst.D,$src" %}
12767   ins_encode %{
12768     bool quad = false;
12769     __ vmovI($dst$$FloatRegister, $src$$constant,
12770              MacroAssembler::VELEM_SIZE_32, quad);
12771   %}
12772   ins_pipe(loadConFD); // FIXME
12773 %}
12774 
12775 // Replicate scalar constant to packed byte values in Double register pair
12776 instruct Repl4I_immU8(vecX dst, immU8 src) %{
12777   predicate(n->as_Vector()->length_in_bytes() == 16 && VM_Version::has_simd());
12778   match(Set dst (ReplicateI src));
12779   size(4);
12780 
12781   format %{ "VMOV.I32  $dst.Q,$src" %}
12782   ins_encode %{
12783     bool quad = true;
12784     __ vmovI($dst$$FloatRegister, $src$$constant,
12785              MacroAssembler::VELEM_SIZE_32, quad);
12786   %}
12787   ins_pipe(loadConFD); // FIXME
12788 %}
12789 
12790 #ifdef AARCH64
12791 // Replicate scalar to packed byte values in Double register pair
12792 instruct Repl2L_reg(vecX dst, iRegL src) %{
12793   predicate(n->as_Vector()->length() == 2);
12794   match(Set dst (ReplicateL src));
12795   size(4*1);
12796   ins_cost(DEFAULT_COST*1); // FIXME
12797 
12798   format %{ "VDUP.2D $dst.Q,$src\t" %}
12799   ins_encode %{
12800     bool quad = true;
12801     __ vdupI($dst$$FloatRegister, $src$$Register,
12802              MacroAssembler::VELEM_SIZE_64, quad);
12803   %}
12804   ins_pipe(ialu_reg); // FIXME
12805 %}
12806 #else /* !AARCH64 */
12807 // Replicate scalar to packed byte values in Double register pair
12808 instruct Repl2L_reg(vecX dst, iRegL src) %{
12809   predicate(n->as_Vector()->length() == 2);
12810   match(Set dst (ReplicateL src));
12811   size(8);
12812   ins_cost(DEFAULT_COST*2); // FIXME
12813 
12814   format %{ "FMDRR $dst.D,$src.lo,$src.hi\t\n"
12815             "FMDRR $dst.D.next,$src.lo,$src.hi" %}
12816   ins_encode %{
12817     __ fmdrr($dst$$FloatRegister, $src$$Register, $src$$Register->successor());
12818     __ fmdrr($dst$$FloatRegister->successor()->successor(),
12819              $src$$Register, $src$$Register->successor());
12820   %}
12821   ins_pipe(ialu_reg); // FIXME
12822 %}
12823 
12824 
12825 // Replicate scalar to packed float values in Double register
12826 instruct Repl2F_regI(vecD dst, iRegI src) %{
12827   predicate(n->as_Vector()->length() == 2);
12828   match(Set dst (ReplicateF src));
12829   size(4);
12830 
12831   format %{ "FMDRR    $dst.D,$src,$src\t" %}
12832   ins_encode %{
12833     __ fmdrr($dst$$FloatRegister, $src$$Register, $src$$Register);
12834   %}
12835   ins_pipe(ialu_reg); // FIXME
12836 %}
12837 
12838 // Replicate scalar to packed float values in Double register
12839 instruct Repl2F_reg_vfp(vecD dst, regF src) %{
12840   predicate(n->as_Vector()->length() == 2);
12841   match(Set dst (ReplicateF src));
12842   size(4*2);
12843   ins_cost(DEFAULT_COST*2); // FIXME
12844 
12845   expand %{
12846     iRegI tmp;
12847     MoveF2I_reg_reg(tmp, src);
12848     Repl2F_regI(dst,tmp);
12849   %}
12850 %}
12851 #endif /* !AARCH64 */
12852 
12853 // Replicate scalar to packed float values in Double register
12854 instruct Repl2F_reg_simd(vecD dst, regF src) %{
12855   predicate(n->as_Vector()->length_in_bytes() == 8 && VM_Version::has_simd());
12856   match(Set dst (ReplicateF src));
12857   size(4);
12858   ins_cost(DEFAULT_COST); // FIXME
12859 
12860   format %{ "VDUP.32  $dst.D,$src.D\t" %}
12861   ins_encode %{
12862     bool quad = false;
12863     __ vdupF($dst$$FloatRegister, $src$$FloatRegister, quad);
12864   %}
12865   ins_pipe(ialu_reg); // FIXME
12866 %}
12867 
12868 #ifndef AARCH64
12869 // Replicate scalar to packed float values in Double register pair
12870 instruct Repl4F_reg(vecX dst, regF src, iRegI tmp) %{
12871   predicate(n->as_Vector()->length() == 4);
12872   match(Set dst (ReplicateF src));
12873   effect(TEMP tmp);
12874   size(4*3);
12875   ins_cost(DEFAULT_COST*3); // FIXME
12876 
12877   format %{ "FMRS     $tmp,$src\n\t"
12878             "FMDRR    $dst.D,$tmp,$tmp\n\t"
12879             "FMDRR    $dst.D.next,$tmp,$tmp\t" %}
12880   ins_encode %{
12881     __ fmrs($tmp$$Register, $src$$FloatRegister);
12882     __ fmdrr($dst$$FloatRegister, $tmp$$Register, $tmp$$Register);
12883     __ fmdrr($dst$$FloatRegister->successor()->successor(),
12884              $tmp$$Register, $tmp$$Register);
12885   %}
12886   ins_pipe(ialu_reg); // FIXME
12887 %}
12888 #endif /* !AARCH64 */
12889 
12890 // Replicate scalar to packed float values in Double register pair
12891 instruct Repl4F_reg_simd(vecX dst, regF src) %{
12892   predicate(n->as_Vector()->length_in_bytes() == 16 && VM_Version::has_simd());
12893   match(Set dst (ReplicateF src));
12894   size(4);
12895   ins_cost(DEFAULT_COST); // FIXME
12896 
12897   format %{ "VDUP.32  $dst.Q,$src.D\t" %}
12898   ins_encode %{
12899     bool quad = true;
12900     __ vdupF($dst$$FloatRegister, $src$$FloatRegister, quad);
12901   %}
12902   ins_pipe(ialu_reg); // FIXME
12903 %}
12904 
12905 #ifndef AARCH64
12906 // Replicate scalar zero constant to packed float values in Double register
12907 instruct Repl2F_immI(vecD dst, immF src, iRegI tmp) %{
12908   predicate(n->as_Vector()->length() == 2);
12909   match(Set dst (ReplicateF src));
12910   effect(TEMP tmp);
12911   size(12);
12912   ins_cost(DEFAULT_COST*4); // FIXME
12913 
12914   format %{ "MOV      $tmp, Repl1($src))\n\t"
12915             "FMDRR    $dst,$tmp,$tmp\t" %}
12916   ins_encode( LdReplImmF(src, dst, tmp) );
12917   ins_pipe(loadConFD); // FIXME
12918 %}
12919 #endif /* !AAARCH64 */
12920 
12921 // Replicate scalar to packed double float values in Double register pair
12922 instruct Repl2D_reg(vecX dst, regD src) %{
12923 #ifdef AARCH64
12924   predicate(n->as_Vector()->length() == 2 && VM_Version::has_simd());
12925   match(Set dst (ReplicateD src));
12926   size(4*1);
12927   ins_cost(DEFAULT_COST*1); // FIXME
12928 
12929   format %{ "VDUP     $dst.2D,$src\t" %}
12930   ins_encode %{
12931     bool quad = true;
12932     __ vdupD($dst$$FloatRegister, $src$$FloatRegister, quad);
12933   %}
12934 #else
12935   predicate(n->as_Vector()->length() == 2);
12936   match(Set dst (ReplicateD src));
12937   size(4*2);
12938   ins_cost(DEFAULT_COST*2); // FIXME
12939 
12940   format %{ "FCPYD    $dst.D.a,$src\n\t"
12941             "FCPYD    $dst.D.b,$src\t" %}
12942   ins_encode %{
12943     FloatRegister dsta = $dst$$FloatRegister;
12944     FloatRegister src = $src$$FloatRegister;
12945     __ fcpyd(dsta, src);
12946     FloatRegister dstb = dsta->successor()->successor();
12947     __ fcpyd(dstb, src);
12948   %}
12949 #endif
12950   ins_pipe(ialu_reg); // FIXME
12951 %}
12952 
12953 // ====================VECTOR ARITHMETIC=======================================
12954 
12955 // --------------------------------- ADD --------------------------------------
12956 
12957 // Bytes vector add
12958 instruct vadd8B_reg(vecD dst, vecD src1, vecD src2) %{
12959   predicate(n->as_Vector()->length() == 8);
12960   match(Set dst (AddVB src1 src2));
12961   format %{ "VADD.I8 $dst,$src1,$src2\t! add packed8B" %}
12962   size(4);
12963   ins_encode %{
12964     bool quad = false;
12965     __ vaddI($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister,
12966              MacroAssembler::VELEM_SIZE_8, quad);
12967   %}
12968   ins_pipe( ialu_reg_reg ); // FIXME
12969 %}
12970 
12971 instruct vadd16B_reg(vecX dst, vecX src1, vecX src2) %{
12972   predicate(n->as_Vector()->length() == 16);
12973   match(Set dst (AddVB src1 src2));
12974   size(4);
12975   format %{ "VADD.I8 $dst.Q,$src1.Q,$src2.Q\t! add packed16B" %}
12976   ins_encode %{
12977     bool quad = true;
12978     __ vaddI($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister,
12979              MacroAssembler::VELEM_SIZE_8, quad);
12980   %}
12981   ins_pipe( ialu_reg_reg ); // FIXME
12982 %}
12983 
12984 // Shorts/Chars vector add
12985 instruct vadd4S_reg(vecD dst, vecD src1, vecD src2) %{
12986   predicate(n->as_Vector()->length() == 4);
12987   match(Set dst (AddVS src1 src2));
12988   size(4);
12989   format %{ "VADD.I16 $dst,$src1,$src2\t! add packed4S" %}
12990   ins_encode %{
12991     bool quad = false;
12992     __ vaddI($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister,
12993              MacroAssembler::VELEM_SIZE_16, quad);
12994   %}
12995   ins_pipe( ialu_reg_reg ); // FIXME
12996 %}
12997 
12998 instruct vadd8S_reg(vecX dst, vecX src1, vecX src2) %{
12999   predicate(n->as_Vector()->length() == 8);
13000   match(Set dst (AddVS src1 src2));
13001   size(4);
13002   format %{ "VADD.I16 $dst.Q,$src1.Q,$src2.Q\t! add packed8S" %}
13003   ins_encode %{
13004     bool quad = true;
13005     __ vaddI($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister,
13006              MacroAssembler::VELEM_SIZE_16, quad);
13007   %}
13008   ins_pipe( ialu_reg_reg ); // FIXME
13009 %}
13010 
13011 // Integers vector add
13012 instruct vadd2I_reg(vecD dst, vecD src1, vecD src2) %{
13013   predicate(n->as_Vector()->length() == 2);
13014   match(Set dst (AddVI src1 src2));
13015   size(4);
13016   format %{ "VADD.I32 $dst.D,$src1.D,$src2.D\t! add packed2I" %}
13017   ins_encode %{
13018     bool quad = false;
13019     __ vaddI($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister,
13020              MacroAssembler::VELEM_SIZE_32, quad);
13021   %}
13022   ins_pipe( ialu_reg_reg ); // FIXME
13023 %}
13024 
13025 instruct vadd4I_reg(vecX dst, vecX src1, vecX src2) %{
13026   predicate(n->as_Vector()->length() == 4);
13027   match(Set dst (AddVI src1 src2));
13028   size(4);
13029   format %{ "VADD.I32 $dst.Q,$src1.Q,$src2.Q\t! add packed4I" %}
13030   ins_encode %{
13031     bool quad = true;
13032     __ vaddI($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister,
13033              MacroAssembler::VELEM_SIZE_32, quad);
13034   %}
13035   ins_pipe( ialu_reg_reg ); // FIXME
13036 %}
13037 
13038 // Longs vector add
13039 instruct vadd2L_reg(vecX dst, vecX src1, vecX src2) %{
13040   predicate(n->as_Vector()->length() == 2);
13041   match(Set dst (AddVL src1 src2));
13042   size(4);
13043   format %{ "VADD.I64 $dst.Q,$src1.Q,$src2.Q\t! add packed2L" %}
13044   ins_encode %{
13045     bool quad = true;
13046     __ vaddI($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister,
13047              MacroAssembler::VELEM_SIZE_64, quad);
13048   %}
13049   ins_pipe( ialu_reg_reg ); // FIXME
13050 %}
13051 
13052 // Floats vector add
13053 instruct vadd2F_reg(vecD dst, vecD src1, vecD src2) %{
13054   predicate(n->as_Vector()->length() == 2 && VM_Version::simd_math_is_compliant());
13055   match(Set dst (AddVF src1 src2));
13056   size(4);
13057   format %{ "VADD.F32 $dst,$src1,$src2\t! add packed2F" %}
13058   ins_encode %{
13059     bool quad = false;
13060     __ vaddF($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister,
13061              MacroAssembler::VFA_SIZE_F32, quad);
13062   %}
13063   ins_pipe( faddD_reg_reg ); // FIXME
13064 %}
13065 
13066 #ifndef AARCH64
13067 instruct vadd2F_reg_vfp(vecD dst, vecD src1, vecD src2) %{
13068   predicate(n->as_Vector()->length() == 2 && !VM_Version::simd_math_is_compliant());
13069   match(Set dst (AddVF src1 src2));
13070   ins_cost(DEFAULT_COST*2); // FIXME
13071 
13072   size(4*2);
13073   format %{ "FADDS  $dst.a,$src1.a,$src2.a\n\t"
13074             "FADDS  $dst.b,$src1.b,$src2.b" %}
13075   ins_encode %{
13076     __ add_float($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister);
13077     __ add_float($dst$$FloatRegister->successor(),
13078              $src1$$FloatRegister->successor(),
13079              $src2$$FloatRegister->successor());
13080   %}
13081 
13082   ins_pipe(faddF_reg_reg); // FIXME
13083 %}
13084 #endif
13085 
13086 instruct vadd4F_reg_simd(vecX dst, vecX src1, vecX src2) %{
13087   predicate(n->as_Vector()->length() == 4 && VM_Version::simd_math_is_compliant());
13088   match(Set dst (AddVF src1 src2));
13089   size(4);
13090   format %{ "VADD.F32 $dst.Q,$src1.Q,$src2.Q\t! add packed4F" %}
13091   ins_encode %{
13092     bool quad = true;
13093     __ vaddF($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister,
13094              MacroAssembler::VFA_SIZE_F32, quad);
13095   %}
13096   ins_pipe( faddD_reg_reg ); // FIXME
13097 %}
13098 
13099 #ifdef AARCH64
13100 instruct vadd2D_reg_simd(vecX dst, vecX src1, vecX src2) %{
13101   predicate(n->as_Vector()->length() == 2 && VM_Version::simd_math_is_compliant());
13102   match(Set dst (AddVD src1 src2));
13103   size(4);
13104   format %{ "VADD.F64 $dst.Q,$src1.Q,$src2.Q\t! add packed2D" %}
13105   ins_encode %{
13106     bool quad = true;
13107     __ vaddF($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister,
13108              MacroAssembler::VFA_SIZE_F64, quad);
13109   %}
13110   ins_pipe( faddD_reg_reg ); // FIXME
13111 %}
13112 #else
13113 instruct vadd4F_reg_vfp(vecX dst, vecX src1, vecX src2) %{
13114   predicate(n->as_Vector()->length() == 4 && !VM_Version::simd_math_is_compliant());
13115   match(Set dst (AddVF src1 src2));
13116   size(4*4);
13117   ins_cost(DEFAULT_COST*4); // FIXME
13118 
13119   format %{ "FADDS  $dst.a,$src1.a,$src2.a\n\t"
13120             "FADDS  $dst.b,$src1.b,$src2.b\n\t"
13121             "FADDS  $dst.c,$src1.c,$src2.c\n\t"
13122             "FADDS  $dst.d,$src1.d,$src2.d" %}
13123 
13124   ins_encode %{
13125     FloatRegister dsta = $dst$$FloatRegister;
13126     FloatRegister src1a = $src1$$FloatRegister;
13127     FloatRegister src2a = $src2$$FloatRegister;
13128     __ add_float(dsta, src1a, src2a);
13129     FloatRegister dstb = dsta->successor();
13130     FloatRegister src1b = src1a->successor();
13131     FloatRegister src2b = src2a->successor();
13132     __ add_float(dstb, src1b, src2b);
13133     FloatRegister dstc = dstb->successor();
13134     FloatRegister src1c = src1b->successor();
13135     FloatRegister src2c = src2b->successor();
13136     __ add_float(dstc, src1c, src2c);
13137     FloatRegister dstd = dstc->successor();
13138     FloatRegister src1d = src1c->successor();
13139     FloatRegister src2d = src2c->successor();
13140     __ add_float(dstd, src1d, src2d);
13141   %}
13142 
13143   ins_pipe(faddF_reg_reg); // FIXME
13144 %}
13145 
13146 instruct vadd2D_reg_vfp(vecX dst, vecX src1, vecX src2) %{
13147   predicate(n->as_Vector()->length() == 2);
13148   match(Set dst (AddVD src1 src2));
13149   size(4*2);
13150   ins_cost(DEFAULT_COST*2); // FIXME
13151 
13152   format %{ "FADDD  $dst.a,$src1.a,$src2.a\n\t"
13153             "FADDD  $dst.b,$src1.b,$src2.b" %}
13154 
13155   ins_encode %{
13156     FloatRegister dsta = $dst$$FloatRegister;
13157     FloatRegister src1a = $src1$$FloatRegister;
13158     FloatRegister src2a = $src2$$FloatRegister;
13159     __ add_double(dsta, src1a, src2a);
13160     FloatRegister dstb = dsta->successor()->successor();
13161     FloatRegister src1b = src1a->successor()->successor();
13162     FloatRegister src2b = src2a->successor()->successor();
13163     __ add_double(dstb, src1b, src2b);
13164   %}
13165 
13166   ins_pipe(faddF_reg_reg); // FIXME
13167 %}
13168 #endif
13169 
13170 
13171 // Bytes vector sub
13172 instruct vsub8B_reg(vecD dst, vecD src1, vecD src2) %{
13173   predicate(n->as_Vector()->length() == 8);
13174   match(Set dst (SubVB src1 src2));
13175   size(4);
13176   format %{ "VSUB.I8 $dst,$src1,$src2\t! sub packed8B" %}
13177   ins_encode %{
13178     bool quad = false;
13179     __ vsubI($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister,
13180              MacroAssembler::VELEM_SIZE_8, quad);
13181   %}
13182   ins_pipe( ialu_reg_reg ); // FIXME
13183 %}
13184 
13185 instruct vsub16B_reg(vecX dst, vecX src1, vecX src2) %{
13186   predicate(n->as_Vector()->length() == 16);
13187   match(Set dst (SubVB src1 src2));
13188   size(4);
13189   format %{ "VSUB.I8 $dst.Q,$src1.Q,$src2.Q\t! sub packed16B" %}
13190   ins_encode %{
13191     bool quad = true;
13192     __ vsubI($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister,
13193              MacroAssembler::VELEM_SIZE_8, quad);
13194   %}
13195   ins_pipe( ialu_reg_reg ); // FIXME
13196 %}
13197 
13198 // Shorts/Chars vector sub
13199 instruct vsub4S_reg(vecD dst, vecD src1, vecD src2) %{
13200   predicate(n->as_Vector()->length() == 4);
13201   match(Set dst (SubVS src1 src2));
13202   size(4);
13203   format %{ "VSUB.I16 $dst,$src1,$src2\t! sub packed4S" %}
13204   ins_encode %{
13205     bool quad = false;
13206     __ vsubI($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister,
13207              MacroAssembler::VELEM_SIZE_16, quad);
13208   %}
13209   ins_pipe( ialu_reg_reg ); // FIXME
13210 %}
13211 
13212 instruct vsub16S_reg(vecX dst, vecX src1, vecX src2) %{
13213   predicate(n->as_Vector()->length() == 8);
13214   match(Set dst (SubVS src1 src2));
13215   size(4);
13216   format %{ "VSUB.I16 $dst.Q,$src1.Q,$src2.Q\t! sub packed8S" %}
13217   ins_encode %{
13218     bool quad = true;
13219     __ vsubI($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister,
13220              MacroAssembler::VELEM_SIZE_16, quad);
13221   %}
13222   ins_pipe( ialu_reg_reg ); // FIXME
13223 %}
13224 
13225 // Integers vector sub
13226 instruct vsub2I_reg(vecD dst, vecD src1, vecD src2) %{
13227   predicate(n->as_Vector()->length() == 2);
13228   match(Set dst (SubVI src1 src2));
13229   size(4);
13230   format %{ "VSUB.I32 $dst,$src1,$src2\t! sub packed2I" %}
13231   ins_encode %{
13232     bool quad = false;
13233     __ vsubI($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister,
13234              MacroAssembler::VELEM_SIZE_32, quad);
13235   %}
13236   ins_pipe( ialu_reg_reg ); // FIXME
13237 %}
13238 
13239 instruct vsub4I_reg(vecX dst, vecX src1, vecX src2) %{
13240   predicate(n->as_Vector()->length() == 4);
13241   match(Set dst (SubVI src1 src2));
13242   size(4);
13243   format %{ "VSUB.I32 $dst.Q,$src1.Q,$src2.Q\t! sub packed4I" %}
13244   ins_encode %{
13245     bool quad = true;
13246     __ vsubI($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister,
13247              MacroAssembler::VELEM_SIZE_32, quad);
13248   %}
13249   ins_pipe( ialu_reg_reg ); // FIXME
13250 %}
13251 
13252 // Longs vector sub
13253 instruct vsub2L_reg(vecX dst, vecX src1, vecX src2) %{
13254   predicate(n->as_Vector()->length() == 2);
13255   match(Set dst (SubVL src1 src2));
13256   size(4);
13257   format %{ "VSUB.I64 $dst.Q,$src1.Q,$src2.Q\t! sub packed2L" %}
13258   ins_encode %{
13259     bool quad = true;
13260     __ vsubI($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister,
13261              MacroAssembler::VELEM_SIZE_64, quad);
13262   %}
13263   ins_pipe( ialu_reg_reg ); // FIXME
13264 %}
13265 
13266 // Floats vector sub
13267 instruct vsub2F_reg(vecD dst, vecD src1, vecD src2) %{
13268   predicate(n->as_Vector()->length() == 2 && VM_Version::simd_math_is_compliant());
13269   match(Set dst (SubVF src1 src2));
13270   size(4);
13271   format %{ "VSUB.F32 $dst,$src1,$src2\t! sub packed2F" %}
13272   ins_encode %{
13273     bool quad = false;
13274     __ vsubF($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister,
13275              MacroAssembler::VFA_SIZE_F32, quad);
13276   %}
13277   ins_pipe( faddF_reg_reg ); // FIXME
13278 %}
13279 
13280 #ifndef AARCH64
13281 instruct vsub2F_reg_vfp(vecD dst, vecD src1, vecD src2) %{
13282   predicate(n->as_Vector()->length() == 2 && !VM_Version::simd_math_is_compliant());
13283   match(Set dst (SubVF src1 src2));
13284   size(4*2);
13285   ins_cost(DEFAULT_COST*2); // FIXME
13286 
13287   format %{ "FSUBS  $dst.a,$src1.a,$src2.a\n\t"
13288             "FSUBS  $dst.b,$src1.b,$src2.b" %}
13289 
13290   ins_encode %{
13291     FloatRegister dsta = $dst$$FloatRegister;
13292     FloatRegister src1a = $src1$$FloatRegister;
13293     FloatRegister src2a = $src2$$FloatRegister;
13294     __ sub_float(dsta, src1a, src2a);
13295     FloatRegister dstb = dsta->successor();
13296     FloatRegister src1b = src1a->successor();
13297     FloatRegister src2b = src2a->successor();
13298     __ sub_float(dstb, src1b, src2b);
13299   %}
13300 
13301   ins_pipe(faddF_reg_reg); // FIXME
13302 %}
13303 #endif
13304 
13305 
13306 instruct vsub4F_reg(vecX dst, vecX src1, vecX src2) %{
13307   predicate(n->as_Vector()->length() == 4 && VM_Version::simd_math_is_compliant());
13308   match(Set dst (SubVF src1 src2));
13309   size(4);
13310   format %{ "VSUB.F32 $dst.Q,$src1.Q,$src2.Q\t! sub packed4F" %}
13311   ins_encode %{
13312     bool quad = true;
13313     __ vsubF($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister,
13314              MacroAssembler::VFA_SIZE_F32, quad);
13315   %}
13316   ins_pipe( faddF_reg_reg ); // FIXME
13317 %}
13318 
13319 #ifdef AARCH64
13320 instruct vsub2D_reg_simd(vecX dst, vecX src1, vecX src2) %{
13321   predicate(n->as_Vector()->length() == 2 && VM_Version::simd_math_is_compliant());
13322   match(Set dst (SubVD src1 src2));
13323   size(4);
13324   format %{ "VSUB.F64 $dst.Q,$src1.Q,$src2.Q\t! add packed2D" %}
13325   ins_encode %{
13326     bool quad = true;
13327     __ vsubF($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister,
13328              MacroAssembler::VFA_SIZE_F64, quad);
13329   %}
13330   ins_pipe( faddD_reg_reg ); // FIXME
13331 %}
13332 #else
13333 instruct vsub4F_reg_vfp(vecX dst, vecX src1, vecX src2) %{
13334   predicate(n->as_Vector()->length() == 4 && !VM_Version::simd_math_is_compliant());
13335   match(Set dst (SubVF src1 src2));
13336   size(4*4);
13337   ins_cost(DEFAULT_COST*4); // FIXME
13338 
13339   format %{ "FSUBS  $dst.a,$src1.a,$src2.a\n\t"
13340             "FSUBS  $dst.b,$src1.b,$src2.b\n\t"
13341             "FSUBS  $dst.c,$src1.c,$src2.c\n\t"
13342             "FSUBS  $dst.d,$src1.d,$src2.d" %}
13343 
13344   ins_encode %{
13345     FloatRegister dsta = $dst$$FloatRegister;
13346     FloatRegister src1a = $src1$$FloatRegister;
13347     FloatRegister src2a = $src2$$FloatRegister;
13348     __ sub_float(dsta, src1a, src2a);
13349     FloatRegister dstb = dsta->successor();
13350     FloatRegister src1b = src1a->successor();
13351     FloatRegister src2b = src2a->successor();
13352     __ sub_float(dstb, src1b, src2b);
13353     FloatRegister dstc = dstb->successor();
13354     FloatRegister src1c = src1b->successor();
13355     FloatRegister src2c = src2b->successor();
13356     __ sub_float(dstc, src1c, src2c);
13357     FloatRegister dstd = dstc->successor();
13358     FloatRegister src1d = src1c->successor();
13359     FloatRegister src2d = src2c->successor();
13360     __ sub_float(dstd, src1d, src2d);
13361   %}
13362 
13363   ins_pipe(faddF_reg_reg); // FIXME
13364 %}
13365 
13366 instruct vsub2D_reg_vfp(vecX dst, vecX src1, vecX src2) %{
13367   predicate(n->as_Vector()->length() == 2);
13368   match(Set dst (SubVD src1 src2));
13369   size(4*2);
13370   ins_cost(DEFAULT_COST*2); // FIXME
13371 
13372   format %{ "FSUBD  $dst.a,$src1.a,$src2.a\n\t"
13373             "FSUBD  $dst.b,$src1.b,$src2.b" %}
13374 
13375   ins_encode %{
13376     FloatRegister dsta = $dst$$FloatRegister;
13377     FloatRegister src1a = $src1$$FloatRegister;
13378     FloatRegister src2a = $src2$$FloatRegister;
13379     __ sub_double(dsta, src1a, src2a);
13380     FloatRegister dstb = dsta->successor()->successor();
13381     FloatRegister src1b = src1a->successor()->successor();
13382     FloatRegister src2b = src2a->successor()->successor();
13383     __ sub_double(dstb, src1b, src2b);
13384   %}
13385 
13386   ins_pipe(faddF_reg_reg); // FIXME
13387 %}
13388 #endif
13389 
13390 // Shorts/Chars vector mul
13391 instruct vmul4S_reg(vecD dst, vecD src1, vecD src2) %{
13392   predicate(n->as_Vector()->length() == 4);
13393   match(Set dst (MulVS src1 src2));
13394   size(4);
13395   format %{ "VMUL.I16 $dst,$src1,$src2\t! mul packed4S" %}
13396   ins_encode %{
13397     __ vmulI($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister,
13398              MacroAssembler::VELEM_SIZE_16, 0);
13399   %}
13400   ins_pipe( ialu_reg_reg ); // FIXME
13401 %}
13402 
13403 instruct vmul8S_reg(vecX dst, vecX src1, vecX src2) %{
13404   predicate(n->as_Vector()->length() == 8);
13405   match(Set dst (MulVS src1 src2));
13406   size(4);
13407   format %{ "VMUL.I16 $dst.Q,$src1.Q,$src2.Q\t! mul packed8S" %}
13408   ins_encode %{
13409     __ vmulI($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister,
13410              MacroAssembler::VELEM_SIZE_16, 1);
13411   %}
13412   ins_pipe( ialu_reg_reg ); // FIXME
13413 %}
13414 
13415 // Integers vector mul
13416 instruct vmul2I_reg(vecD dst, vecD src1, vecD src2) %{
13417   predicate(n->as_Vector()->length() == 2);
13418   match(Set dst (MulVI src1 src2));
13419   size(4);
13420   format %{ "VMUL.I32 $dst,$src1,$src2\t! mul packed2I" %}
13421   ins_encode %{
13422     __ vmulI($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister,
13423              MacroAssembler::VELEM_SIZE_32, 0);
13424   %}
13425   ins_pipe( ialu_reg_reg ); // FIXME
13426 %}
13427 
13428 instruct vmul4I_reg(vecX dst, vecX src1, vecX src2) %{
13429   predicate(n->as_Vector()->length() == 4);
13430   match(Set dst (MulVI src1 src2));
13431   size(4);
13432   format %{ "VMUL.I32 $dst.Q,$src1.Q,$src2.Q\t! mul packed4I" %}
13433   ins_encode %{
13434     __ vmulI($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister,
13435              MacroAssembler::VELEM_SIZE_32, 1);
13436   %}
13437   ins_pipe( ialu_reg_reg ); // FIXME
13438 %}
13439 
13440 // Floats vector mul
13441 instruct vmul2F_reg(vecD dst, vecD src1, vecD src2) %{
13442   predicate(n->as_Vector()->length() == 2 && VM_Version::simd_math_is_compliant());
13443   match(Set dst (MulVF src1 src2));
13444   size(4);
13445   format %{ "VMUL.F32 $dst,$src1,$src2\t! mul packed2F" %}
13446   ins_encode %{
13447     __ vmulF($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister,
13448              MacroAssembler::VFA_SIZE_F32, 0);
13449   %}
13450   ins_pipe( fmulF_reg_reg ); // FIXME
13451 %}
13452 
13453 #ifndef AARCH64
13454 instruct vmul2F_reg_vfp(vecD dst, vecD src1, vecD src2) %{
13455   predicate(n->as_Vector()->length() == 2 && !VM_Version::simd_math_is_compliant());
13456   match(Set dst (MulVF src1 src2));
13457   size(4*2);
13458   ins_cost(DEFAULT_COST*2); // FIXME
13459 
13460   format %{ "FMULS  $dst.a,$src1.a,$src2.a\n\t"
13461             "FMULS  $dst.b,$src1.b,$src2.b" %}
13462   ins_encode %{
13463     __ mul_float($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister);
13464     __ mul_float($dst$$FloatRegister->successor(),
13465              $src1$$FloatRegister->successor(),
13466              $src2$$FloatRegister->successor());
13467   %}
13468 
13469   ins_pipe(fmulF_reg_reg); // FIXME
13470 %}
13471 #endif
13472 
13473 instruct vmul4F_reg(vecX dst, vecX src1, vecX src2) %{
13474   predicate(n->as_Vector()->length() == 4 && VM_Version::simd_math_is_compliant());
13475   match(Set dst (MulVF src1 src2));
13476   size(4);
13477   format %{ "VMUL.F32 $dst.Q,$src1.Q,$src2.Q\t! mul packed4F" %}
13478   ins_encode %{
13479     __ vmulF($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister,
13480              MacroAssembler::VFA_SIZE_F32, 1);
13481   %}
13482   ins_pipe( fmulF_reg_reg ); // FIXME
13483 %}
13484 
13485 #ifndef AARCH64
13486 instruct vmul4F_reg_vfp(vecX dst, vecX src1, vecX src2) %{
13487   predicate(n->as_Vector()->length() == 4 && !VM_Version::simd_math_is_compliant());
13488   match(Set dst (MulVF src1 src2));
13489   size(4*4);
13490   ins_cost(DEFAULT_COST*4); // FIXME
13491 
13492   format %{ "FMULS  $dst.a,$src1.a,$src2.a\n\t"
13493             "FMULS  $dst.b,$src1.b,$src2.b\n\t"
13494             "FMULS  $dst.c,$src1.c,$src2.c\n\t"
13495             "FMULS  $dst.d,$src1.d,$src2.d" %}
13496 
13497   ins_encode %{
13498     FloatRegister dsta = $dst$$FloatRegister;
13499     FloatRegister src1a = $src1$$FloatRegister;
13500     FloatRegister src2a = $src2$$FloatRegister;
13501     __ mul_float(dsta, src1a, src2a);
13502     FloatRegister dstb = dsta->successor();
13503     FloatRegister src1b = src1a->successor();
13504     FloatRegister src2b = src2a->successor();
13505     __ mul_float(dstb, src1b, src2b);
13506     FloatRegister dstc = dstb->successor();
13507     FloatRegister src1c = src1b->successor();
13508     FloatRegister src2c = src2b->successor();
13509     __ mul_float(dstc, src1c, src2c);
13510     FloatRegister dstd = dstc->successor();
13511     FloatRegister src1d = src1c->successor();
13512     FloatRegister src2d = src2c->successor();
13513     __ mul_float(dstd, src1d, src2d);
13514   %}
13515 
13516   ins_pipe(fmulF_reg_reg); // FIXME
13517 %}
13518 #endif
13519 
13520 #ifdef AARCH64
13521 instruct vmul2D_reg(vecX dst, vecX src1, vecX src2) %{
13522   predicate(n->as_Vector()->length() == 2 && VM_Version::has_simd());
13523   match(Set dst (MulVD src1 src2));
13524   size(4*1);
13525   ins_cost(DEFAULT_COST*1); // FIXME
13526 
13527   format %{ "FMUL.2D $dst,$src1,$src2\t! double[2]" %}
13528   ins_encode %{
13529     int quad = 1;
13530     __ vmulF($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister,
13531              MacroAssembler::VFA_SIZE_F64, quad);
13532   %}
13533 
13534   ins_pipe(fdivF_reg_reg); // FIXME
13535 %}
13536 #else
13537 instruct vmul2D_reg_vfp(vecX dst, vecX src1, vecX src2) %{
13538   predicate(n->as_Vector()->length() == 2);
13539   match(Set dst (MulVD src1 src2));
13540   size(4*2);
13541   ins_cost(DEFAULT_COST*2); // FIXME
13542 
13543   format %{ "FMULD  $dst.D.a,$src1.D.a,$src2.D.a\n\t"
13544             "FMULD  $dst.D.b,$src1.D.b,$src2.D.b" %}
13545   ins_encode %{
13546     FloatRegister dsta = $dst$$FloatRegister;
13547     FloatRegister src1a = $src1$$FloatRegister;
13548     FloatRegister src2a = $src2$$FloatRegister;
13549     __ mul_double(dsta, src1a, src2a);
13550     FloatRegister dstb = dsta->successor()->successor();
13551     FloatRegister src1b = src1a->successor()->successor();
13552     FloatRegister src2b = src2a->successor()->successor();
13553     __ mul_double(dstb, src1b, src2b);
13554   %}
13555 
13556   ins_pipe(fmulD_reg_reg); // FIXME
13557 %}
13558 #endif
13559 
13560 
13561 // Floats vector div
13562 instruct vdiv2F_reg_vfp(vecD dst, vecD src1, vecD src2) %{
13563   predicate(n->as_Vector()->length() == 2);
13564   match(Set dst (DivVF src1 src2));
13565 #ifdef AARCH64
13566   size(4*1);
13567   ins_cost(DEFAULT_COST*1); // FIXME
13568 
13569   format %{ "FDIV.2S $dst,$src1,$src2\t! float[2]" %}
13570   ins_encode %{
13571     int quad = 0;
13572     __ vdivF($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister,
13573              MacroAssembler::VFA_SIZE_F32, quad);
13574   %}
13575 
13576   ins_pipe(fdivF_reg_reg); // FIXME
13577 #else
13578   size(4*2);
13579   ins_cost(DEFAULT_COST*2); // FIXME
13580 
13581   format %{ "FDIVS  $dst.a,$src1.a,$src2.a\n\t"
13582             "FDIVS  $dst.b,$src1.b,$src2.b" %}
13583   ins_encode %{
13584     __ div_float($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister);
13585     __ div_float($dst$$FloatRegister->successor(),
13586              $src1$$FloatRegister->successor(),
13587              $src2$$FloatRegister->successor());
13588   %}
13589 
13590   ins_pipe(fdivF_reg_reg); // FIXME
13591 #endif
13592 %}
13593 
13594 instruct vdiv4F_reg_vfp(vecX dst, vecX src1, vecX src2) %{
13595   predicate(n->as_Vector()->length() == 4);
13596   match(Set dst (DivVF src1 src2));
13597 #ifdef AARCH64
13598   size(4*1);
13599   ins_cost(DEFAULT_COST*1); // FIXME
13600 
13601   format %{ "FDIV.4S $dst,$src1,$src2\t! float[4]" %}
13602   ins_encode %{
13603     int quad = 1;
13604     __ vdivF($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister,
13605              MacroAssembler::VFA_SIZE_F32, quad);
13606   %}
13607 
13608   ins_pipe(fdivF_reg_reg); // FIXME
13609 #else
13610   size(4*4);
13611   ins_cost(DEFAULT_COST*4); // FIXME
13612 
13613   format %{ "FDIVS  $dst.a,$src1.a,$src2.a\n\t"
13614             "FDIVS  $dst.b,$src1.b,$src2.b\n\t"
13615             "FDIVS  $dst.c,$src1.c,$src2.c\n\t"
13616             "FDIVS  $dst.d,$src1.d,$src2.d" %}
13617 
13618   ins_encode %{
13619     FloatRegister dsta = $dst$$FloatRegister;
13620     FloatRegister src1a = $src1$$FloatRegister;
13621     FloatRegister src2a = $src2$$FloatRegister;
13622     __ div_float(dsta, src1a, src2a);
13623     FloatRegister dstb = dsta->successor();
13624     FloatRegister src1b = src1a->successor();
13625     FloatRegister src2b = src2a->successor();
13626     __ div_float(dstb, src1b, src2b);
13627     FloatRegister dstc = dstb->successor();
13628     FloatRegister src1c = src1b->successor();
13629     FloatRegister src2c = src2b->successor();
13630     __ div_float(dstc, src1c, src2c);
13631     FloatRegister dstd = dstc->successor();
13632     FloatRegister src1d = src1c->successor();
13633     FloatRegister src2d = src2c->successor();
13634     __ div_float(dstd, src1d, src2d);
13635   %}
13636 
13637   ins_pipe(fdivF_reg_reg); // FIXME
13638 #endif
13639 %}
13640 
13641 #ifdef AARCH64
13642 instruct vdiv2D_reg(vecX dst, vecX src1, vecX src2) %{
13643   predicate(n->as_Vector()->length() == 2 && VM_Version::has_simd());
13644   match(Set dst (DivVD src1 src2));
13645   size(4*1);
13646   ins_cost(DEFAULT_COST*1); // FIXME
13647 
13648   format %{ "FDIV.2D $dst,$src1,$src2\t! double[2]" %}
13649   ins_encode %{
13650     int quad = 1;
13651     __ vdivF($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister,
13652              MacroAssembler::VFA_SIZE_F64, quad);
13653   %}
13654 
13655   ins_pipe(fdivF_reg_reg); // FIXME
13656 %}
13657 #else
13658 instruct vdiv2D_reg_vfp(vecX dst, vecX src1, vecX src2) %{
13659   predicate(n->as_Vector()->length() == 2);
13660   match(Set dst (DivVD src1 src2));
13661   size(4*2);
13662   ins_cost(DEFAULT_COST*2); // FIXME
13663 
13664   format %{ "FDIVD  $dst.D.a,$src1.D.a,$src2.D.a\n\t"
13665             "FDIVD  $dst.D.b,$src1.D.b,$src2.D.b" %}
13666   ins_encode %{
13667     FloatRegister dsta = $dst$$FloatRegister;
13668     FloatRegister src1a = $src1$$FloatRegister;
13669     FloatRegister src2a = $src2$$FloatRegister;
13670     __ div_double(dsta, src1a, src2a);
13671     FloatRegister dstb = dsta->successor()->successor();
13672     FloatRegister src1b = src1a->successor()->successor();
13673     FloatRegister src2b = src2a->successor()->successor();
13674     __ div_double(dstb, src1b, src2b);
13675   %}
13676 
13677   ins_pipe(fdivD_reg_reg); // FIXME
13678 %}
13679 #endif
13680 
13681 // --------------------------------- NEG --------------------------------------
13682 
13683 instruct vneg8B_reg(vecD dst, vecD src) %{
13684   predicate(n->as_Vector()->length_in_bytes() == 8);
13685   effect(DEF dst, USE src);
13686   size(4);
13687   ins_cost(DEFAULT_COST); // FIXME
13688   format %{ "VNEG.S8 $dst.D,$src.D\t! neg packed8B" %}
13689   ins_encode %{
13690     bool quad = false;
13691     __ vnegI($dst$$FloatRegister, $src$$FloatRegister,
13692               MacroAssembler::VELEM_SIZE_8, quad);
13693   %}
13694   ins_pipe( ialu_reg_reg ); // FIXME
13695 %}
13696 
13697 instruct vneg16B_reg(vecX dst, vecX src) %{
13698   predicate(n->as_Vector()->length_in_bytes() == 16);
13699   effect(DEF dst, USE src);
13700   size(4);
13701   ins_cost(DEFAULT_COST); // FIXME
13702   format %{ "VNEG.S8 $dst.Q,$src.Q\t! neg0 packed16B" %}
13703   ins_encode %{
13704     bool _float = false;
13705     bool quad = true;
13706     __ vnegI($dst$$FloatRegister, $src$$FloatRegister,
13707               MacroAssembler::VELEM_SIZE_8, quad);
13708   %}
13709   ins_pipe( ialu_reg_reg ); // FIXME
13710 %}
13711 
13712 // ------------------------------ Shift ---------------------------------------
13713 
13714 instruct vslcntD(vecD dst, iRegI cnt) %{
13715   predicate(n->as_Vector()->length_in_bytes() == 8 && VM_Version::has_simd());
13716   match(Set dst (LShiftCntV cnt));
13717   size(4);
13718   ins_cost(DEFAULT_COST); // FIXME
13719   expand %{
13720     Repl8B_reg_simd(dst, cnt);
13721   %}
13722 %}
13723 
13724 instruct vslcntX(vecX dst, iRegI cnt) %{
13725   predicate(n->as_Vector()->length_in_bytes() == 16 && VM_Version::has_simd());
13726   match(Set dst (LShiftCntV cnt));
13727   size(4);
13728   ins_cost(DEFAULT_COST); // FIXME
13729   expand %{
13730     Repl16B_reg(dst, cnt);
13731   %}
13732 %}
13733 
13734 // Low bits of vector "shift" elements are used, so it
13735 // doesn't matter if we treat it as ints or bytes here.
13736 instruct vsrcntD(vecD dst, iRegI cnt) %{
13737   predicate(n->as_Vector()->length_in_bytes() == 8 && VM_Version::has_simd());
13738   match(Set dst (RShiftCntV cnt));
13739   size(4*2);
13740   ins_cost(DEFAULT_COST*2); // FIXME
13741 
13742   format %{ "VDUP.8 $dst.D,$cnt\n\t"
13743             "VNEG.S8 $dst.D,$dst.D\t! neg packed8B" %}
13744   ins_encode %{
13745     bool quad = false;
13746     __ vdupI($dst$$FloatRegister, $cnt$$Register,
13747              MacroAssembler::VELEM_SIZE_8, quad);
13748     __ vnegI($dst$$FloatRegister, $dst$$FloatRegister,
13749               MacroAssembler::VELEM_SIZE_8, quad);
13750   %}
13751   ins_pipe( ialu_reg_reg ); // FIXME
13752 %}
13753 
13754 instruct vsrcntX(vecX dst, iRegI cnt) %{
13755   predicate(n->as_Vector()->length_in_bytes() == 16 && VM_Version::has_simd());
13756   match(Set dst (RShiftCntV cnt));
13757   size(4*2);
13758   ins_cost(DEFAULT_COST*2); // FIXME
13759   format %{ "VDUP.8 $dst.Q,$cnt\n\t"
13760             "VNEG.S8 $dst.Q,$dst.Q\t! neg packed16B" %}
13761   ins_encode %{
13762     bool quad = true;
13763     __ vdupI($dst$$FloatRegister, $cnt$$Register,
13764              MacroAssembler::VELEM_SIZE_8, quad);
13765     __ vnegI($dst$$FloatRegister, $dst$$FloatRegister,
13766               MacroAssembler::VELEM_SIZE_8, quad);
13767   %}
13768   ins_pipe( ialu_reg_reg ); // FIXME
13769 %}
13770 
13771 // Byte vector logical left/right shift based on sign
13772 instruct vsh8B_reg(vecD dst, vecD src, vecD shift) %{
13773   predicate(n->as_Vector()->length() == 8);
13774   effect(DEF dst, USE src, USE shift);
13775   size(4);
13776   ins_cost(DEFAULT_COST); // FIXME
13777   format %{
13778     "VSHL.U8 $dst.D,$src.D,$shift.D\t! logical left/right shift packed8B"
13779   %}
13780   ins_encode %{
13781     bool quad = false;
13782     __ vshlUI($dst$$FloatRegister, $shift$$FloatRegister, $src$$FloatRegister,
13783               MacroAssembler::VELEM_SIZE_8, quad);
13784   %}
13785   ins_pipe( ialu_reg_reg ); // FIXME
13786 %}
13787 
13788 instruct vsh16B_reg(vecX dst, vecX src, vecX shift) %{
13789   predicate(n->as_Vector()->length() == 16);
13790   effect(DEF dst, USE src, USE shift);
13791   size(4);
13792   ins_cost(DEFAULT_COST); // FIXME
13793   format %{
13794     "VSHL.U8 $dst.Q,$src.Q,$shift.Q\t! logical left/right shift packed16B"
13795   %}
13796   ins_encode %{
13797     bool quad = true;
13798     __ vshlUI($dst$$FloatRegister, $shift$$FloatRegister, $src$$FloatRegister,
13799               MacroAssembler::VELEM_SIZE_8, quad);
13800   %}
13801   ins_pipe( ialu_reg_reg ); // FIXME
13802 %}
13803 
13804 // Shorts/Char vector logical left/right shift based on sign
13805 instruct vsh4S_reg(vecD dst, vecD src, vecD shift) %{
13806   predicate(n->as_Vector()->length() == 4);
13807   effect(DEF dst, USE src, USE shift);
13808   size(4);
13809   ins_cost(DEFAULT_COST); // FIXME
13810   format %{
13811     "VSHL.U16 $dst.D,$src.D,$shift.D\t! logical left/right shift packed4S"
13812   %}
13813   ins_encode %{
13814     bool quad = false;
13815     __ vshlUI($dst$$FloatRegister, $shift$$FloatRegister, $src$$FloatRegister,
13816               MacroAssembler::VELEM_SIZE_16, quad);
13817   %}
13818   ins_pipe( ialu_reg_reg ); // FIXME
13819 %}
13820 
13821 instruct vsh8S_reg(vecX dst, vecX src, vecX shift) %{
13822   predicate(n->as_Vector()->length() == 8);
13823   effect(DEF dst, USE src, USE shift);
13824   size(4);
13825   ins_cost(DEFAULT_COST); // FIXME
13826   format %{
13827     "VSHL.U16 $dst.Q,$src.Q,$shift.Q\t! logical left/right shift packed8S"
13828   %}
13829   ins_encode %{
13830     bool quad = true;
13831     __ vshlUI($dst$$FloatRegister, $shift$$FloatRegister, $src$$FloatRegister,
13832               MacroAssembler::VELEM_SIZE_16, quad);
13833   %}
13834   ins_pipe( ialu_reg_reg ); // FIXME
13835 %}
13836 
13837 // Integers vector logical left/right shift based on sign
13838 instruct vsh2I_reg(vecD dst, vecD src, vecD shift) %{
13839   predicate(n->as_Vector()->length() == 2);
13840   effect(DEF dst, USE src, USE shift);
13841   size(4);
13842   ins_cost(DEFAULT_COST); // FIXME
13843   format %{
13844     "VSHL.U32 $dst.D,$src.D,$shift.D\t! logical left/right shift packed2I"
13845   %}
13846   ins_encode %{
13847     bool quad = false;
13848     __ vshlUI($dst$$FloatRegister, $shift$$FloatRegister, $src$$FloatRegister,
13849               MacroAssembler::VELEM_SIZE_32, quad);
13850   %}
13851   ins_pipe( ialu_reg_reg ); // FIXME
13852 %}
13853 
13854 instruct vsh4I_reg(vecX dst, vecX src, vecX shift) %{
13855   predicate(n->as_Vector()->length() == 4);
13856   effect(DEF dst, USE src, USE shift);
13857   size(4);
13858   ins_cost(DEFAULT_COST); // FIXME
13859   format %{
13860     "VSHL.U32 $dst.Q,$src.Q,$shift.Q\t! logical left/right shift packed4I"
13861   %}
13862   ins_encode %{
13863     bool quad = true;
13864     __ vshlUI($dst$$FloatRegister, $shift$$FloatRegister, $src$$FloatRegister,
13865               MacroAssembler::VELEM_SIZE_32, quad);
13866   %}
13867   ins_pipe( ialu_reg_reg ); // FIXME
13868 %}
13869 
13870 // Longs vector logical left/right shift based on sign
13871 instruct vsh2L_reg(vecX dst, vecX src, vecX shift) %{
13872   predicate(n->as_Vector()->length() == 2);
13873   effect(DEF dst, USE src, USE shift);
13874   size(4);
13875   ins_cost(DEFAULT_COST); // FIXME
13876   format %{
13877     "VSHL.U64 $dst.Q,$src.Q,$shift.Q\t! logical left/right shift packed2L"
13878   %}
13879   ins_encode %{
13880     bool quad = true;
13881     __ vshlUI($dst$$FloatRegister, $shift$$FloatRegister, $src$$FloatRegister,
13882               MacroAssembler::VELEM_SIZE_64, quad);
13883   %}
13884   ins_pipe( ialu_reg_reg ); // FIXME
13885 %}
13886 
13887 // ------------------------------ LeftShift -----------------------------------
13888 
13889 // Byte vector left shift
13890 instruct vsl8B_reg(vecD dst, vecD src, vecD shift) %{
13891   predicate(n->as_Vector()->length() == 8);
13892   match(Set dst (LShiftVB src shift));
13893   size(4*1);
13894   ins_cost(DEFAULT_COST*1); // FIXME
13895   expand %{
13896     vsh8B_reg(dst, src, shift);
13897   %}
13898 %}
13899 
13900 instruct vsl16B_reg(vecX dst, vecX src, vecX shift) %{
13901   predicate(n->as_Vector()->length() == 16);
13902   match(Set dst (LShiftVB src shift));
13903   size(4*1);
13904   ins_cost(DEFAULT_COST*1); // FIXME
13905   expand %{
13906     vsh16B_reg(dst, src, shift);
13907   %}
13908 %}
13909 
13910 instruct vsl8B_immI(vecD dst, vecD src, immI shift) %{
13911   predicate(n->as_Vector()->length() == 8);
13912   match(Set dst (LShiftVB src shift));
13913   size(4);
13914   ins_cost(DEFAULT_COST); // FIXME
13915   format %{
13916     "VSHL.I8 $dst.D,$src.D,$shift\t! logical left shift packed8B"
13917   %}
13918   ins_encode %{
13919     bool quad = false;
13920     __ vshli($dst$$FloatRegister, $src$$FloatRegister, 8, $shift$$constant,
13921              quad);
13922   %}
13923   ins_pipe( ialu_reg_reg ); // FIXME
13924 %}
13925 
13926 instruct vsl16B_immI(vecX dst, vecX src, immI shift) %{
13927   predicate(n->as_Vector()->length() == 16);
13928   match(Set dst (LShiftVB src shift));
13929   size(4);
13930   ins_cost(DEFAULT_COST); // FIXME
13931   format %{
13932     "VSHL.I8 $dst.Q,$src.Q,$shift\t! logical left shift packed16B"
13933   %}
13934   ins_encode %{
13935     bool quad = true;
13936     __ vshli($dst$$FloatRegister, $src$$FloatRegister, 8, $shift$$constant,
13937              quad);
13938   %}
13939   ins_pipe( ialu_reg_reg ); // FIXME
13940 %}
13941 
13942 // Shorts/Chars vector logical left/right shift
13943 instruct vsl4S_reg(vecD dst, vecD src, vecD shift) %{
13944   predicate(n->as_Vector()->length() == 4);
13945   match(Set dst (LShiftVS src shift));
13946   match(Set dst (URShiftVS src shift));
13947   size(4*1);
13948   ins_cost(DEFAULT_COST*1); // FIXME
13949   expand %{
13950     vsh4S_reg(dst, src, shift);
13951   %}
13952 %}
13953 
13954 instruct vsl8S_reg(vecX dst, vecX src, vecX shift) %{
13955   predicate(n->as_Vector()->length() == 8);
13956   match(Set dst (LShiftVS src shift));
13957   match(Set dst (URShiftVS src shift));
13958   size(4*1);
13959   ins_cost(DEFAULT_COST*1); // FIXME
13960   expand %{
13961     vsh8S_reg(dst, src, shift);
13962   %}
13963 %}
13964 
13965 instruct vsl4S_immI(vecD dst, vecD src, immI shift) %{
13966   predicate(n->as_Vector()->length() == 4);
13967   match(Set dst (LShiftVS src shift));
13968   size(4);
13969   ins_cost(DEFAULT_COST); // FIXME
13970   format %{
13971     "VSHL.I16 $dst.D,$src.D,$shift\t! logical left shift packed4S"
13972   %}
13973   ins_encode %{
13974     bool quad = false;
13975     __ vshli($dst$$FloatRegister, $src$$FloatRegister, 16, $shift$$constant,
13976              quad);
13977   %}
13978   ins_pipe( ialu_reg_reg ); // FIXME
13979 %}
13980 
13981 instruct vsl8S_immI(vecX dst, vecX src, immI shift) %{
13982   predicate(n->as_Vector()->length() == 8);
13983   match(Set dst (LShiftVS src shift));
13984   size(4);
13985   ins_cost(DEFAULT_COST); // FIXME
13986   format %{
13987     "VSHL.I16 $dst.Q,$src.Q,$shift\t! logical left shift packed8S"
13988   %}
13989   ins_encode %{
13990     bool quad = true;
13991     __ vshli($dst$$FloatRegister, $src$$FloatRegister, 16, $shift$$constant,
13992              quad);
13993   %}
13994   ins_pipe( ialu_reg_reg ); // FIXME
13995 %}
13996 
13997 // Integers vector logical left/right shift
13998 instruct vsl2I_reg(vecD dst, vecD src, vecD shift) %{
13999   predicate(n->as_Vector()->length() == 2 && VM_Version::has_simd());
14000   match(Set dst (LShiftVI src shift));
14001   match(Set dst (URShiftVI src shift));
14002   size(4*1);
14003   ins_cost(DEFAULT_COST*1); // FIXME
14004   expand %{
14005     vsh2I_reg(dst, src, shift);
14006   %}
14007 %}
14008 
14009 instruct vsl4I_reg(vecX dst, vecX src, vecX shift) %{
14010   predicate(n->as_Vector()->length() == 4 && VM_Version::has_simd());
14011   match(Set dst (LShiftVI src shift));
14012   match(Set dst (URShiftVI src shift));
14013   size(4*1);
14014   ins_cost(DEFAULT_COST*1); // FIXME
14015   expand %{
14016     vsh4I_reg(dst, src, shift);
14017   %}
14018 %}
14019 
14020 instruct vsl2I_immI(vecD dst, vecD src, immI shift) %{
14021   predicate(n->as_Vector()->length() == 2 && VM_Version::has_simd());
14022   match(Set dst (LShiftVI src shift));
14023   size(4);
14024   ins_cost(DEFAULT_COST); // FIXME
14025   format %{
14026     "VSHL.I32 $dst.D,$src.D,$shift\t! logical left shift packed2I"
14027   %}
14028   ins_encode %{
14029     bool quad = false;
14030     __ vshli($dst$$FloatRegister, $src$$FloatRegister, 32, $shift$$constant,
14031              quad);
14032   %}
14033   ins_pipe( ialu_reg_reg ); // FIXME
14034 %}
14035 
14036 instruct vsl4I_immI(vecX dst, vecX src, immI shift) %{
14037   predicate(n->as_Vector()->length() == 4 && VM_Version::has_simd());
14038   match(Set dst (LShiftVI src shift));
14039   size(4);
14040   ins_cost(DEFAULT_COST); // FIXME
14041   format %{
14042     "VSHL.I32 $dst.Q,$src.Q,$shift\t! logical left shift packed4I"
14043   %}
14044   ins_encode %{
14045     bool quad = true;
14046     __ vshli($dst$$FloatRegister, $src$$FloatRegister, 32, $shift$$constant,
14047              quad);
14048   %}
14049   ins_pipe( ialu_reg_reg ); // FIXME
14050 %}
14051 
14052 // Longs vector logical left/right shift
14053 instruct vsl2L_reg(vecX dst, vecX src, vecX shift) %{
14054   predicate(n->as_Vector()->length() == 2);
14055   match(Set dst (LShiftVL src shift));
14056   match(Set dst (URShiftVL src shift));
14057   size(4*1);
14058   ins_cost(DEFAULT_COST*1); // FIXME
14059   expand %{
14060     vsh2L_reg(dst, src, shift);
14061   %}
14062 %}
14063 
14064 instruct vsl2L_immI(vecX dst, vecX src, immI shift) %{
14065   predicate(n->as_Vector()->length() == 2);
14066   match(Set dst (LShiftVL src shift));
14067   size(4);
14068   ins_cost(DEFAULT_COST); // FIXME
14069   format %{
14070     "VSHL.I64 $dst.Q,$src.Q,$shift\t! logical left shift packed2L"
14071   %}
14072   ins_encode %{
14073     bool quad = true;
14074     __ vshli($dst$$FloatRegister, $src$$FloatRegister, 64, $shift$$constant,
14075              quad);
14076   %}
14077   ins_pipe( ialu_reg_reg ); // FIXME
14078 %}
14079 
14080 // ----------------------- LogicalRightShift -----------------------------------
14081 
14082 // Bytes/Shorts vector logical right shift produces incorrect Java result
14083 // for negative data because java code convert short value into int with
14084 // sign extension before a shift.
14085 
14086 // Chars vector logical right shift
14087 instruct vsrl4S_immI(vecD dst, vecD src, immI shift) %{
14088   predicate(n->as_Vector()->length() == 4);
14089   match(Set dst (URShiftVS src shift));
14090   size(4);
14091   ins_cost(DEFAULT_COST); // FIXME
14092   format %{
14093     "VSHR.U16 $dst.D,$src.D,$shift\t! logical right shift packed4S"
14094   %}
14095   ins_encode %{
14096     bool quad = false;
14097     __ vshrUI($dst$$FloatRegister, $src$$FloatRegister, 16, $shift$$constant,
14098              quad);
14099   %}
14100   ins_pipe( ialu_reg_reg ); // FIXME
14101 %}
14102 
14103 instruct vsrl8S_immI(vecX dst, vecX src, immI shift) %{
14104   predicate(n->as_Vector()->length() == 8);
14105   match(Set dst (URShiftVS src shift));
14106   size(4);
14107   ins_cost(DEFAULT_COST); // FIXME
14108   format %{
14109     "VSHR.U16 $dst.Q,$src.Q,$shift\t! logical right shift packed8S"
14110   %}
14111   ins_encode %{
14112     bool quad = true;
14113     __ vshrUI($dst$$FloatRegister, $src$$FloatRegister, 16, $shift$$constant,
14114              quad);
14115   %}
14116   ins_pipe( ialu_reg_reg ); // FIXME
14117 %}
14118 
14119 // Integers vector logical right shift
14120 instruct vsrl2I_immI(vecD dst, vecD src, immI shift) %{
14121   predicate(n->as_Vector()->length() == 2 && VM_Version::has_simd());
14122   match(Set dst (URShiftVI src shift));
14123   size(4);
14124   ins_cost(DEFAULT_COST); // FIXME
14125   format %{
14126     "VSHR.U32 $dst.D,$src.D,$shift\t! logical right shift packed2I"
14127   %}
14128   ins_encode %{
14129     bool quad = false;
14130     __ vshrUI($dst$$FloatRegister, $src$$FloatRegister, 32, $shift$$constant,
14131              quad);
14132   %}
14133   ins_pipe( ialu_reg_reg ); // FIXME
14134 %}
14135 
14136 instruct vsrl4I_immI(vecX dst, vecX src, immI shift) %{
14137   predicate(n->as_Vector()->length() == 4 && VM_Version::has_simd());
14138   match(Set dst (URShiftVI src shift));
14139   size(4);
14140   ins_cost(DEFAULT_COST); // FIXME
14141   format %{
14142     "VSHR.U32 $dst.Q,$src.Q,$shift\t! logical right shift packed4I"
14143   %}
14144   ins_encode %{
14145     bool quad = true;
14146     __ vshrUI($dst$$FloatRegister, $src$$FloatRegister, 32, $shift$$constant,
14147              quad);
14148   %}
14149   ins_pipe( ialu_reg_reg ); // FIXME
14150 %}
14151 
14152 // Longs vector logical right shift
14153 instruct vsrl2L_immI(vecX dst, vecX src, immI shift) %{
14154   predicate(n->as_Vector()->length() == 2);
14155   match(Set dst (URShiftVL src shift));
14156   size(4);
14157   ins_cost(DEFAULT_COST); // FIXME
14158   format %{
14159     "VSHR.U64 $dst.Q,$src.Q,$shift\t! logical right shift packed2L"
14160   %}
14161   ins_encode %{
14162     bool quad = true;
14163     __ vshrUI($dst$$FloatRegister, $src$$FloatRegister, 64, $shift$$constant,
14164              quad);
14165   %}
14166   ins_pipe( ialu_reg_reg ); // FIXME
14167 %}
14168 
14169 // ------------------- ArithmeticRightShift -----------------------------------
14170 
14171 // Bytes vector arithmetic left/right shift based on sign
14172 instruct vsha8B_reg(vecD dst, vecD src, vecD shift) %{
14173   predicate(n->as_Vector()->length() == 8);
14174   effect(DEF dst, USE src, USE shift);
14175   size(4);
14176   ins_cost(DEFAULT_COST); // FIXME
14177   format %{
14178     "VSHL.S8 $dst.D,$src.D,$shift.D\t! arithmetic right shift packed8B"
14179   %}
14180   ins_encode %{
14181     bool quad = false;
14182     __ vshlSI($dst$$FloatRegister, $shift$$FloatRegister, $src$$FloatRegister,
14183               MacroAssembler::VELEM_SIZE_8, quad);
14184   %}
14185   ins_pipe( ialu_reg_reg ); // FIXME
14186 %}
14187 
14188 instruct vsha16B_reg(vecX dst, vecX src, vecX shift) %{
14189   predicate(n->as_Vector()->length() == 16);
14190   effect(DEF dst, USE src, USE shift);
14191   size(4);
14192   ins_cost(DEFAULT_COST); // FIXME
14193   format %{
14194     "VSHL.S8 $dst.Q,$src.Q,$shift.Q\t! arithmetic right shift packed16B"
14195   %}
14196   ins_encode %{
14197     bool quad = true;
14198     __ vshlSI($dst$$FloatRegister, $shift$$FloatRegister, $src$$FloatRegister,
14199               MacroAssembler::VELEM_SIZE_8, quad);
14200   %}
14201   ins_pipe( ialu_reg_reg ); // FIXME
14202 %}
14203 
14204 // Shorts vector arithmetic left/right shift based on sign
14205 instruct vsha4S_reg(vecD dst, vecD src, vecD shift) %{
14206   predicate(n->as_Vector()->length() == 4);
14207   effect(DEF dst, USE src, USE shift);
14208   size(4);
14209   ins_cost(DEFAULT_COST); // FIXME
14210   format %{
14211     "VSHL.S16 $dst.D,$src.D,$shift.D\t! arithmetic right shift packed4S"
14212   %}
14213   ins_encode %{
14214     bool quad = false;
14215     __ vshlSI($dst$$FloatRegister, $shift$$FloatRegister, $src$$FloatRegister,
14216               MacroAssembler::VELEM_SIZE_16, quad);
14217   %}
14218   ins_pipe( ialu_reg_reg ); // FIXME
14219 %}
14220 
14221 instruct vsha8S_reg(vecX dst, vecX src, vecX shift) %{
14222   predicate(n->as_Vector()->length() == 8);
14223   effect(DEF dst, USE src, USE shift);
14224   size(4);
14225   ins_cost(DEFAULT_COST); // FIXME
14226   format %{
14227     "VSHL.S16 $dst.Q,$src.Q,$shift.Q\t! arithmetic right shift packed8S"
14228   %}
14229   ins_encode %{
14230     bool quad = true;
14231     __ vshlSI($dst$$FloatRegister, $shift$$FloatRegister, $src$$FloatRegister,
14232               MacroAssembler::VELEM_SIZE_16, quad);
14233   %}
14234   ins_pipe( ialu_reg_reg ); // FIXME
14235 %}
14236 
14237 // Integers vector arithmetic left/right shift based on sign
14238 instruct vsha2I_reg(vecD dst, vecD src, vecD shift) %{
14239   predicate(n->as_Vector()->length() == 2);
14240   effect(DEF dst, USE src, USE shift);
14241   size(4);
14242   ins_cost(DEFAULT_COST); // FIXME
14243   format %{
14244     "VSHL.S32 $dst.D,$src.D,$shift.D\t! arithmetic right shift packed2I"
14245   %}
14246   ins_encode %{
14247     bool quad = false;
14248     __ vshlSI($dst$$FloatRegister, $shift$$FloatRegister, $src$$FloatRegister,
14249               MacroAssembler::VELEM_SIZE_32, quad);
14250   %}
14251   ins_pipe( ialu_reg_reg ); // FIXME
14252 %}
14253 
14254 instruct vsha4I_reg(vecX dst, vecX src, vecX shift) %{
14255   predicate(n->as_Vector()->length() == 4);
14256   effect(DEF dst, USE src, USE shift);
14257   size(4);
14258   ins_cost(DEFAULT_COST); // FIXME
14259   format %{
14260     "VSHL.S32 $dst.Q,$src.Q,$shift.Q\t! arithmetic right shift packed4I"
14261   %}
14262   ins_encode %{
14263     bool quad = true;
14264     __ vshlSI($dst$$FloatRegister, $shift$$FloatRegister, $src$$FloatRegister,
14265               MacroAssembler::VELEM_SIZE_32, quad);
14266   %}
14267   ins_pipe( ialu_reg_reg ); // FIXME
14268 %}
14269 
14270 // Longs vector arithmetic left/right shift based on sign
14271 instruct vsha2L_reg(vecX dst, vecX src, vecX shift) %{
14272   predicate(n->as_Vector()->length() == 2);
14273   effect(DEF dst, USE src, USE shift);
14274   size(4);
14275   ins_cost(DEFAULT_COST); // FIXME
14276   format %{
14277     "VSHL.S64 $dst.Q,$src.Q,$shift.Q\t! arithmetic right shift packed2L"
14278   %}
14279   ins_encode %{
14280     bool quad = true;
14281     __ vshlSI($dst$$FloatRegister, $shift$$FloatRegister, $src$$FloatRegister,
14282               MacroAssembler::VELEM_SIZE_64, quad);
14283   %}
14284   ins_pipe( ialu_reg_reg ); // FIXME
14285 %}
14286 
14287 // Byte vector arithmetic right shift
14288 
14289 instruct vsra8B_reg(vecD dst, vecD src, vecD shift) %{
14290   predicate(n->as_Vector()->length() == 8);
14291   match(Set dst (RShiftVB src shift));
14292   size(4);
14293   ins_cost(DEFAULT_COST); // FIXME
14294   expand %{
14295     vsha8B_reg(dst, src, shift);
14296   %}
14297 %}
14298 
14299 instruct vsrl16B_reg(vecX dst, vecX src, vecX shift) %{
14300   predicate(n->as_Vector()->length() == 16);
14301   match(Set dst (RShiftVB src shift));
14302   size(4);
14303   ins_cost(DEFAULT_COST); // FIXME
14304   expand %{
14305     vsha16B_reg(dst, src, shift);
14306   %}
14307 %}
14308 
14309 instruct vsrl8B_immI(vecD dst, vecD src, immI shift) %{
14310   predicate(n->as_Vector()->length() == 8);
14311   match(Set dst (RShiftVB src shift));
14312   size(4);
14313   ins_cost(DEFAULT_COST); // FIXME
14314   format %{
14315     "VSHR.S8 $dst.D,$src.D,$shift\t! logical right shift packed8B"
14316   %}
14317   ins_encode %{
14318     bool quad = false;
14319     __ vshrSI($dst$$FloatRegister, $src$$FloatRegister, 8, $shift$$constant,
14320              quad);
14321   %}
14322   ins_pipe( ialu_reg_reg ); // FIXME
14323 %}
14324 
14325 instruct vsrl16B_immI(vecX dst, vecX src, immI shift) %{
14326   predicate(n->as_Vector()->length() == 16);
14327   match(Set dst (RShiftVB src shift));
14328   size(4);
14329   ins_cost(DEFAULT_COST); // FIXME
14330   format %{
14331     "VSHR.S8 $dst.Q,$src.Q,$shift\t! logical right shift packed16B"
14332   %}
14333   ins_encode %{
14334     bool quad = true;
14335     __ vshrSI($dst$$FloatRegister, $src$$FloatRegister, 8, $shift$$constant,
14336              quad);
14337   %}
14338   ins_pipe( ialu_reg_reg ); // FIXME
14339 %}
14340 
14341 // Shorts vector arithmetic right shift
14342 instruct vsra4S_reg(vecD dst, vecD src, vecD shift) %{
14343   predicate(n->as_Vector()->length() == 4);
14344   match(Set dst (RShiftVS src shift));
14345   size(4);
14346   ins_cost(DEFAULT_COST); // FIXME
14347   expand %{
14348     vsha4S_reg(dst, src, shift);
14349   %}
14350 %}
14351 
14352 instruct vsra8S_reg(vecX dst, vecX src, vecX shift) %{
14353   predicate(n->as_Vector()->length() == 8);
14354   match(Set dst (RShiftVS src shift));
14355   size(4);
14356   ins_cost(DEFAULT_COST); // FIXME
14357   expand %{
14358     vsha8S_reg(dst, src, shift);
14359   %}
14360 %}
14361 
14362 instruct vsra4S_immI(vecD dst, vecD src, immI shift) %{
14363   predicate(n->as_Vector()->length() == 4);
14364   match(Set dst (RShiftVS src shift));
14365   size(4);
14366   ins_cost(DEFAULT_COST); // FIXME
14367   format %{
14368     "VSHR.S16 $dst.D,$src.D,$shift\t! logical right shift packed4S"
14369   %}
14370   ins_encode %{
14371     bool quad = false;
14372     __ vshrSI($dst$$FloatRegister, $src$$FloatRegister, 16, $shift$$constant,
14373              quad);
14374   %}
14375   ins_pipe( ialu_reg_reg ); // FIXME
14376 %}
14377 
14378 instruct vsra8S_immI(vecX dst, vecX src, immI shift) %{
14379   predicate(n->as_Vector()->length() == 8);
14380   match(Set dst (RShiftVS src shift));
14381   size(4);
14382   ins_cost(DEFAULT_COST); // FIXME
14383   format %{
14384     "VSHR.S16 $dst.Q,$src.Q,$shift\t! logical right shift packed8S"
14385   %}
14386   ins_encode %{
14387     bool quad = true;
14388     __ vshrSI($dst$$FloatRegister, $src$$FloatRegister, 16, $shift$$constant,
14389              quad);
14390   %}
14391   ins_pipe( ialu_reg_reg ); // FIXME
14392 %}
14393 
14394 // Integers vector arithmetic right shift
14395 instruct vsra2I_reg(vecD dst, vecD src, vecD shift) %{
14396   predicate(n->as_Vector()->length() == 2);
14397   match(Set dst (RShiftVI src shift));
14398   size(4);
14399   ins_cost(DEFAULT_COST); // FIXME
14400   expand %{
14401     vsha2I_reg(dst, src, shift);
14402   %}
14403 %}
14404 
14405 instruct vsra4I_reg(vecX dst, vecX src, vecX shift) %{
14406   predicate(n->as_Vector()->length() == 4);
14407   match(Set dst (RShiftVI src shift));
14408   size(4);
14409   ins_cost(DEFAULT_COST); // FIXME
14410   expand %{
14411     vsha4I_reg(dst, src, shift);
14412   %}
14413 %}
14414 
14415 instruct vsra2I_immI(vecD dst, vecD src, immI shift) %{
14416   predicate(n->as_Vector()->length() == 2);
14417   match(Set dst (RShiftVI src shift));
14418   size(4);
14419   ins_cost(DEFAULT_COST); // FIXME
14420   format %{
14421     "VSHR.S32 $dst.D,$src.D,$shift\t! logical right shift packed2I"
14422   %}
14423   ins_encode %{
14424     bool quad = false;
14425     __ vshrSI($dst$$FloatRegister, $src$$FloatRegister, 32, $shift$$constant,
14426              quad);
14427   %}
14428   ins_pipe( ialu_reg_reg ); // FIXME
14429 %}
14430 
14431 instruct vsra4I_immI(vecX dst, vecX src, immI shift) %{
14432   predicate(n->as_Vector()->length() == 4);
14433   match(Set dst (RShiftVI src shift));
14434   size(4);
14435   ins_cost(DEFAULT_COST); // FIXME
14436   format %{
14437     "VSHR.S32 $dst.Q,$src.Q,$shift\t! logical right shift packed4I"
14438   %}
14439   ins_encode %{
14440     bool quad = true;
14441     __ vshrSI($dst$$FloatRegister, $src$$FloatRegister, 32, $shift$$constant,
14442              quad);
14443   %}
14444   ins_pipe( ialu_reg_reg ); // FIXME
14445 %}
14446 
14447 // Longs vector arithmetic right shift
14448 instruct vsra2L_reg(vecX dst, vecX src, vecX shift) %{
14449   predicate(n->as_Vector()->length() == 2);
14450   match(Set dst (RShiftVL src shift));
14451   size(4);
14452   ins_cost(DEFAULT_COST); // FIXME
14453   expand %{
14454     vsha2L_reg(dst, src, shift);
14455   %}
14456 %}
14457 
14458 instruct vsra2L_immI(vecX dst, vecX src, immI shift) %{
14459   predicate(n->as_Vector()->length() == 2);
14460   match(Set dst (RShiftVL src shift));
14461   size(4);
14462   ins_cost(DEFAULT_COST); // FIXME
14463   format %{
14464     "VSHR.S64 $dst.Q,$src.Q,$shift\t! logical right shift packed2L"
14465   %}
14466   ins_encode %{
14467     bool quad = true;
14468     __ vshrSI($dst$$FloatRegister, $src$$FloatRegister, 64, $shift$$constant,
14469              quad);
14470   %}
14471   ins_pipe( ialu_reg_reg ); // FIXME
14472 %}
14473 
14474 // --------------------------------- AND --------------------------------------
14475 
14476 instruct vandD(vecD dst, vecD src1, vecD src2) %{
14477   predicate(n->as_Vector()->length_in_bytes() == 8);
14478   match(Set dst (AndV src1 src2));
14479   format %{ "VAND    $dst.D,$src1.D,$src2.D\t! and vectors (8 bytes)" %}
14480   ins_encode %{
14481     bool quad = false;
14482     __ vandI($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister,
14483              quad);
14484   %}
14485   ins_pipe( ialu_reg_reg ); // FIXME
14486 %}
14487 
14488 instruct vandX(vecX dst, vecX src1, vecX src2) %{
14489   predicate(n->as_Vector()->length_in_bytes() == 16);
14490   match(Set dst (AndV src1 src2));
14491   format %{ "VAND    $dst.Q,$src1.Q,$src2.Q\t! and vectors (16 bytes)" %}
14492   ins_encode %{
14493     bool quad = true;
14494     __ vandI($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister,
14495              quad);
14496   %}
14497   ins_pipe( ialu_reg_reg ); // FIXME
14498 %}
14499 
14500 // --------------------------------- OR ---------------------------------------
14501 
14502 instruct vorD(vecD dst, vecD src1, vecD src2) %{
14503   predicate(n->as_Vector()->length_in_bytes() == 8);
14504   match(Set dst (OrV src1 src2));
14505   format %{ "VOR     $dst.D,$src1.D,$src2.D\t! and vectors (8 bytes)" %}
14506   ins_encode %{
14507     bool quad = false;
14508     __ vorI($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister,
14509             quad);
14510   %}
14511   ins_pipe( ialu_reg_reg ); // FIXME
14512 %}
14513 
14514 instruct vorX(vecX dst, vecX src1, vecX src2) %{
14515   predicate(n->as_Vector()->length_in_bytes() == 16);
14516   match(Set dst (OrV src1 src2));
14517   format %{ "VOR     $dst.Q,$src1.Q,$src2.Q\t! and vectors (16 bytes)" %}
14518   ins_encode %{
14519     bool quad = true;
14520     __ vorI($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister,
14521             quad);
14522   %}
14523   ins_pipe( ialu_reg_reg ); // FIXME
14524 %}
14525 
14526 // --------------------------------- XOR --------------------------------------
14527 
14528 instruct vxorD(vecD dst, vecD src1, vecD src2) %{
14529   predicate(n->as_Vector()->length_in_bytes() == 8);
14530   match(Set dst (XorV src1 src2));
14531   format %{ "VXOR    $dst.D,$src1.D,$src2.D\t! and vectors (8 bytes)" %}
14532   ins_encode %{
14533     bool quad = false;
14534     __ vxorI($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister,
14535              quad);
14536   %}
14537   ins_pipe( ialu_reg_reg ); // FIXME
14538 %}
14539 
14540 instruct vxorX(vecX dst, vecX src1, vecX src2) %{
14541   predicate(n->as_Vector()->length_in_bytes() == 16);
14542   match(Set dst (XorV src1 src2));
14543   format %{ "VXOR    $dst.Q,$src1.Q,$src2.Q\t! and vectors (16 bytes)" %}
14544   ins_encode %{
14545     bool quad = true;
14546     __ vxorI($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister,
14547              quad);
14548   %}
14549   ins_pipe( ialu_reg_reg ); // FIXME
14550 %}
14551 
14552 
14553 //----------PEEPHOLE RULES-----------------------------------------------------
14554 // These must follow all instruction definitions as they use the names
14555 // defined in the instructions definitions.
14556 //
14557 // peepmatch ( root_instr_name [preceding_instruction]* );
14558 //
14559 // peepconstraint %{
14560 // (instruction_number.operand_name relational_op instruction_number.operand_name
14561 //  [, ...] );
14562 // // instruction numbers are zero-based using left to right order in peepmatch
14563 //
14564 // peepreplace ( instr_name  ( [instruction_number.operand_name]* ) );
14565 // // provide an instruction_number.operand_name for each operand that appears
14566 // // in the replacement instruction's match rule
14567 //
14568 // ---------VM FLAGS---------------------------------------------------------
14569 //
14570 // All peephole optimizations can be turned off using -XX:-OptoPeephole
14571 //
14572 // Each peephole rule is given an identifying number starting with zero and
14573 // increasing by one in the order seen by the parser.  An individual peephole
14574 // can be enabled, and all others disabled, by using -XX:OptoPeepholeAt=#
14575 // on the command-line.
14576 //
14577 // ---------CURRENT LIMITATIONS----------------------------------------------
14578 //
14579 // Only match adjacent instructions in same basic block
14580 // Only equality constraints
14581 // Only constraints between operands, not (0.dest_reg == EAX_enc)
14582 // Only one replacement instruction
14583 //
14584 // ---------EXAMPLE----------------------------------------------------------
14585 //
14586 // // pertinent parts of existing instructions in architecture description
14587 // instruct movI(eRegI dst, eRegI src) %{
14588 //   match(Set dst (CopyI src));
14589 // %}
14590 //
14591 // instruct incI_eReg(eRegI dst, immI1 src, eFlagsReg cr) %{
14592 //   match(Set dst (AddI dst src));
14593 //   effect(KILL cr);
14594 // %}
14595 //
14596 // // Change (inc mov) to lea
14597 // peephole %{
14598 //   // increment preceeded by register-register move
14599 //   peepmatch ( incI_eReg movI );
14600 //   // require that the destination register of the increment
14601 //   // match the destination register of the move
14602 //   peepconstraint ( 0.dst == 1.dst );
14603 //   // construct a replacement instruction that sets
14604 //   // the destination to ( move's source register + one )
14605 //   peepreplace ( incI_eReg_immI1( 0.dst 1.src 0.src ) );
14606 // %}
14607 //
14608 
14609 // // Change load of spilled value to only a spill
14610 // instruct storeI(memory mem, eRegI src) %{
14611 //   match(Set mem (StoreI mem src));
14612 // %}
14613 //
14614 // instruct loadI(eRegI dst, memory mem) %{
14615 //   match(Set dst (LoadI mem));
14616 // %}
14617 //
14618 // peephole %{
14619 //   peepmatch ( loadI storeI );
14620 //   peepconstraint ( 1.src == 0.dst, 1.mem == 0.mem );
14621 //   peepreplace ( storeI( 1.mem 1.mem 1.src ) );
14622 // %}
14623 
14624 //----------SMARTSPILL RULES---------------------------------------------------
14625 // These must follow all instruction definitions as they use the names
14626 // defined in the instructions definitions.
14627 //
14628 // ARM will probably not have any of these rules due to RISC instruction set.
14629 
14630 //----------PIPELINE-----------------------------------------------------------
14631 // Rules which define the behavior of the target architectures pipeline.