1 /*
   2  * Copyright (c) 1997, 2011, Oracle and/or its affiliates. All rights reserved.
   3  * Copyright (c) 2014, 2015, Red Hat Inc. All rights reserved.
   4  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   5  *
   6  * This code is free software; you can redistribute it and/or modify it
   7  * under the terms of the GNU General Public License version 2 only, as
   8  * published by the Free Software Foundation.
   9  *
  10  * This code is distributed in the hope that it will be useful, but WITHOUT
  11  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  12  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  13  * version 2 for more details (a copy is included in the LICENSE file that
  14  * accompanied this code).
  15  *
  16  * You should have received a copy of the GNU General Public License version
  17  * 2 along with this work; if not, write to the Free Software Foundation,
  18  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  19  *
  20  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  21  * or visit www.oracle.com if you need additional information or have any
  22  * questions.
  23  *
  24  */
  25 
  26 #ifndef CPU_AARCH64_VM_ASSEMBLER_AARCH64_HPP
  27 #define CPU_AARCH64_VM_ASSEMBLER_AARCH64_HPP
  28 
  29 #include "asm/register.hpp"
  30 
  31 // definitions of various symbolic names for machine registers
  32 
  33 // First intercalls between C and Java which use 8 general registers
  34 // and 8 floating registers
  35 
  36 // we also have to copy between x86 and ARM registers but that's a
  37 // secondary complication -- not all code employing C call convention
  38 // executes as x86 code though -- we generate some of it
  39 
  40 class Argument VALUE_OBJ_CLASS_SPEC {
  41  public:
  42   enum {
  43     n_int_register_parameters_c   = 8,  // r0, r1, ... r7 (c_rarg0, c_rarg1, ...)
  44     n_float_register_parameters_c = 8,  // v0, v1, ... v7 (c_farg0, c_farg1, ... )
  45 
  46     n_int_register_parameters_j   = 8, // r1, ... r7, r0 (rj_rarg0, j_rarg1, ...
  47     n_float_register_parameters_j = 8  // v0, v1, ... v7 (j_farg0, j_farg1, ...
  48   };
  49 };
  50 
  51 REGISTER_DECLARATION(Register, c_rarg0, r0);
  52 REGISTER_DECLARATION(Register, c_rarg1, r1);
  53 REGISTER_DECLARATION(Register, c_rarg2, r2);
  54 REGISTER_DECLARATION(Register, c_rarg3, r3);
  55 REGISTER_DECLARATION(Register, c_rarg4, r4);
  56 REGISTER_DECLARATION(Register, c_rarg5, r5);
  57 REGISTER_DECLARATION(Register, c_rarg6, r6);
  58 REGISTER_DECLARATION(Register, c_rarg7, r7);
  59 
  60 REGISTER_DECLARATION(FloatRegister, c_farg0, v0);
  61 REGISTER_DECLARATION(FloatRegister, c_farg1, v1);
  62 REGISTER_DECLARATION(FloatRegister, c_farg2, v2);
  63 REGISTER_DECLARATION(FloatRegister, c_farg3, v3);
  64 REGISTER_DECLARATION(FloatRegister, c_farg4, v4);
  65 REGISTER_DECLARATION(FloatRegister, c_farg5, v5);
  66 REGISTER_DECLARATION(FloatRegister, c_farg6, v6);
  67 REGISTER_DECLARATION(FloatRegister, c_farg7, v7);
  68 
  69 // Symbolically name the register arguments used by the Java calling convention.
  70 // We have control over the convention for java so we can do what we please.
  71 // What pleases us is to offset the java calling convention so that when
  72 // we call a suitable jni method the arguments are lined up and we don't
  73 // have to do much shuffling. A suitable jni method is non-static and a
  74 // small number of arguments
  75 //
  76 //  |--------------------------------------------------------------------|
  77 //  | c_rarg0  c_rarg1  c_rarg2 c_rarg3 c_rarg4 c_rarg5 c_rarg6 c_rarg7  |
  78 //  |--------------------------------------------------------------------|
  79 //  | r0       r1       r2      r3      r4      r5      r6      r7       |
  80 //  |--------------------------------------------------------------------|
  81 //  | j_rarg7  j_rarg0  j_rarg1 j_rarg2 j_rarg3 j_rarg4 j_rarg5 j_rarg6  |
  82 //  |--------------------------------------------------------------------|
  83 
  84 
  85 REGISTER_DECLARATION(Register, j_rarg0, c_rarg1);
  86 REGISTER_DECLARATION(Register, j_rarg1, c_rarg2);
  87 REGISTER_DECLARATION(Register, j_rarg2, c_rarg3);
  88 REGISTER_DECLARATION(Register, j_rarg3, c_rarg4);
  89 REGISTER_DECLARATION(Register, j_rarg4, c_rarg5);
  90 REGISTER_DECLARATION(Register, j_rarg5, c_rarg6);
  91 REGISTER_DECLARATION(Register, j_rarg6, c_rarg7);
  92 REGISTER_DECLARATION(Register, j_rarg7, c_rarg0);
  93 
  94 // Java floating args are passed as per C
  95 
  96 REGISTER_DECLARATION(FloatRegister, j_farg0, v0);
  97 REGISTER_DECLARATION(FloatRegister, j_farg1, v1);
  98 REGISTER_DECLARATION(FloatRegister, j_farg2, v2);
  99 REGISTER_DECLARATION(FloatRegister, j_farg3, v3);
 100 REGISTER_DECLARATION(FloatRegister, j_farg4, v4);
 101 REGISTER_DECLARATION(FloatRegister, j_farg5, v5);
 102 REGISTER_DECLARATION(FloatRegister, j_farg6, v6);
 103 REGISTER_DECLARATION(FloatRegister, j_farg7, v7);
 104 
 105 // registers used to hold VM data either temporarily within a method
 106 // or across method calls
 107 
 108 // volatile (caller-save) registers
 109 
 110 // r8 is used for indirect result location return
 111 // we use it and r9 as scratch registers
 112 REGISTER_DECLARATION(Register, rscratch1, r8);
 113 REGISTER_DECLARATION(Register, rscratch2, r9);
 114 
 115 // current method -- must be in a call-clobbered register
 116 REGISTER_DECLARATION(Register, rmethod,   r12);
 117 
 118 // non-volatile (callee-save) registers are r16-29
 119 // of which the following are dedicated global state
 120 
 121 // link register
 122 REGISTER_DECLARATION(Register, lr,        r30);
 123 // frame pointer
 124 REGISTER_DECLARATION(Register, rfp,       r29);
 125 // current thread
 126 REGISTER_DECLARATION(Register, rthread,   r28);
 127 // base of heap
 128 REGISTER_DECLARATION(Register, rheapbase, r27);
 129 // constant pool cache
 130 REGISTER_DECLARATION(Register, rcpool,    r26);
 131 // monitors allocated on stack
 132 REGISTER_DECLARATION(Register, rmonitors, r25);
 133 // locals on stack
 134 REGISTER_DECLARATION(Register, rlocals,   r24);
 135 // bytecode pointer
 136 REGISTER_DECLARATION(Register, rbcp,      r22);
 137 // Dispatch table base
 138 REGISTER_DECLARATION(Register, rdispatch,      r21);
 139 // Java stack pointer
 140 REGISTER_DECLARATION(Register, esp,      r20);
 141 
 142 // TODO : x86 uses rbp to save SP in method handle code
 143 // we may need to do the same with fp
 144 // JSR 292 fixed register usages:
 145 //REGISTER_DECLARATION(Register, r_mh_SP_save, r29);
 146 
 147 #define assert_cond(ARG1) assert(ARG1, #ARG1)
 148 
 149 namespace asm_util {
 150   uint32_t encode_logical_immediate(bool is32, uint64_t imm);
 151 };
 152 
 153 using namespace asm_util;
 154 
 155 
 156 class Assembler;
 157 
 158 class Instruction_aarch64 {
 159   unsigned insn;
 160 #ifdef ASSERT
 161   unsigned bits;
 162 #endif
 163   Assembler *assem;
 164 
 165 public:
 166 
 167   Instruction_aarch64(class Assembler *as) {
 168 #ifdef ASSERT
 169     bits = 0;
 170 #endif
 171     insn = 0;
 172     assem = as;
 173   }
 174 
 175   inline ~Instruction_aarch64();
 176 
 177   unsigned &get_insn() { return insn; }
 178 #ifdef ASSERT
 179   unsigned &get_bits() { return bits; }
 180 #endif
 181 
 182   static inline int32_t extend(unsigned val, int hi = 31, int lo = 0) {
 183     union {
 184       unsigned u;
 185       int n;
 186     };
 187 
 188     u = val << (31 - hi);
 189     n = n >> (31 - hi + lo);
 190     return n;
 191   }
 192 
 193   static inline uint32_t extract(uint32_t val, int msb, int lsb) {
 194     int nbits = msb - lsb + 1;
 195     assert_cond(msb >= lsb);
 196     uint32_t mask = (1U << nbits) - 1;
 197     uint32_t result = val >> lsb;
 198     result &= mask;
 199     return result;
 200   }
 201 
 202   static inline int32_t sextract(uint32_t val, int msb, int lsb) {
 203     uint32_t uval = extract(val, msb, lsb);
 204     return extend(uval, msb - lsb);
 205   }
 206 
 207   static void patch(address a, int msb, int lsb, unsigned long val) {
 208     int nbits = msb - lsb + 1;
 209     guarantee(val < (1U << nbits), "Field too big for insn");
 210     assert_cond(msb >= lsb);
 211     unsigned mask = (1U << nbits) - 1;
 212     val <<= lsb;
 213     mask <<= lsb;
 214     unsigned target = *(unsigned *)a;
 215     target &= ~mask;
 216     target |= val;
 217     *(unsigned *)a = target;
 218   }
 219 
 220   static void spatch(address a, int msb, int lsb, long val) {
 221     int nbits = msb - lsb + 1;
 222     long chk = val >> (nbits - 1);
 223     guarantee (chk == -1 || chk == 0, "Field too big for insn");
 224     unsigned uval = val;
 225     unsigned mask = (1U << nbits) - 1;
 226     uval &= mask;
 227     uval <<= lsb;
 228     mask <<= lsb;
 229     unsigned target = *(unsigned *)a;
 230     target &= ~mask;
 231     target |= uval;
 232     *(unsigned *)a = target;
 233   }
 234 
 235   void f(unsigned val, int msb, int lsb) {
 236     int nbits = msb - lsb + 1;
 237     guarantee(val < (1U << nbits), "Field too big for insn");
 238     assert_cond(msb >= lsb);
 239     unsigned mask = (1U << nbits) - 1;
 240     val <<= lsb;
 241     mask <<= lsb;
 242     insn |= val;
 243     assert_cond((bits & mask) == 0);
 244 #ifdef ASSERT
 245     bits |= mask;
 246 #endif
 247   }
 248 
 249   void f(unsigned val, int bit) {
 250     f(val, bit, bit);
 251   }
 252 
 253   void sf(long val, int msb, int lsb) {
 254     int nbits = msb - lsb + 1;
 255     long chk = val >> (nbits - 1);
 256     guarantee (chk == -1 || chk == 0, "Field too big for insn");
 257     unsigned uval = val;
 258     unsigned mask = (1U << nbits) - 1;
 259     uval &= mask;
 260     f(uval, lsb + nbits - 1, lsb);
 261   }
 262 
 263   void rf(Register r, int lsb) {
 264     f(r->encoding_nocheck(), lsb + 4, lsb);
 265   }
 266 
 267   // reg|ZR
 268   void zrf(Register r, int lsb) {
 269     f(r->encoding_nocheck() - (r == zr), lsb + 4, lsb);
 270   }
 271 
 272   // reg|SP
 273   void srf(Register r, int lsb) {
 274     f(r == sp ? 31 : r->encoding_nocheck(), lsb + 4, lsb);
 275   }
 276 
 277   void rf(FloatRegister r, int lsb) {
 278     f(r->encoding_nocheck(), lsb + 4, lsb);
 279   }
 280 
 281   unsigned get(int msb = 31, int lsb = 0) {
 282     int nbits = msb - lsb + 1;
 283     unsigned mask = ((1U << nbits) - 1) << lsb;
 284     assert_cond(bits & mask == mask);
 285     return (insn & mask) >> lsb;
 286   }
 287 
 288   void fixed(unsigned value, unsigned mask) {
 289     assert_cond ((mask & bits) == 0);
 290 #ifdef ASSERT
 291     bits |= mask;
 292 #endif
 293     insn |= value;
 294   }
 295 };
 296 
 297 #define starti Instruction_aarch64 do_not_use(this); set_current(&do_not_use)
 298 
 299 class PrePost {
 300   int _offset;
 301   Register _r;
 302 public:
 303   PrePost(Register reg, int o) : _r(reg), _offset(o) { }
 304   int offset() { return _offset; }
 305   Register reg() { return _r; }
 306 };
 307 
 308 class Pre : public PrePost {
 309 public:
 310   Pre(Register reg, int o) : PrePost(reg, o) { }
 311 };
 312 class Post : public PrePost {
 313 public:
 314   Post(Register reg, int o) : PrePost(reg, o) { }
 315 };
 316 
 317 namespace ext
 318 {
 319   enum operation { uxtb, uxth, uxtw, uxtx, sxtb, sxth, sxtw, sxtx };
 320 };
 321 
 322 // abs methods which cannot overflow and so are well-defined across
 323 // the entire domain of integer types.
 324 static inline unsigned int uabs(unsigned int n) {
 325   union {
 326     unsigned int result;
 327     int value;
 328   };
 329   result = n;
 330   if (value < 0) result = -result;
 331   return result;
 332 }
 333 static inline unsigned long uabs(unsigned long n) {
 334   union {
 335     unsigned long result;
 336     long value;
 337   };
 338   result = n;
 339   if (value < 0) result = -result;
 340   return result;
 341 }
 342 static inline unsigned long uabs(long n) { return uabs((unsigned long)n); }
 343 static inline unsigned long uabs(int n) { return uabs((unsigned int)n); }
 344 
 345 // Addressing modes
 346 class Address VALUE_OBJ_CLASS_SPEC {
 347  public:
 348 
 349   enum mode { no_mode, base_plus_offset, pre, post, pcrel,
 350               base_plus_offset_reg, literal };
 351 
 352   // Shift and extend for base reg + reg offset addressing
 353   class extend {
 354     int _option, _shift;
 355     ext::operation _op;
 356   public:
 357     extend() { }
 358     extend(int s, int o, ext::operation op) : _shift(s), _option(o), _op(op) { }
 359     int option() const{ return _option; }
 360     int shift() const { return _shift; }
 361     ext::operation op() const { return _op; }
 362   };
 363   class uxtw : public extend {
 364   public:
 365     uxtw(int shift = -1): extend(shift, 0b010, ext::uxtw) { }
 366   };
 367   class lsl : public extend {
 368   public:
 369     lsl(int shift = -1): extend(shift, 0b011, ext::uxtx) { }
 370   };
 371   class sxtw : public extend {
 372   public:
 373     sxtw(int shift = -1): extend(shift, 0b110, ext::sxtw) { }
 374   };
 375   class sxtx : public extend {
 376   public:
 377     sxtx(int shift = -1): extend(shift, 0b111, ext::sxtx) { }
 378   };
 379 
 380  private:
 381   Register _base;
 382   Register _index;
 383   long _offset;
 384   enum mode _mode;
 385   extend _ext;
 386 
 387   RelocationHolder _rspec;
 388 
 389   // Typically we use AddressLiterals we want to use their rval
 390   // However in some situations we want the lval (effect address) of
 391   // the item.  We provide a special factory for making those lvals.
 392   bool _is_lval;
 393 
 394   // If the target is far we'll need to load the ea of this to a
 395   // register to reach it. Otherwise if near we can do PC-relative
 396   // addressing.
 397   address          _target;
 398 
 399  public:
 400   Address()
 401     : _mode(no_mode) { }
 402   Address(Register r)
 403     : _mode(base_plus_offset), _base(r), _offset(0), _index(noreg), _target(0) { }
 404   Address(Register r, int o)
 405     : _mode(base_plus_offset), _base(r), _offset(o), _index(noreg), _target(0) { }
 406   Address(Register r, long o)
 407     : _mode(base_plus_offset), _base(r), _offset(o), _index(noreg), _target(0) { }
 408   Address(Register r, unsigned long o)
 409     : _mode(base_plus_offset), _base(r), _offset(o), _index(noreg), _target(0) { }
 410 #ifdef ASSERT
 411   Address(Register r, ByteSize disp)
 412     : _mode(base_plus_offset), _base(r), _offset(in_bytes(disp)),
 413       _index(noreg), _target(0) { }
 414 #endif
 415   Address(Register r, Register r1, extend ext = lsl())
 416     : _mode(base_plus_offset_reg), _base(r), _index(r1),
 417     _ext(ext), _offset(0), _target(0) { }
 418   Address(Pre p)
 419     : _mode(pre), _base(p.reg()), _offset(p.offset()) { }
 420   Address(Post p)
 421     : _mode(post), _base(p.reg()), _offset(p.offset()), _target(0) { }
 422   Address(address target, RelocationHolder const& rspec)
 423     : _mode(literal),
 424       _rspec(rspec),
 425       _is_lval(false),
 426       _target(target)  { }
 427   Address(address target, relocInfo::relocType rtype = relocInfo::external_word_type);
 428   Address(Register base, RegisterOrConstant index, extend ext = lsl())
 429     : _base (base),
 430       _ext(ext), _offset(0), _target(0) {
 431     if (index.is_register()) {
 432       _mode = base_plus_offset_reg;
 433       _index = index.as_register();
 434     } else {
 435       guarantee(ext.option() == ext::uxtx, "should be");
 436       assert(index.is_constant(), "should be");
 437       _mode = base_plus_offset;
 438       _offset = index.as_constant() << ext.shift();
 439     }
 440   }
 441 
 442   Register base() const {
 443     guarantee((_mode == base_plus_offset | _mode == base_plus_offset_reg
 444                | _mode == post),
 445               "wrong mode");
 446     return _base;
 447   }
 448   long offset() const {
 449     return _offset;
 450   }
 451   Register index() const {
 452     return _index;
 453   }
 454   mode getMode() const {
 455     return _mode;
 456   }
 457   bool uses(Register reg) const { return _base == reg || _index == reg; }
 458   address target() const { return _target; }
 459   const RelocationHolder& rspec() const { return _rspec; }
 460 
 461   void encode(Instruction_aarch64 *i) const {
 462     i->f(0b111, 29, 27);
 463     i->srf(_base, 5);
 464 
 465     switch(_mode) {
 466     case base_plus_offset:
 467       {
 468         unsigned size = i->get(31, 30);
 469         if (i->get(26, 26) && i->get(23, 23)) {
 470           // SIMD Q Type - Size = 128 bits
 471           assert(size == 0, "bad size");
 472           size = 0b100;
 473         }
 474         unsigned mask = (1 << size) - 1;
 475         if (_offset < 0 || _offset & mask)
 476           {
 477             i->f(0b00, 25, 24);
 478             i->f(0, 21), i->f(0b00, 11, 10);
 479             i->sf(_offset, 20, 12);
 480           } else {
 481             i->f(0b01, 25, 24);
 482             i->f(_offset >> size, 21, 10);
 483           }
 484       }
 485       break;
 486 
 487     case base_plus_offset_reg:
 488       {
 489         i->f(0b00, 25, 24);
 490         i->f(1, 21);
 491         i->rf(_index, 16);
 492         i->f(_ext.option(), 15, 13);
 493         unsigned size = i->get(31, 30);
 494         if (i->get(26, 26) && i->get(23, 23)) {
 495           // SIMD Q Type - Size = 128 bits
 496           assert(size == 0, "bad size");
 497           size = 0b100;
 498         }
 499         if (size == 0) // It's a byte
 500           i->f(_ext.shift() >= 0, 12);
 501         else {
 502           if (_ext.shift() > 0)
 503             assert(_ext.shift() == (int)size, "bad shift");
 504           i->f(_ext.shift() > 0, 12);
 505         }
 506         i->f(0b10, 11, 10);
 507       }
 508       break;
 509 
 510     case pre:
 511       i->f(0b00, 25, 24);
 512       i->f(0, 21), i->f(0b11, 11, 10);
 513       i->sf(_offset, 20, 12);
 514       break;
 515 
 516     case post:
 517       i->f(0b00, 25, 24);
 518       i->f(0, 21), i->f(0b01, 11, 10);
 519       i->sf(_offset, 20, 12);
 520       break;
 521 
 522     default:
 523       ShouldNotReachHere();
 524     }
 525   }
 526 
 527   void encode_pair(Instruction_aarch64 *i) const {
 528     switch(_mode) {
 529     case base_plus_offset:
 530       i->f(0b010, 25, 23);
 531       break;
 532     case pre:
 533       i->f(0b011, 25, 23);
 534       break;
 535     case post:
 536       i->f(0b001, 25, 23);
 537       break;
 538     default:
 539       ShouldNotReachHere();
 540     }
 541 
 542     unsigned size; // Operand shift in 32-bit words
 543 
 544     if (i->get(26, 26)) { // float
 545       switch(i->get(31, 30)) {
 546       case 0b10:
 547         size = 2; break;
 548       case 0b01:
 549         size = 1; break;
 550       case 0b00:
 551         size = 0; break;
 552       default:
 553         ShouldNotReachHere();
 554       }
 555     } else {
 556       size = i->get(31, 31);
 557     }
 558 
 559     size = 4 << size;
 560     guarantee(_offset % size == 0, "bad offset");
 561     i->sf(_offset / size, 21, 15);
 562     i->srf(_base, 5);
 563   }
 564 
 565   void encode_nontemporal_pair(Instruction_aarch64 *i) const {
 566     // Only base + offset is allowed
 567     i->f(0b000, 25, 23);
 568     unsigned size = i->get(31, 31);
 569     size = 4 << size;
 570     guarantee(_offset % size == 0, "bad offset");
 571     i->sf(_offset / size, 21, 15);
 572     i->srf(_base, 5);
 573     guarantee(_mode == Address::base_plus_offset,
 574               "Bad addressing mode for non-temporal op");
 575   }
 576 
 577   void lea(MacroAssembler *, Register) const;
 578 
 579   static bool offset_ok_for_immed(long offset, int shift = 0) {
 580     unsigned mask = (1 << shift) - 1;
 581     if (offset < 0 || offset & mask) {
 582       return (uabs(offset) < (1 << (20 - 12))); // Unscaled offset
 583     } else {
 584       return ((offset >> shift) < (1 << (21 - 10 + 1))); // Scaled, unsigned offset
 585     }
 586   }
 587 };
 588 
 589 // Convience classes
 590 class RuntimeAddress: public Address {
 591 
 592   public:
 593 
 594   RuntimeAddress(address target) : Address(target, relocInfo::runtime_call_type) {}
 595 
 596 };
 597 
 598 class OopAddress: public Address {
 599 
 600   public:
 601 
 602   OopAddress(address target) : Address(target, relocInfo::oop_type){}
 603 
 604 };
 605 
 606 class ExternalAddress: public Address {
 607  private:
 608   static relocInfo::relocType reloc_for_target(address target) {
 609     // Sometimes ExternalAddress is used for values which aren't
 610     // exactly addresses, like the card table base.
 611     // external_word_type can't be used for values in the first page
 612     // so just skip the reloc in that case.
 613     return external_word_Relocation::can_be_relocated(target) ? relocInfo::external_word_type : relocInfo::none;
 614   }
 615 
 616  public:
 617 
 618   ExternalAddress(address target) : Address(target, reloc_for_target(target)) {}
 619 
 620 };
 621 
 622 class InternalAddress: public Address {
 623 
 624   public:
 625 
 626   InternalAddress(address target) : Address(target, relocInfo::internal_word_type) {}
 627 };
 628 
 629 const int FPUStateSizeInWords = 32 * 2;
 630 typedef enum {
 631   PLDL1KEEP = 0b00000, PLDL1STRM, PLDL2KEEP, PLDL2STRM, PLDL3KEEP, PLDL3STRM,
 632   PSTL1KEEP = 0b10000, PSTL1STRM, PSTL2KEEP, PSTL2STRM, PSTL3KEEP, PSTL3STRM,
 633   PLIL1KEEP = 0b01000, PLIL1STRM, PLIL2KEEP, PLIL2STRM, PLIL3KEEP, PLIL3STRM
 634 } prfop;
 635 
 636 class Assembler : public AbstractAssembler {
 637 
 638 #ifndef PRODUCT
 639   static const unsigned long asm_bp;
 640 
 641   void emit_long(jint x) {
 642     if ((unsigned long)pc() == asm_bp)
 643       asm volatile ("nop");
 644     AbstractAssembler::emit_int32(x);
 645   }
 646 #else
 647   void emit_long(jint x) {
 648     AbstractAssembler::emit_int32(x);
 649   }
 650 #endif
 651 
 652 public:
 653 
 654   enum { instruction_size = 4 };
 655 
 656   Address adjust(Register base, int offset, bool preIncrement) {
 657     if (preIncrement)
 658       return Address(Pre(base, offset));
 659     else
 660       return Address(Post(base, offset));
 661   }
 662 
 663   Address pre(Register base, int offset) {
 664     return adjust(base, offset, true);
 665   }
 666 
 667   Address post (Register base, int offset) {
 668     return adjust(base, offset, false);
 669   }
 670 
 671   Instruction_aarch64* current;
 672 
 673   void set_current(Instruction_aarch64* i) { current = i; }
 674 
 675   void f(unsigned val, int msb, int lsb) {
 676     current->f(val, msb, lsb);
 677   }
 678   void f(unsigned val, int msb) {
 679     current->f(val, msb, msb);
 680   }
 681   void sf(long val, int msb, int lsb) {
 682     current->sf(val, msb, lsb);
 683   }
 684   void rf(Register reg, int lsb) {
 685     current->rf(reg, lsb);
 686   }
 687   void srf(Register reg, int lsb) {
 688     current->srf(reg, lsb);
 689   }
 690   void zrf(Register reg, int lsb) {
 691     current->zrf(reg, lsb);
 692   }
 693   void rf(FloatRegister reg, int lsb) {
 694     current->rf(reg, lsb);
 695   }
 696   void fixed(unsigned value, unsigned mask) {
 697     current->fixed(value, mask);
 698   }
 699 
 700   void emit() {
 701     emit_long(current->get_insn());
 702     assert_cond(current->get_bits() == 0xffffffff);
 703     current = NULL;
 704   }
 705 
 706   typedef void (Assembler::* uncond_branch_insn)(address dest);
 707   typedef void (Assembler::* compare_and_branch_insn)(Register Rt, address dest);
 708   typedef void (Assembler::* test_and_branch_insn)(Register Rt, int bitpos, address dest);
 709   typedef void (Assembler::* prefetch_insn)(address target, prfop);
 710 
 711   void wrap_label(Label &L, uncond_branch_insn insn);
 712   void wrap_label(Register r, Label &L, compare_and_branch_insn insn);
 713   void wrap_label(Register r, int bitpos, Label &L, test_and_branch_insn insn);
 714   void wrap_label(Label &L, prfop, prefetch_insn insn);
 715 
 716   // PC-rel. addressing
 717 
 718   void adr(Register Rd, address dest);
 719   void _adrp(Register Rd, address dest);
 720 
 721   void adr(Register Rd, const Address &dest);
 722   void _adrp(Register Rd, const Address &dest);
 723 
 724   void adr(Register Rd, Label &L) {
 725     wrap_label(Rd, L, &Assembler::Assembler::adr);
 726   }
 727   void _adrp(Register Rd, Label &L) {
 728     wrap_label(Rd, L, &Assembler::_adrp);
 729   }
 730 
 731   void adrp(Register Rd, const Address &dest, unsigned long &offset);
 732 
 733 #undef INSN
 734 
 735   void add_sub_immediate(Register Rd, Register Rn, unsigned uimm, int op,
 736                          int negated_op);
 737 
 738   // Add/subtract (immediate)
 739 #define INSN(NAME, decode, negated)                                     \
 740   void NAME(Register Rd, Register Rn, unsigned imm, unsigned shift) {   \
 741     starti;                                                             \
 742     f(decode, 31, 29), f(0b10001, 28, 24), f(shift, 23, 22), f(imm, 21, 10); \
 743     zrf(Rd, 0), srf(Rn, 5);                                             \
 744   }                                                                     \
 745                                                                         \
 746   void NAME(Register Rd, Register Rn, unsigned imm) {                   \
 747     starti;                                                             \
 748     add_sub_immediate(Rd, Rn, imm, decode, negated);                    \
 749   }
 750 
 751   INSN(addsw, 0b001, 0b011);
 752   INSN(subsw, 0b011, 0b001);
 753   INSN(adds,  0b101, 0b111);
 754   INSN(subs,  0b111, 0b101);
 755 
 756 #undef INSN
 757 
 758 #define INSN(NAME, decode, negated)                     \
 759   void NAME(Register Rd, Register Rn, unsigned imm) {   \
 760     starti;                                             \
 761     add_sub_immediate(Rd, Rn, imm, decode, negated);    \
 762   }
 763 
 764   INSN(addw, 0b000, 0b010);
 765   INSN(subw, 0b010, 0b000);
 766   INSN(add,  0b100, 0b110);
 767   INSN(sub,  0b110, 0b100);
 768 
 769 #undef INSN
 770 
 771  // Logical (immediate)
 772 #define INSN(NAME, decode, is32)                                \
 773   void NAME(Register Rd, Register Rn, uint64_t imm) {           \
 774     starti;                                                     \
 775     uint32_t val = encode_logical_immediate(is32, imm);         \
 776     f(decode, 31, 29), f(0b100100, 28, 23), f(val, 22, 10);     \
 777     srf(Rd, 0), zrf(Rn, 5);                                     \
 778   }
 779 
 780   INSN(andw, 0b000, true);
 781   INSN(orrw, 0b001, true);
 782   INSN(eorw, 0b010, true);
 783   INSN(andr,  0b100, false);
 784   INSN(orr,  0b101, false);
 785   INSN(eor,  0b110, false);
 786 
 787 #undef INSN
 788 
 789 #define INSN(NAME, decode, is32)                                \
 790   void NAME(Register Rd, Register Rn, uint64_t imm) {           \
 791     starti;                                                     \
 792     uint32_t val = encode_logical_immediate(is32, imm);         \
 793     f(decode, 31, 29), f(0b100100, 28, 23), f(val, 22, 10);     \
 794     zrf(Rd, 0), zrf(Rn, 5);                                     \
 795   }
 796 
 797   INSN(ands, 0b111, false);
 798   INSN(andsw, 0b011, true);
 799 
 800 #undef INSN
 801 
 802   // Move wide (immediate)
 803 #define INSN(NAME, opcode)                                              \
 804   void NAME(Register Rd, unsigned imm, unsigned shift = 0) {            \
 805     assert_cond((shift/16)*16 == shift);                                \
 806     starti;                                                             \
 807     f(opcode, 31, 29), f(0b100101, 28, 23), f(shift/16, 22, 21),        \
 808       f(imm, 20, 5);                                                    \
 809     rf(Rd, 0);                                                          \
 810   }
 811 
 812   INSN(movnw, 0b000);
 813   INSN(movzw, 0b010);
 814   INSN(movkw, 0b011);
 815   INSN(movn, 0b100);
 816   INSN(movz, 0b110);
 817   INSN(movk, 0b111);
 818 
 819 #undef INSN
 820 
 821   // Bitfield
 822 #define INSN(NAME, opcode)                                              \
 823   void NAME(Register Rd, Register Rn, unsigned immr, unsigned imms) {   \
 824     starti;                                                             \
 825     f(opcode, 31, 22), f(immr, 21, 16), f(imms, 15, 10);                \
 826     rf(Rn, 5), rf(Rd, 0);                                               \
 827   }
 828 
 829   INSN(sbfmw, 0b0001001100);
 830   INSN(bfmw,  0b0011001100);
 831   INSN(ubfmw, 0b0101001100);
 832   INSN(sbfm,  0b1001001101);
 833   INSN(bfm,   0b1011001101);
 834   INSN(ubfm,  0b1101001101);
 835 
 836 #undef INSN
 837 
 838   // Extract
 839 #define INSN(NAME, opcode)                                              \
 840   void NAME(Register Rd, Register Rn, Register Rm, unsigned imms) {     \
 841     starti;                                                             \
 842     f(opcode, 31, 21), f(imms, 15, 10);                                 \
 843     rf(Rm, 16), rf(Rn, 5), rf(Rd, 0);                                   \
 844   }
 845 
 846   INSN(extrw, 0b00010011100);
 847   INSN(extr,  0b10010011110);
 848 
 849 #undef INSN
 850 
 851   // The maximum range of a branch is fixed for the AArch64
 852   // architecture.  In debug mode we shrink it in order to test
 853   // trampolines, but not so small that branches in the interpreter
 854   // are out of range.
 855   static const unsigned long branch_range = NOT_DEBUG(128 * M) DEBUG_ONLY(2 * M);
 856 
 857   static bool reachable_from_branch_at(address branch, address target) {
 858     return uabs(target - branch) < branch_range;
 859   }
 860 
 861   // Unconditional branch (immediate)
 862 #define INSN(NAME, opcode)                                              \
 863   void NAME(address dest) {                                             \
 864     starti;                                                             \
 865     long offset = (dest - pc()) >> 2;                                   \
 866     DEBUG_ONLY(assert(reachable_from_branch_at(pc(), dest), "debug only")); \
 867     f(opcode, 31), f(0b00101, 30, 26), sf(offset, 25, 0);               \
 868   }                                                                     \
 869   void NAME(Label &L) {                                                 \
 870     wrap_label(L, &Assembler::NAME);                                    \
 871   }                                                                     \
 872   void NAME(const Address &dest);
 873 
 874   INSN(b, 0);
 875   INSN(bl, 1);
 876 
 877 #undef INSN
 878 
 879   // Compare & branch (immediate)
 880 #define INSN(NAME, opcode)                              \
 881   void NAME(Register Rt, address dest) {                \
 882     long offset = (dest - pc()) >> 2;                   \
 883     starti;                                             \
 884     f(opcode, 31, 24), sf(offset, 23, 5), rf(Rt, 0);    \
 885   }                                                     \
 886   void NAME(Register Rt, Label &L) {                    \
 887     wrap_label(Rt, L, &Assembler::NAME);                \
 888   }
 889 
 890   INSN(cbzw,  0b00110100);
 891   INSN(cbnzw, 0b00110101);
 892   INSN(cbz,   0b10110100);
 893   INSN(cbnz,  0b10110101);
 894 
 895 #undef INSN
 896 
 897   // Test & branch (immediate)
 898 #define INSN(NAME, opcode)                                              \
 899   void NAME(Register Rt, int bitpos, address dest) {                    \
 900     long offset = (dest - pc()) >> 2;                                   \
 901     int b5 = bitpos >> 5;                                               \
 902     bitpos &= 0x1f;                                                     \
 903     starti;                                                             \
 904     f(b5, 31), f(opcode, 30, 24), f(bitpos, 23, 19), sf(offset, 18, 5); \
 905     rf(Rt, 0);                                                          \
 906   }                                                                     \
 907   void NAME(Register Rt, int bitpos, Label &L) {                        \
 908     wrap_label(Rt, bitpos, L, &Assembler::NAME);                        \
 909   }
 910 
 911   INSN(tbz,  0b0110110);
 912   INSN(tbnz, 0b0110111);
 913 
 914 #undef INSN
 915 
 916   // Conditional branch (immediate)
 917   enum Condition
 918     {EQ, NE, HS, CS=HS, LO, CC=LO, MI, PL, VS, VC, HI, LS, GE, LT, GT, LE, AL, NV};
 919 
 920   void br(Condition  cond, address dest) {
 921     long offset = (dest - pc()) >> 2;
 922     starti;
 923     f(0b0101010, 31, 25), f(0, 24), sf(offset, 23, 5), f(0, 4), f(cond, 3, 0);
 924   }
 925 
 926 #define INSN(NAME, cond)                        \
 927   void NAME(address dest) {                     \
 928     br(cond, dest);                             \
 929   }
 930 
 931   INSN(beq, EQ);
 932   INSN(bne, NE);
 933   INSN(bhs, HS);
 934   INSN(bcs, CS);
 935   INSN(blo, LO);
 936   INSN(bcc, CC);
 937   INSN(bmi, MI);
 938   INSN(bpl, PL);
 939   INSN(bvs, VS);
 940   INSN(bvc, VC);
 941   INSN(bhi, HI);
 942   INSN(bls, LS);
 943   INSN(bge, GE);
 944   INSN(blt, LT);
 945   INSN(bgt, GT);
 946   INSN(ble, LE);
 947   INSN(bal, AL);
 948   INSN(bnv, NV);
 949 
 950   void br(Condition cc, Label &L);
 951 
 952 #undef INSN
 953 
 954   // Exception generation
 955   void generate_exception(int opc, int op2, int LL, unsigned imm) {
 956     starti;
 957     f(0b11010100, 31, 24);
 958     f(opc, 23, 21), f(imm, 20, 5), f(op2, 4, 2), f(LL, 1, 0);
 959   }
 960 
 961 #define INSN(NAME, opc, op2, LL)                \
 962   void NAME(unsigned imm) {                     \
 963     generate_exception(opc, op2, LL, imm);      \
 964   }
 965 
 966   INSN(svc, 0b000, 0, 0b01);
 967   INSN(hvc, 0b000, 0, 0b10);
 968   INSN(smc, 0b000, 0, 0b11);
 969   INSN(brk, 0b001, 0, 0b00);
 970   INSN(hlt, 0b010, 0, 0b00);
 971   INSN(dpcs1, 0b101, 0, 0b01);
 972   INSN(dpcs2, 0b101, 0, 0b10);
 973   INSN(dpcs3, 0b101, 0, 0b11);
 974 
 975 #undef INSN
 976 
 977   // System
 978   void system(int op0, int op1, int CRn, int CRm, int op2,
 979               Register rt = (Register)0b11111)
 980   {
 981     starti;
 982     f(0b11010101000, 31, 21);
 983     f(op0, 20, 19);
 984     f(op1, 18, 16);
 985     f(CRn, 15, 12);
 986     f(CRm, 11, 8);
 987     f(op2, 7, 5);
 988     rf(rt, 0);
 989   }
 990 
 991   void hint(int imm) {
 992     system(0b00, 0b011, 0b0010, imm, 0b000);
 993   }
 994 
 995   void nop() {
 996     hint(0);
 997   }
 998   // we only provide mrs and msr for the special purpose system
 999   // registers where op1 (instr[20:19]) == 11 and, (currently) only
1000   // use it for FPSR n.b msr has L (instr[21]) == 0 mrs has L == 1
1001 
1002   void msr(int op1, int CRn, int CRm, int op2, Register rt) {
1003     starti;
1004     f(0b1101010100011, 31, 19);
1005     f(op1, 18, 16);
1006     f(CRn, 15, 12);
1007     f(CRm, 11, 8);
1008     f(op2, 7, 5);
1009     // writing zr is ok
1010     zrf(rt, 0);
1011   }
1012 
1013   void mrs(int op1, int CRn, int CRm, int op2, Register rt) {
1014     starti;
1015     f(0b1101010100111, 31, 19);
1016     f(op1, 18, 16);
1017     f(CRn, 15, 12);
1018     f(CRm, 11, 8);
1019     f(op2, 7, 5);
1020     // reading to zr is a mistake
1021     rf(rt, 0);
1022   }
1023 
1024   enum barrier {OSHLD = 0b0001, OSHST, OSH, NSHLD=0b0101, NSHST, NSH,
1025                 ISHLD = 0b1001, ISHST, ISH, LD=0b1101, ST, SY};
1026 
1027   void dsb(barrier imm) {
1028     system(0b00, 0b011, 0b00011, imm, 0b100);
1029   }
1030 
1031   void dmb(barrier imm) {
1032     system(0b00, 0b011, 0b00011, imm, 0b101);
1033   }
1034 
1035   void isb() {
1036     system(0b00, 0b011, 0b00011, SY, 0b110);
1037   }
1038 
1039   void dc(Register Rt) {
1040     system(0b01, 0b011, 0b0111, 0b1011, 0b001, Rt);
1041   }
1042 
1043   void ic(Register Rt) {
1044     system(0b01, 0b011, 0b0111, 0b0101, 0b001, Rt);
1045   }
1046 
1047   // A more convenient access to dmb for our purposes
1048   enum Membar_mask_bits {
1049     // We can use ISH for a barrier because the ARM ARM says "This
1050     // architecture assumes that all Processing Elements that use the
1051     // same operating system or hypervisor are in the same Inner
1052     // Shareable shareability domain."
1053     StoreStore = ISHST,
1054     LoadStore  = ISHLD,
1055     LoadLoad   = ISHLD,
1056     StoreLoad  = ISH,
1057     AnyAny     = ISH
1058   };
1059 
1060   void membar(Membar_mask_bits order_constraint) {
1061     dmb(Assembler::barrier(order_constraint));
1062   }
1063 
1064   // Unconditional branch (register)
1065   void branch_reg(Register R, int opc) {
1066     starti;
1067     f(0b1101011, 31, 25);
1068     f(opc, 24, 21);
1069     f(0b11111000000, 20, 10);
1070     rf(R, 5);
1071     f(0b00000, 4, 0);
1072   }
1073 
1074 #define INSN(NAME, opc)                         \
1075   void NAME(Register R) {                       \
1076     branch_reg(R, opc);                         \
1077   }
1078 
1079   INSN(br, 0b0000);
1080   INSN(blr, 0b0001);
1081   INSN(ret, 0b0010);
1082 
1083   void ret(void *p); // This forces a compile-time error for ret(0)
1084 
1085 #undef INSN
1086 
1087 #define INSN(NAME, opc)                         \
1088   void NAME() {                 \
1089     branch_reg((Register)0b11111, opc);         \
1090   }
1091 
1092   INSN(eret, 0b0100);
1093   INSN(drps, 0b0101);
1094 
1095 #undef INSN
1096 
1097   // Load/store exclusive
1098   enum operand_size { byte, halfword, word, xword };
1099 
1100   void load_store_exclusive(Register Rs, Register Rt1, Register Rt2,
1101     Register Rn, enum operand_size sz, int op, int o0) {
1102     starti;
1103     f(sz, 31, 30), f(0b001000, 29, 24), f(op, 23, 21);
1104     rf(Rs, 16), f(o0, 15), rf(Rt2, 10), rf(Rn, 5), rf(Rt1, 0);
1105   }
1106 
1107 #define INSN4(NAME, sz, op, o0) /* Four registers */                    \
1108   void NAME(Register Rs, Register Rt1, Register Rt2, Register Rn) {     \
1109     load_store_exclusive(Rs, Rt1, Rt2, Rn, sz, op, o0);                 \
1110   }
1111 
1112 #define INSN3(NAME, sz, op, o0) /* Three registers */                   \
1113   void NAME(Register Rs, Register Rt, Register Rn) {                    \
1114     load_store_exclusive(Rs, Rt, (Register)0b11111, Rn, sz, op, o0);    \
1115   }
1116 
1117 #define INSN2(NAME, sz, op, o0) /* Two registers */                     \
1118   void NAME(Register Rt, Register Rn) {                                 \
1119     load_store_exclusive((Register)0b11111, Rt, (Register)0b11111,      \
1120                          Rn, sz, op, o0);                               \
1121   }
1122 
1123 #define INSN_FOO(NAME, sz, op, o0) /* Three registers, encoded differently */ \
1124   void NAME(Register Rt1, Register Rt2, Register Rn) {                  \
1125     load_store_exclusive((Register)0b11111, Rt1, Rt2, Rn, sz, op, o0);  \
1126   }
1127 
1128   // bytes
1129   INSN3(stxrb, byte, 0b000, 0);
1130   INSN3(stlxrb, byte, 0b000, 1);
1131   INSN2(ldxrb, byte, 0b010, 0);
1132   INSN2(ldaxrb, byte, 0b010, 1);
1133   INSN2(stlrb, byte, 0b100, 1);
1134   INSN2(ldarb, byte, 0b110, 1);
1135 
1136   // halfwords
1137   INSN3(stxrh, halfword, 0b000, 0);
1138   INSN3(stlxrh, halfword, 0b000, 1);
1139   INSN2(ldxrh, halfword, 0b010, 0);
1140   INSN2(ldaxrh, halfword, 0b010, 1);
1141   INSN2(stlrh, halfword, 0b100, 1);
1142   INSN2(ldarh, halfword, 0b110, 1);
1143 
1144   // words
1145   INSN3(stxrw, word, 0b000, 0);
1146   INSN3(stlxrw, word, 0b000, 1);
1147   INSN4(stxpw, word, 0b001, 0);
1148   INSN4(stlxpw, word, 0b001, 1);
1149   INSN2(ldxrw, word, 0b010, 0);
1150   INSN2(ldaxrw, word, 0b010, 1);
1151   INSN_FOO(ldxpw, word, 0b011, 0);
1152   INSN_FOO(ldaxpw, word, 0b011, 1);
1153   INSN2(stlrw, word, 0b100, 1);
1154   INSN2(ldarw, word, 0b110, 1);
1155 
1156   // xwords
1157   INSN3(stxr, xword, 0b000, 0);
1158   INSN3(stlxr, xword, 0b000, 1);
1159   INSN4(stxp, xword, 0b001, 0);
1160   INSN4(stlxp, xword, 0b001, 1);
1161   INSN2(ldxr, xword, 0b010, 0);
1162   INSN2(ldaxr, xword, 0b010, 1);
1163   INSN_FOO(ldxp, xword, 0b011, 0);
1164   INSN_FOO(ldaxp, xword, 0b011, 1);
1165   INSN2(stlr, xword, 0b100, 1);
1166   INSN2(ldar, xword, 0b110, 1);
1167 
1168 #undef INSN2
1169 #undef INSN3
1170 #undef INSN4
1171 #undef INSN_FOO
1172 
1173   // Load register (literal)
1174 #define INSN(NAME, opc, V)                                              \
1175   void NAME(Register Rt, address dest) {                                \
1176     long offset = (dest - pc()) >> 2;                                   \
1177     starti;                                                             \
1178     f(opc, 31, 30), f(0b011, 29, 27), f(V, 26), f(0b00, 25, 24),        \
1179       sf(offset, 23, 5);                                                \
1180     rf(Rt, 0);                                                          \
1181   }                                                                     \
1182   void NAME(Register Rt, address dest, relocInfo::relocType rtype) {    \
1183     InstructionMark im(this);                                           \
1184     guarantee(rtype == relocInfo::internal_word_type,                   \
1185               "only internal_word_type relocs make sense here");        \
1186     code_section()->relocate(inst_mark(), InternalAddress(dest).rspec()); \
1187     NAME(Rt, dest);                                                     \
1188   }                                                                     \
1189   void NAME(Register Rt, Label &L) {                                    \
1190     wrap_label(Rt, L, &Assembler::NAME);                                \
1191   }
1192 
1193   INSN(ldrw, 0b00, 0);
1194   INSN(ldr, 0b01, 0);
1195   INSN(ldrsw, 0b10, 0);
1196 
1197 #undef INSN
1198 
1199 #define INSN(NAME, opc, V)                                              \
1200   void NAME(FloatRegister Rt, address dest) {                           \
1201     long offset = (dest - pc()) >> 2;                                   \
1202     starti;                                                             \
1203     f(opc, 31, 30), f(0b011, 29, 27), f(V, 26), f(0b00, 25, 24),        \
1204       sf(offset, 23, 5);                                                \
1205     rf((Register)Rt, 0);                                                \
1206   }
1207 
1208   INSN(ldrs, 0b00, 1);
1209   INSN(ldrd, 0b01, 1);
1210   INSN(ldrq, 0x10, 1);
1211 
1212 #undef INSN
1213 
1214 #define INSN(NAME, opc, V)                                              \
1215   void NAME(address dest, prfop op = PLDL1KEEP) {                       \
1216     long offset = (dest - pc()) >> 2;                                   \
1217     starti;                                                             \
1218     f(opc, 31, 30), f(0b011, 29, 27), f(V, 26), f(0b00, 25, 24),        \
1219       sf(offset, 23, 5);                                                \
1220     f(op, 4, 0);                                                        \
1221   }                                                                     \
1222   void NAME(Label &L, prfop op = PLDL1KEEP) {                           \
1223     wrap_label(L, op, &Assembler::NAME);                                \
1224   }
1225 
1226   INSN(prfm, 0b11, 0);
1227 
1228 #undef INSN
1229 
1230   // Load/store
1231   void ld_st1(int opc, int p1, int V, int L,
1232               Register Rt1, Register Rt2, Address adr, bool no_allocate) {
1233     starti;
1234     f(opc, 31, 30), f(p1, 29, 27), f(V, 26), f(L, 22);
1235     zrf(Rt2, 10), zrf(Rt1, 0);
1236     if (no_allocate) {
1237       adr.encode_nontemporal_pair(current);
1238     } else {
1239       adr.encode_pair(current);
1240     }
1241   }
1242 
1243   // Load/store register pair (offset)
1244 #define INSN(NAME, size, p1, V, L, no_allocate)         \
1245   void NAME(Register Rt1, Register Rt2, Address adr) {  \
1246     ld_st1(size, p1, V, L, Rt1, Rt2, adr, no_allocate); \
1247    }
1248 
1249   INSN(stpw, 0b00, 0b101, 0, 0, false);
1250   INSN(ldpw, 0b00, 0b101, 0, 1, false);
1251   INSN(ldpsw, 0b01, 0b101, 0, 1, false);
1252   INSN(stp, 0b10, 0b101, 0, 0, false);
1253   INSN(ldp, 0b10, 0b101, 0, 1, false);
1254 
1255   // Load/store no-allocate pair (offset)
1256   INSN(stnpw, 0b00, 0b101, 0, 0, true);
1257   INSN(ldnpw, 0b00, 0b101, 0, 1, true);
1258   INSN(stnp, 0b10, 0b101, 0, 0, true);
1259   INSN(ldnp, 0b10, 0b101, 0, 1, true);
1260 
1261 #undef INSN
1262 
1263 #define INSN(NAME, size, p1, V, L, no_allocate)                         \
1264   void NAME(FloatRegister Rt1, FloatRegister Rt2, Address adr) {        \
1265     ld_st1(size, p1, V, L, (Register)Rt1, (Register)Rt2, adr, no_allocate); \
1266    }
1267 
1268   INSN(stps, 0b00, 0b101, 1, 0, false);
1269   INSN(ldps, 0b00, 0b101, 1, 1, false);
1270   INSN(stpd, 0b01, 0b101, 1, 0, false);
1271   INSN(ldpd, 0b01, 0b101, 1, 1, false);
1272   INSN(stpq, 0b10, 0b101, 1, 0, false);
1273   INSN(ldpq, 0b10, 0b101, 1, 1, false);
1274 
1275 #undef INSN
1276 
1277   // Load/store register (all modes)
1278   void ld_st2(Register Rt, const Address &adr, int size, int op, int V = 0) {
1279     starti;
1280 
1281     f(V, 26); // general reg?
1282     zrf(Rt, 0);
1283 
1284     // Encoding for literal loads is done here (rather than pushed
1285     // down into Address::encode) because the encoding of this
1286     // instruction is too different from all of the other forms to
1287     // make it worth sharing.
1288     if (adr.getMode() == Address::literal) {
1289       assert(size == 0b10 || size == 0b11, "bad operand size in ldr");
1290       assert(op == 0b01, "literal form can only be used with loads");
1291       f(size & 0b01, 31, 30), f(0b011, 29, 27), f(0b00, 25, 24);
1292       long offset = (adr.target() - pc()) >> 2;
1293       sf(offset, 23, 5);
1294       code_section()->relocate(pc(), adr.rspec());
1295       return;
1296     }
1297 
1298     f(size, 31, 30);
1299     f(op, 23, 22); // str
1300     adr.encode(current);
1301   }
1302 
1303 #define INSN(NAME, size, op)                            \
1304   void NAME(Register Rt, const Address &adr) {          \
1305     ld_st2(Rt, adr, size, op);                          \
1306   }                                                     \
1307 
1308   INSN(str, 0b11, 0b00);
1309   INSN(strw, 0b10, 0b00);
1310   INSN(strb, 0b00, 0b00);
1311   INSN(strh, 0b01, 0b00);
1312 
1313   INSN(ldr, 0b11, 0b01);
1314   INSN(ldrw, 0b10, 0b01);
1315   INSN(ldrb, 0b00, 0b01);
1316   INSN(ldrh, 0b01, 0b01);
1317 
1318   INSN(ldrsb, 0b00, 0b10);
1319   INSN(ldrsbw, 0b00, 0b11);
1320   INSN(ldrsh, 0b01, 0b10);
1321   INSN(ldrshw, 0b01, 0b11);
1322   INSN(ldrsw, 0b10, 0b10);
1323 
1324 #undef INSN
1325 
1326 #define INSN(NAME, size, op)                                    \
1327   void NAME(const Address &adr, prfop pfop = PLDL1KEEP) {       \
1328     ld_st2((Register)pfop, adr, size, op);                      \
1329   }
1330 
1331   INSN(prfm, 0b11, 0b10); // FIXME: PRFM should not be used with
1332                           // writeback modes, but the assembler
1333                           // doesn't enfore that.
1334 
1335 #undef INSN
1336 
1337 #define INSN(NAME, size, op)                            \
1338   void NAME(FloatRegister Rt, const Address &adr) {     \
1339     ld_st2((Register)Rt, adr, size, op, 1);             \
1340   }
1341 
1342   INSN(strd, 0b11, 0b00);
1343   INSN(strs, 0b10, 0b00);
1344   INSN(ldrd, 0b11, 0b01);
1345   INSN(ldrs, 0b10, 0b01);
1346   INSN(strq, 0b00, 0b10);
1347   INSN(ldrq, 0x00, 0b11);
1348 
1349 #undef INSN
1350 
1351   enum shift_kind { LSL, LSR, ASR, ROR };
1352 
1353   void op_shifted_reg(unsigned decode,
1354                       enum shift_kind kind, unsigned shift,
1355                       unsigned size, unsigned op) {
1356     f(size, 31);
1357     f(op, 30, 29);
1358     f(decode, 28, 24);
1359     f(shift, 15, 10);
1360     f(kind, 23, 22);
1361   }
1362 
1363   // Logical (shifted register)
1364 #define INSN(NAME, size, op, N)                                 \
1365   void NAME(Register Rd, Register Rn, Register Rm,              \
1366             enum shift_kind kind = LSL, unsigned shift = 0) {   \
1367     starti;                                                     \
1368     f(N, 21);                                                   \
1369     zrf(Rm, 16), zrf(Rn, 5), zrf(Rd, 0);                        \
1370     op_shifted_reg(0b01010, kind, shift, size, op);             \
1371   }
1372 
1373   INSN(andr, 1, 0b00, 0);
1374   INSN(orr, 1, 0b01, 0);
1375   INSN(eor, 1, 0b10, 0);
1376   INSN(ands, 1, 0b11, 0);
1377   INSN(andw, 0, 0b00, 0);
1378   INSN(orrw, 0, 0b01, 0);
1379   INSN(eorw, 0, 0b10, 0);
1380   INSN(andsw, 0, 0b11, 0);
1381 
1382   INSN(bic, 1, 0b00, 1);
1383   INSN(orn, 1, 0b01, 1);
1384   INSN(eon, 1, 0b10, 1);
1385   INSN(bics, 1, 0b11, 1);
1386   INSN(bicw, 0, 0b00, 1);
1387   INSN(ornw, 0, 0b01, 1);
1388   INSN(eonw, 0, 0b10, 1);
1389   INSN(bicsw, 0, 0b11, 1);
1390 
1391 #undef INSN
1392 
1393   // Add/subtract (shifted register)
1394 #define INSN(NAME, size, op)                            \
1395   void NAME(Register Rd, Register Rn, Register Rm,      \
1396             enum shift_kind kind, unsigned shift = 0) { \
1397     starti;                                             \
1398     f(0, 21);                                           \
1399     assert_cond(kind != ROR);                           \
1400     zrf(Rd, 0), zrf(Rn, 5), zrf(Rm, 16);                \
1401     op_shifted_reg(0b01011, kind, shift, size, op);     \
1402   }
1403 
1404   INSN(add, 1, 0b000);
1405   INSN(sub, 1, 0b10);
1406   INSN(addw, 0, 0b000);
1407   INSN(subw, 0, 0b10);
1408 
1409   INSN(adds, 1, 0b001);
1410   INSN(subs, 1, 0b11);
1411   INSN(addsw, 0, 0b001);
1412   INSN(subsw, 0, 0b11);
1413 
1414 #undef INSN
1415 
1416   // Add/subtract (extended register)
1417 #define INSN(NAME, op)                                                  \
1418   void NAME(Register Rd, Register Rn, Register Rm,                      \
1419            ext::operation option, int amount = 0) {                     \
1420     starti;                                                             \
1421     zrf(Rm, 16), srf(Rn, 5), srf(Rd, 0);                                \
1422     add_sub_extended_reg(op, 0b01011, Rd, Rn, Rm, 0b00, option, amount); \
1423   }
1424 
1425   void add_sub_extended_reg(unsigned op, unsigned decode,
1426     Register Rd, Register Rn, Register Rm,
1427     unsigned opt, ext::operation option, unsigned imm) {
1428     guarantee(imm <= 4, "shift amount must be < 4");
1429     f(op, 31, 29), f(decode, 28, 24), f(opt, 23, 22), f(1, 21);
1430     f(option, 15, 13), f(imm, 12, 10);
1431   }
1432 
1433   INSN(addw, 0b000);
1434   INSN(subw, 0b010);
1435   INSN(add, 0b100);
1436   INSN(sub, 0b110);
1437 
1438 #undef INSN
1439 
1440 #define INSN(NAME, op)                                                  \
1441   void NAME(Register Rd, Register Rn, Register Rm,                      \
1442            ext::operation option, int amount = 0) {                     \
1443     starti;                                                             \
1444     zrf(Rm, 16), srf(Rn, 5), zrf(Rd, 0);                                \
1445     add_sub_extended_reg(op, 0b01011, Rd, Rn, Rm, 0b00, option, amount); \
1446   }
1447 
1448   INSN(addsw, 0b001);
1449   INSN(subsw, 0b011);
1450   INSN(adds, 0b101);
1451   INSN(subs, 0b111);
1452 
1453 #undef INSN
1454 
1455   // Aliases for short forms of add and sub
1456 #define INSN(NAME)                                      \
1457   void NAME(Register Rd, Register Rn, Register Rm) {    \
1458     if (Rd == sp || Rn == sp)                           \
1459       NAME(Rd, Rn, Rm, ext::uxtx);                      \
1460     else                                                \
1461       NAME(Rd, Rn, Rm, LSL);                            \
1462   }
1463 
1464   INSN(addw);
1465   INSN(subw);
1466   INSN(add);
1467   INSN(sub);
1468 
1469   INSN(addsw);
1470   INSN(subsw);
1471   INSN(adds);
1472   INSN(subs);
1473 
1474 #undef INSN
1475 
1476   // Add/subtract (with carry)
1477   void add_sub_carry(unsigned op, Register Rd, Register Rn, Register Rm) {
1478     starti;
1479     f(op, 31, 29);
1480     f(0b11010000, 28, 21);
1481     f(0b000000, 15, 10);
1482     zrf(Rm, 16), zrf(Rn, 5), zrf(Rd, 0);
1483   }
1484 
1485   #define INSN(NAME, op)                                \
1486     void NAME(Register Rd, Register Rn, Register Rm) {  \
1487       add_sub_carry(op, Rd, Rn, Rm);                    \
1488     }
1489 
1490   INSN(adcw, 0b000);
1491   INSN(adcsw, 0b001);
1492   INSN(sbcw, 0b010);
1493   INSN(sbcsw, 0b011);
1494   INSN(adc, 0b100);
1495   INSN(adcs, 0b101);
1496   INSN(sbc,0b110);
1497   INSN(sbcs, 0b111);
1498 
1499 #undef INSN
1500 
1501   // Conditional compare (both kinds)
1502   void conditional_compare(unsigned op, int o2, int o3,
1503                            Register Rn, unsigned imm5, unsigned nzcv,
1504                            unsigned cond) {
1505     f(op, 31, 29);
1506     f(0b11010010, 28, 21);
1507     f(cond, 15, 12);
1508     f(o2, 10);
1509     f(o3, 4);
1510     f(nzcv, 3, 0);
1511     f(imm5, 20, 16), rf(Rn, 5);
1512   }
1513 
1514 #define INSN(NAME, op)                                                  \
1515   void NAME(Register Rn, Register Rm, int imm, Condition cond) {        \
1516     starti;                                                             \
1517     f(0, 11);                                                           \
1518     conditional_compare(op, 0, 0, Rn, (uintptr_t)Rm, imm, cond);        \
1519   }                                                                     \
1520                                                                         \
1521   void NAME(Register Rn, int imm5, int imm, Condition cond) {   \
1522     starti;                                                             \
1523     f(1, 11);                                                           \
1524     conditional_compare(op, 0, 0, Rn, imm5, imm, cond);                 \
1525   }
1526 
1527   INSN(ccmnw, 0b001);
1528   INSN(ccmpw, 0b011);
1529   INSN(ccmn, 0b101);
1530   INSN(ccmp, 0b111);
1531 
1532 #undef INSN
1533 
1534   // Conditional select
1535   void conditional_select(unsigned op, unsigned op2,
1536                           Register Rd, Register Rn, Register Rm,
1537                           unsigned cond) {
1538     starti;
1539     f(op, 31, 29);
1540     f(0b11010100, 28, 21);
1541     f(cond, 15, 12);
1542     f(op2, 11, 10);
1543     zrf(Rm, 16), zrf(Rn, 5), rf(Rd, 0);
1544   }
1545 
1546 #define INSN(NAME, op, op2)                                             \
1547   void NAME(Register Rd, Register Rn, Register Rm, Condition cond) { \
1548     conditional_select(op, op2, Rd, Rn, Rm, cond);                      \
1549   }
1550 
1551   INSN(cselw, 0b000, 0b00);
1552   INSN(csincw, 0b000, 0b01);
1553   INSN(csinvw, 0b010, 0b00);
1554   INSN(csnegw, 0b010, 0b01);
1555   INSN(csel, 0b100, 0b00);
1556   INSN(csinc, 0b100, 0b01);
1557   INSN(csinv, 0b110, 0b00);
1558   INSN(csneg, 0b110, 0b01);
1559 
1560 #undef INSN
1561 
1562   // Data processing
1563   void data_processing(unsigned op29, unsigned opcode,
1564                        Register Rd, Register Rn) {
1565     f(op29, 31, 29), f(0b11010110, 28, 21);
1566     f(opcode, 15, 10);
1567     rf(Rn, 5), rf(Rd, 0);
1568   }
1569 
1570   // (1 source)
1571 #define INSN(NAME, op29, opcode2, opcode)       \
1572   void NAME(Register Rd, Register Rn) {         \
1573     starti;                                     \
1574     f(opcode2, 20, 16);                         \
1575     data_processing(op29, opcode, Rd, Rn);      \
1576   }
1577 
1578   INSN(rbitw,  0b010, 0b00000, 0b00000);
1579   INSN(rev16w, 0b010, 0b00000, 0b00001);
1580   INSN(revw,   0b010, 0b00000, 0b00010);
1581   INSN(clzw,   0b010, 0b00000, 0b00100);
1582   INSN(clsw,   0b010, 0b00000, 0b00101);
1583 
1584   INSN(rbit,   0b110, 0b00000, 0b00000);
1585   INSN(rev16,  0b110, 0b00000, 0b00001);
1586   INSN(rev32,  0b110, 0b00000, 0b00010);
1587   INSN(rev,    0b110, 0b00000, 0b00011);
1588   INSN(clz,    0b110, 0b00000, 0b00100);
1589   INSN(cls,    0b110, 0b00000, 0b00101);
1590 
1591 #undef INSN
1592 
1593   // (2 sources)
1594 #define INSN(NAME, op29, opcode)                        \
1595   void NAME(Register Rd, Register Rn, Register Rm) {    \
1596     starti;                                             \
1597     rf(Rm, 16);                                         \
1598     data_processing(op29, opcode, Rd, Rn);              \
1599   }
1600 
1601   INSN(udivw, 0b000, 0b000010);
1602   INSN(sdivw, 0b000, 0b000011);
1603   INSN(lslvw, 0b000, 0b001000);
1604   INSN(lsrvw, 0b000, 0b001001);
1605   INSN(asrvw, 0b000, 0b001010);
1606   INSN(rorvw, 0b000, 0b001011);
1607 
1608   INSN(udiv, 0b100, 0b000010);
1609   INSN(sdiv, 0b100, 0b000011);
1610   INSN(lslv, 0b100, 0b001000);
1611   INSN(lsrv, 0b100, 0b001001);
1612   INSN(asrv, 0b100, 0b001010);
1613   INSN(rorv, 0b100, 0b001011);
1614 
1615 #undef INSN
1616 
1617   // (3 sources)
1618   void data_processing(unsigned op54, unsigned op31, unsigned o0,
1619                        Register Rd, Register Rn, Register Rm,
1620                        Register Ra) {
1621     starti;
1622     f(op54, 31, 29), f(0b11011, 28, 24);
1623     f(op31, 23, 21), f(o0, 15);
1624     zrf(Rm, 16), zrf(Ra, 10), zrf(Rn, 5), zrf(Rd, 0);
1625   }
1626 
1627 #define INSN(NAME, op54, op31, o0)                                      \
1628   void NAME(Register Rd, Register Rn, Register Rm, Register Ra) {       \
1629     data_processing(op54, op31, o0, Rd, Rn, Rm, Ra);                    \
1630   }
1631 
1632   INSN(maddw, 0b000, 0b000, 0);
1633   INSN(msubw, 0b000, 0b000, 1);
1634   INSN(madd, 0b100, 0b000, 0);
1635   INSN(msub, 0b100, 0b000, 1);
1636   INSN(smaddl, 0b100, 0b001, 0);
1637   INSN(smsubl, 0b100, 0b001, 1);
1638   INSN(umaddl, 0b100, 0b101, 0);
1639   INSN(umsubl, 0b100, 0b101, 1);
1640 
1641 #undef INSN
1642 
1643 #define INSN(NAME, op54, op31, o0)                      \
1644   void NAME(Register Rd, Register Rn, Register Rm) {    \
1645     data_processing(op54, op31, o0, Rd, Rn, Rm, (Register)31);  \
1646   }
1647 
1648   INSN(smulh, 0b100, 0b010, 0);
1649   INSN(umulh, 0b100, 0b110, 0);
1650 
1651 #undef INSN
1652 
1653   // Floating-point data-processing (1 source)
1654   void data_processing(unsigned op31, unsigned type, unsigned opcode,
1655                        FloatRegister Vd, FloatRegister Vn) {
1656     starti;
1657     f(op31, 31, 29);
1658     f(0b11110, 28, 24);
1659     f(type, 23, 22), f(1, 21), f(opcode, 20, 15), f(0b10000, 14, 10);
1660     rf(Vn, 5), rf(Vd, 0);
1661   }
1662 
1663 #define INSN(NAME, op31, type, opcode)                  \
1664   void NAME(FloatRegister Vd, FloatRegister Vn) {       \
1665     data_processing(op31, type, opcode, Vd, Vn);        \
1666   }
1667 
1668 private:
1669   INSN(i_fmovs, 0b000, 0b00, 0b000000);
1670 public:
1671   INSN(fabss, 0b000, 0b00, 0b000001);
1672   INSN(fnegs, 0b000, 0b00, 0b000010);
1673   INSN(fsqrts, 0b000, 0b00, 0b000011);
1674   INSN(fcvts, 0b000, 0b00, 0b000101);   // Single-precision to double-precision
1675 
1676 private:
1677   INSN(i_fmovd, 0b000, 0b01, 0b000000);
1678 public:
1679   INSN(fabsd, 0b000, 0b01, 0b000001);
1680   INSN(fnegd, 0b000, 0b01, 0b000010);
1681   INSN(fsqrtd, 0b000, 0b01, 0b000011);
1682   INSN(fcvtd, 0b000, 0b01, 0b000100);   // Double-precision to single-precision
1683 
1684   void fmovd(FloatRegister Vd, FloatRegister Vn) {
1685     assert(Vd != Vn, "should be");
1686     i_fmovd(Vd, Vn);
1687   }
1688 
1689   void fmovs(FloatRegister Vd, FloatRegister Vn) {
1690     assert(Vd != Vn, "should be");
1691     i_fmovs(Vd, Vn);
1692   }
1693 
1694 #undef INSN
1695 
1696   // Floating-point data-processing (2 source)
1697   void data_processing(unsigned op31, unsigned type, unsigned opcode,
1698                        FloatRegister Vd, FloatRegister Vn, FloatRegister Vm) {
1699     starti;
1700     f(op31, 31, 29);
1701     f(0b11110, 28, 24);
1702     f(type, 23, 22), f(1, 21), f(opcode, 15, 12), f(0b10, 11, 10);
1703     rf(Vm, 16), rf(Vn, 5), rf(Vd, 0);
1704   }
1705 
1706 #define INSN(NAME, op31, type, opcode)                  \
1707   void NAME(FloatRegister Vd, FloatRegister Vn, FloatRegister Vm) {     \
1708     data_processing(op31, type, opcode, Vd, Vn, Vm);    \
1709   }
1710 
1711   INSN(fmuls, 0b000, 0b00, 0b0000);
1712   INSN(fdivs, 0b000, 0b00, 0b0001);
1713   INSN(fadds, 0b000, 0b00, 0b0010);
1714   INSN(fsubs, 0b000, 0b00, 0b0011);
1715   INSN(fnmuls, 0b000, 0b00, 0b1000);
1716 
1717   INSN(fmuld, 0b000, 0b01, 0b0000);
1718   INSN(fdivd, 0b000, 0b01, 0b0001);
1719   INSN(faddd, 0b000, 0b01, 0b0010);
1720   INSN(fsubd, 0b000, 0b01, 0b0011);
1721   INSN(fnmuld, 0b000, 0b01, 0b1000);
1722 
1723 #undef INSN
1724 
1725    // Floating-point data-processing (3 source)
1726   void data_processing(unsigned op31, unsigned type, unsigned o1, unsigned o0,
1727                        FloatRegister Vd, FloatRegister Vn, FloatRegister Vm,
1728                        FloatRegister Va) {
1729     starti;
1730     f(op31, 31, 29);
1731     f(0b11111, 28, 24);
1732     f(type, 23, 22), f(o1, 21), f(o0, 15);
1733     rf(Vm, 16), rf(Va, 10), rf(Vn, 5), rf(Vd, 0);
1734   }
1735 
1736 #define INSN(NAME, op31, type, o1, o0)                                  \
1737   void NAME(FloatRegister Vd, FloatRegister Vn, FloatRegister Vm,       \
1738             FloatRegister Va) {                                         \
1739     data_processing(op31, type, o1, o0, Vd, Vn, Vm, Va);                \
1740   }
1741 
1742   INSN(fmadds, 0b000, 0b00, 0, 0);
1743   INSN(fmsubs, 0b000, 0b00, 0, 1);
1744   INSN(fnmadds, 0b000, 0b00, 1, 0);
1745   INSN(fnmsubs, 0b000, 0b00, 1, 1);
1746 
1747   INSN(fmaddd, 0b000, 0b01, 0, 0);
1748   INSN(fmsubd, 0b000, 0b01, 0, 1);
1749   INSN(fnmaddd, 0b000, 0b01, 1, 0);
1750   INSN(fnmsub, 0b000, 0b01, 1, 1);
1751 
1752 #undef INSN
1753 
1754    // Floating-point conditional select
1755   void fp_conditional_select(unsigned op31, unsigned type,
1756                              unsigned op1, unsigned op2,
1757                              Condition cond, FloatRegister Vd,
1758                              FloatRegister Vn, FloatRegister Vm) {
1759     starti;
1760     f(op31, 31, 29);
1761     f(0b11110, 28, 24);
1762     f(type, 23, 22);
1763     f(op1, 21, 21);
1764     f(op2, 11, 10);
1765     f(cond, 15, 12);
1766     rf(Vm, 16), rf(Vn, 5), rf(Vd, 0);
1767   }
1768 
1769 #define INSN(NAME, op31, type, op1, op2)                                \
1770   void NAME(FloatRegister Vd, FloatRegister Vn,                         \
1771             FloatRegister Vm, Condition cond) {                         \
1772     fp_conditional_select(op31, type, op1, op2, cond, Vd, Vn, Vm);      \
1773   }
1774 
1775   INSN(fcsels, 0b000, 0b00, 0b1, 0b11);
1776   INSN(fcseld, 0b000, 0b01, 0b1, 0b11);
1777 
1778 #undef INSN
1779 
1780    // Floating-point<->integer conversions
1781   void float_int_convert(unsigned op31, unsigned type,
1782                          unsigned rmode, unsigned opcode,
1783                          Register Rd, Register Rn) {
1784     starti;
1785     f(op31, 31, 29);
1786     f(0b11110, 28, 24);
1787     f(type, 23, 22), f(1, 21), f(rmode, 20, 19);
1788     f(opcode, 18, 16), f(0b000000, 15, 10);
1789     zrf(Rn, 5), zrf(Rd, 0);
1790   }
1791 
1792 #define INSN(NAME, op31, type, rmode, opcode)                           \
1793   void NAME(Register Rd, FloatRegister Vn) {                            \
1794     float_int_convert(op31, type, rmode, opcode, Rd, (Register)Vn);     \
1795   }
1796 
1797   INSN(fcvtzsw, 0b000, 0b00, 0b11, 0b000);
1798   INSN(fcvtzs,  0b100, 0b00, 0b11, 0b000);
1799   INSN(fcvtzdw, 0b000, 0b01, 0b11, 0b000);
1800   INSN(fcvtzd,  0b100, 0b01, 0b11, 0b000);
1801 
1802   INSN(fmovs, 0b000, 0b00, 0b00, 0b110);
1803   INSN(fmovd, 0b100, 0b01, 0b00, 0b110);
1804 
1805   // INSN(fmovhid, 0b100, 0b10, 0b01, 0b110);
1806 
1807 #undef INSN
1808 
1809 #define INSN(NAME, op31, type, rmode, opcode)                           \
1810   void NAME(FloatRegister Vd, Register Rn) {                            \
1811     float_int_convert(op31, type, rmode, opcode, (Register)Vd, Rn);     \
1812   }
1813 
1814   INSN(fmovs, 0b000, 0b00, 0b00, 0b111);
1815   INSN(fmovd, 0b100, 0b01, 0b00, 0b111);
1816 
1817   INSN(scvtfws, 0b000, 0b00, 0b00, 0b010);
1818   INSN(scvtfs,  0b100, 0b00, 0b00, 0b010);
1819   INSN(scvtfwd, 0b000, 0b01, 0b00, 0b010);
1820   INSN(scvtfd,  0b100, 0b01, 0b00, 0b010);
1821 
1822   // INSN(fmovhid, 0b100, 0b10, 0b01, 0b111);
1823 
1824 #undef INSN
1825 
1826   // Floating-point compare
1827   void float_compare(unsigned op31, unsigned type,
1828                      unsigned op, unsigned op2,
1829                      FloatRegister Vn, FloatRegister Vm = (FloatRegister)0) {
1830     starti;
1831     f(op31, 31, 29);
1832     f(0b11110, 28, 24);
1833     f(type, 23, 22), f(1, 21);
1834     f(op, 15, 14), f(0b1000, 13, 10), f(op2, 4, 0);
1835     rf(Vn, 5), rf(Vm, 16);
1836   }
1837 
1838 
1839 #define INSN(NAME, op31, type, op, op2)                 \
1840   void NAME(FloatRegister Vn, FloatRegister Vm) {       \
1841     float_compare(op31, type, op, op2, Vn, Vm);         \
1842   }
1843 
1844 #define INSN1(NAME, op31, type, op, op2)        \
1845   void NAME(FloatRegister Vn, double d) {       \
1846     assert_cond(d == 0.0);                      \
1847     float_compare(op31, type, op, op2, Vn);     \
1848   }
1849 
1850   INSN(fcmps, 0b000, 0b00, 0b00, 0b00000);
1851   INSN1(fcmps, 0b000, 0b00, 0b00, 0b01000);
1852   // INSN(fcmpes, 0b000, 0b00, 0b00, 0b10000);
1853   // INSN1(fcmpes, 0b000, 0b00, 0b00, 0b11000);
1854 
1855   INSN(fcmpd, 0b000,   0b01, 0b00, 0b00000);
1856   INSN1(fcmpd, 0b000,  0b01, 0b00, 0b01000);
1857   // INSN(fcmped, 0b000,  0b01, 0b00, 0b10000);
1858   // INSN1(fcmped, 0b000, 0b01, 0b00, 0b11000);
1859 
1860 #undef INSN
1861 #undef INSN1
1862 
1863   // Floating-point Move (immediate)
1864 private:
1865   unsigned pack(double value);
1866 
1867   void fmov_imm(FloatRegister Vn, double value, unsigned size) {
1868     starti;
1869     f(0b00011110, 31, 24), f(size, 23, 22), f(1, 21);
1870     f(pack(value), 20, 13), f(0b10000000, 12, 5);
1871     rf(Vn, 0);
1872   }
1873 
1874 public:
1875 
1876   void fmovs(FloatRegister Vn, double value) {
1877     if (value)
1878       fmov_imm(Vn, value, 0b00);
1879     else
1880       fmovs(Vn, zr);
1881   }
1882   void fmovd(FloatRegister Vn, double value) {
1883     if (value)
1884       fmov_imm(Vn, value, 0b01);
1885     else
1886       fmovd(Vn, zr);
1887   }
1888 
1889 /* SIMD extensions
1890  *
1891  * We just use FloatRegister in the following. They are exactly the same
1892  * as SIMD registers.
1893  */
1894  public:
1895 
1896   enum SIMD_Arrangement {
1897        T8B, T16B, T4H, T8H, T2S, T4S, T1D, T2D
1898   };
1899 
1900   enum SIMD_RegVariant {
1901        B, H, S, D, Q
1902   };
1903 
1904 #define INSN(NAME, op)                                            \
1905   void NAME(FloatRegister Rt, SIMD_RegVariant T, const Address &adr) {   \
1906     ld_st2((Register)Rt, adr, (int)T & 3, op + ((T==Q) ? 0b10:0b00), 1); \
1907   }                                                                      \
1908 
1909   INSN(ldr, 1);
1910   INSN(str, 0);
1911 
1912 #undef INSN
1913 
1914  private:
1915 
1916   void ld_st(FloatRegister Vt, SIMD_Arrangement T, Register Xn, int op1, int op2) {
1917     starti;
1918     f(0,31), f((int)T & 1, 30);
1919     f(op1, 29, 21), f(0, 20, 16), f(op2, 15, 12);
1920     f((int)T >> 1, 11, 10), rf(Xn, 5), rf(Vt, 0);
1921   }
1922   void ld_st(FloatRegister Vt, SIMD_Arrangement T, Register Xn,
1923              int imm, int op1, int op2) {
1924     starti;
1925     f(0,31), f((int)T & 1, 30);
1926     f(op1 | 0b100, 29, 21), f(0b11111, 20, 16), f(op2, 15, 12);
1927     f((int)T >> 1, 11, 10), rf(Xn, 5), rf(Vt, 0);
1928   }
1929   void ld_st(FloatRegister Vt, SIMD_Arrangement T, Register Xn,
1930              Register Xm, int op1, int op2) {
1931     starti;
1932     f(0,31), f((int)T & 1, 30);
1933     f(op1 | 0b100, 29, 21), rf(Xm, 16), f(op2, 15, 12);
1934     f((int)T >> 1, 11, 10), rf(Xn, 5), rf(Vt, 0);
1935   }
1936 
1937  void ld_st(FloatRegister Vt, SIMD_Arrangement T, Address a, int op1, int op2) {
1938    switch (a.getMode()) {
1939    case Address::base_plus_offset:
1940      guarantee(a.offset() == 0, "no offset allowed here");
1941      ld_st(Vt, T, a.base(), op1, op2);
1942      break;
1943    case Address::post:
1944      ld_st(Vt, T, a.base(), a.offset(), op1, op2);
1945      break;
1946    case Address::base_plus_offset_reg:
1947      ld_st(Vt, T, a.base(), a.index(), op1, op2);
1948      break;
1949    default:
1950      ShouldNotReachHere();
1951    }
1952  }
1953 
1954  public:
1955 
1956 #define INSN1(NAME, op1, op2)                                   \
1957   void NAME(FloatRegister Vt, SIMD_Arrangement T, const Address &a) {   \
1958    ld_st(Vt, T, a, op1, op2);                                           \
1959  }
1960 
1961 #define INSN2(NAME, op1, op2)                                           \
1962   void NAME(FloatRegister Vt, FloatRegister Vt2, SIMD_Arrangement T, const Address &a) { \
1963     assert(Vt->successor() == Vt2, "Registers must be ordered");        \
1964     ld_st(Vt, T, a, op1, op2);                                          \
1965   }
1966 
1967 #define INSN3(NAME, op1, op2)                                           \
1968   void NAME(FloatRegister Vt, FloatRegister Vt2, FloatRegister Vt3,     \
1969             SIMD_Arrangement T, const Address &a) {                     \
1970     assert(Vt->successor() == Vt2 && Vt2->successor() == Vt3,           \
1971            "Registers must be ordered");                                \
1972     ld_st(Vt, T, a, op1, op2);                                          \
1973   }
1974 
1975 #define INSN4(NAME, op1, op2)                                           \
1976   void NAME(FloatRegister Vt, FloatRegister Vt2, FloatRegister Vt3,     \
1977             FloatRegister Vt4, SIMD_Arrangement T, const Address &a) {  \
1978     assert(Vt->successor() == Vt2 && Vt2->successor() == Vt3 &&         \
1979            Vt3->successor() == Vt4, "Registers must be ordered");       \
1980     ld_st(Vt, T, a, op1, op2);                                          \
1981   }
1982 
1983   INSN1(ld1,  0b001100010, 0b0111);
1984   INSN2(ld1,  0b001100010, 0b1010);
1985   INSN3(ld1,  0b001100010, 0b0110);
1986   INSN4(ld1,  0b001100010, 0b0010);
1987 
1988   INSN2(ld2,  0b001100010, 0b1000);
1989   INSN3(ld3,  0b001100010, 0b0100);
1990   INSN4(ld4,  0b001100010, 0b0000);
1991 
1992   INSN1(st1,  0b001100000, 0b0111);
1993   INSN2(st1,  0b001100000, 0b1010);
1994   INSN3(st1,  0b001100000, 0b0110);
1995   INSN4(st1,  0b001100000, 0b0010);
1996 
1997   INSN2(st2,  0b001100000, 0b1000);
1998   INSN3(st3,  0b001100000, 0b0100);
1999   INSN4(st4,  0b001100000, 0b0000);
2000 
2001   INSN1(ld1r, 0b001101010, 0b1100);
2002   INSN2(ld2r, 0b001101011, 0b1100);
2003   INSN3(ld3r, 0b001101010, 0b1110);
2004   INSN4(ld4r, 0b001101011, 0b1110);
2005 
2006 #undef INSN1
2007 #undef INSN2
2008 #undef INSN3
2009 #undef INSN4
2010 
2011 #define INSN(NAME, opc)                                                                 \
2012   void NAME(FloatRegister Vd, SIMD_Arrangement T, FloatRegister Vn, FloatRegister Vm) { \
2013     starti;                                                                             \
2014     assert(T == T8B || T == T16B, "must be T8B or T16B");                               \
2015     f(0, 31), f((int)T & 1, 30), f(opc, 29, 21);                                        \
2016     rf(Vm, 16), f(0b000111, 15, 10), rf(Vn, 5), rf(Vd, 0);                              \
2017   }
2018 
2019   INSN(eor,  0b101110001);
2020   INSN(orr,  0b001110101);
2021   INSN(andr, 0b001110001);
2022   INSN(bic,  0b001110011);
2023   INSN(bif,  0b101110111);
2024   INSN(bit,  0b101110101);
2025   INSN(bsl,  0b101110011);
2026   INSN(orn,  0b001110111);
2027 
2028 #undef INSN
2029 
2030 #define INSN(NAME, opc, opc2)                                                                 \
2031   void NAME(FloatRegister Vd, SIMD_Arrangement T, FloatRegister Vn, FloatRegister Vm) { \
2032     starti;                                                                             \
2033     f(0, 31), f((int)T & 1, 30), f(opc, 29), f(0b01110, 28, 24);                        \
2034     f((int)T >> 1, 23, 22), f(1, 21), rf(Vm, 16), f(opc2, 15, 10);                      \
2035     rf(Vn, 5), rf(Vd, 0);                                                               \
2036   }
2037 
2038   INSN(addv, 0, 0b100001);
2039   INSN(subv, 1, 0b100001);
2040   INSN(mulv, 0, 0b100111);
2041   INSN(sshl, 0, 0b010001);
2042   INSN(ushl, 1, 0b010001);
2043 
2044 #undef INSN
2045 
2046 #define INSN(NAME, opc, opc2) \
2047   void NAME(FloatRegister Vd, SIMD_Arrangement T, FloatRegister Vn) {                   \
2048     starti;                                                                             \
2049     f(0, 31), f((int)T & 1, 30), f(opc, 29), f(0b01110, 28, 24);                        \
2050     f((int)T >> 1, 23, 22), f(opc2, 21, 10);                                            \
2051     rf(Vn, 5), rf(Vd, 0);                                                               \
2052   }
2053 
2054   INSN(absr,  0, 0b100000101110);
2055   INSN(negr,  1, 0b100000101110);
2056   INSN(notr,  1, 0b100000010110);
2057   INSN(addv,  0, 0b110001101110);
2058   INSN(cls,   0, 0b100000010010);
2059   INSN(clz,   1, 0b100000010010);
2060   INSN(cnt,   0, 0b100000010110);
2061 
2062 #undef INSN
2063 
2064 #define INSN(NAME, op0, cmode0) \
2065   void NAME(FloatRegister Vd, SIMD_Arrangement T, unsigned imm8, unsigned lsl = 0) {   \
2066     unsigned cmode = cmode0;                                                           \
2067     unsigned op = op0;                                                                 \
2068     starti;                                                                            \
2069     assert(lsl == 0 ||                                                                 \
2070            ((T == T4H || T == T8H) && lsl == 8) ||                                     \
2071            ((T == T2S || T == T4S) && ((lsl >> 3) < 4)), "invalid shift");             \
2072     cmode |= lsl >> 2;                                                                 \
2073     if (T == T4H || T == T8H) cmode |= 0b1000;                                         \
2074     if (!(T == T4H || T == T8H || T == T2S || T == T4S)) {                             \
2075       assert(op == 0 && cmode0 == 0, "must be MOVI");                                  \
2076       cmode = 0b1110;                                                                  \
2077       if (T == T1D || T == T2D) op = 1;                                                \
2078     }                                                                                  \
2079     f(0, 31), f((int)T & 1, 30), f(op, 29), f(0b0111100000, 28, 19);                   \
2080     f(imm8 >> 5, 18, 16), f(cmode, 15, 12), f(0x01, 11, 10), f(imm8 & 0b11111, 9, 5);  \
2081     rf(Vd, 0);                                                                         \
2082   }
2083 
2084   INSN(movi, 0, 0);
2085   INSN(orri, 0, 1);
2086   INSN(mvni, 1, 0);
2087   INSN(bici, 1, 1);
2088 
2089 #undef INSN
2090 
2091 #define INSN(NAME, op1, op2, op3) \
2092   void NAME(FloatRegister Vd, SIMD_Arrangement T, FloatRegister Vn, FloatRegister Vm) { \
2093     starti;                                                                             \
2094     assert(T == T2S || T == T4S || T == T2D, "invalid arrangement");                    \
2095     f(0, 31), f((int)T & 1, 30), f(op1, 29), f(0b01110, 28, 24), f(op2, 23);            \
2096     f(T==T2D ? 1:0, 22); f(1, 21), rf(Vm, 16), f(op3, 15, 10), rf(Vn, 5), rf(Vd, 0);    \
2097   }
2098 
2099   INSN(fadd, 0, 0, 0b110101);
2100   INSN(fdiv, 1, 0, 0b111111);
2101   INSN(fmul, 1, 0, 0b110111);
2102   INSN(fsub, 0, 1, 0b110101);
2103 
2104 #undef INSN
2105 
2106 #define INSN(NAME, opc)                                                                 \
2107   void NAME(FloatRegister Vd, SIMD_Arrangement T, FloatRegister Vn, FloatRegister Vm) { \
2108     starti;                                                                             \
2109     assert(T == T4S, "arrangement must be T4S");                                        \
2110     f(0b01011110000, 31, 21), rf(Vm, 16), f(opc, 15, 10), rf(Vn, 5), rf(Vd, 0);         \
2111   }
2112 
2113   INSN(sha1c,     0b000000);
2114   INSN(sha1m,     0b001000);
2115   INSN(sha1p,     0b000100);
2116   INSN(sha1su0,   0b001100);
2117   INSN(sha256h2,  0b010100);
2118   INSN(sha256h,   0b010000);
2119   INSN(sha256su1, 0b011000);
2120 
2121 #undef INSN
2122 
2123 #define INSN(NAME, opc)                                                                 \
2124   void NAME(FloatRegister Vd, SIMD_Arrangement T, FloatRegister Vn) {                   \
2125     starti;                                                                             \
2126     assert(T == T4S, "arrangement must be T4S");                                        \
2127     f(0b0101111000101000, 31, 16), f(opc, 15, 10), rf(Vn, 5), rf(Vd, 0);                \
2128   }
2129 
2130   INSN(sha1h,     0b000010);
2131   INSN(sha1su1,   0b000110);
2132   INSN(sha256su0, 0b001010);
2133 
2134 #undef INSN
2135 
2136 #define INSN(NAME, opc)                           \
2137   void NAME(FloatRegister Vd, FloatRegister Vn) { \
2138     starti;                                       \
2139     f(opc, 31, 10), rf(Vn, 5), rf(Vd, 0);         \
2140   }
2141 
2142   INSN(aese, 0b0100111000101000010010);
2143   INSN(aesd, 0b0100111000101000010110);
2144   INSN(aesmc, 0b0100111000101000011010);
2145   INSN(aesimc, 0b0100111000101000011110);
2146 
2147 #undef INSN
2148 
2149   void ins(FloatRegister Vd, SIMD_RegVariant T, FloatRegister Vn, int didx, int sidx) {
2150     starti;
2151     assert(T != Q, "invalid register variant");
2152     f(0b01101110000, 31, 21), f(((didx<<1)|1)<<(int)T, 20, 16), f(0, 15);
2153     f(sidx<<(int)T, 14, 11), f(1, 10), rf(Vn, 5), rf(Vd, 0);
2154   }
2155 
2156   void umov(Register Rd, FloatRegister Vn, SIMD_RegVariant T, int idx) {
2157     starti;
2158     f(0, 31), f(T==D ? 1:0, 30), f(0b001110000, 29, 21);
2159     f(((idx<<1)|1)<<(int)T, 20, 16), f(0b001111, 15, 10);
2160     rf(Vn, 5), rf(Rd, 0);
2161   }
2162 
2163 #define INSN(NAME, opc, opc2) \
2164   void NAME(FloatRegister Vd, SIMD_Arrangement T, FloatRegister Vn, int shift){         \
2165     starti;                                                                             \
2166     /* The encodings for the immh:immb fields (bits 22:16) are                          \
2167      *   0001 xxx       8B/16B, shift = xxx                                             \
2168      *   001x xxx       4H/8H,  shift = xxxx                                            \
2169      *   01xx xxx       2S/4S,  shift = xxxxx                                           \
2170      *   1xxx xxx       1D/2D,  shift = xxxxxx (1D is RESERVED)                         \
2171      */                                                                                 \
2172     assert((1 << ((T>>1)+3)) > shift, "Invalid Shift value");                           \
2173     f(0, 31), f(T & 1, 30), f(opc, 29), f(0b011110, 28, 23),                            \
2174     f((1 << ((T>>1)+3))|shift, 22, 16); f(opc2, 15, 10), rf(Vn, 5), rf(Vd, 0);          \
2175   }
2176 
2177   INSN(shl,  0, 0b010101);
2178   INSN(sshr, 0, 0b000001);
2179   INSN(ushr, 1, 0b000001);
2180 
2181 #undef INSN
2182 
2183   void ushll(FloatRegister Vd, SIMD_Arrangement Ta, FloatRegister Vn, SIMD_Arrangement Tb, int shift) {
2184     starti;
2185     /* The encodings for the immh:immb fields (bits 22:16) are
2186      *   0001 xxx       8H, 8B/16b shift = xxx
2187      *   001x xxx       4S, 4H/8H  shift = xxxx
2188      *   01xx xxx       2D, 2S/4S  shift = xxxxx
2189      *   1xxx xxx       RESERVED
2190      */
2191     assert((Tb >> 1) + 1 == (Ta >> 1), "Incompatible arrangement");
2192     assert((1 << ((Tb>>1)+3)) > shift, "Invalid shift value");
2193     f(0, 31), f(Tb & 1, 30), f(0b1011110, 29, 23), f((1 << ((Tb>>1)+3))|shift, 22, 16);
2194     f(0b101001, 15, 10), rf(Vn, 5), rf(Vd, 0);
2195   }
2196   void ushll2(FloatRegister Vd, SIMD_Arrangement Ta, FloatRegister Vn,  SIMD_Arrangement Tb, int shift) {
2197     ushll(Vd, Ta, Vn, Tb, shift);
2198   }
2199 
2200   void uzp1(FloatRegister Vd, FloatRegister Vn, FloatRegister Vm,  SIMD_Arrangement T, int op = 0){
2201     starti;
2202     f(0, 31), f((T & 0x1), 30), f(0b001110, 29, 24), f((T >> 1), 23, 22), f(0, 21);
2203     rf(Vm, 16), f(0, 15), f(op, 14), f(0b0110, 13, 10), rf(Vn, 5), rf(Vd, 0);
2204   }
2205   void uzp2(FloatRegister Vd, FloatRegister Vn, FloatRegister Vm,  SIMD_Arrangement T){
2206     uzp1(Vd, Vn, Vm, T, 1);
2207   }
2208 
2209   // Move from general purpose register
2210   //   mov  Vd.T[index], Rn
2211   void mov(FloatRegister Vd, SIMD_Arrangement T, int index, Register Xn) {
2212     starti;
2213     f(0b01001110000, 31, 21), f(((1 << (T >> 1)) | (index << ((T >> 1) + 1))), 20, 16);
2214     f(0b000111, 15, 10), rf(Xn, 5), rf(Vd, 0);
2215   }
2216 
2217   // Move to general purpose register
2218   //   mov  Rd, Vn.T[index]
2219   void mov(Register Xd, FloatRegister Vn, SIMD_Arrangement T, int index) {
2220     starti;
2221     f(0, 31), f((T >= T1D) ? 1:0, 30), f(0b001110000, 29, 21);
2222     f(((1 << (T >> 1)) | (index << ((T >> 1) + 1))), 20, 16);
2223     f(0b001111, 15, 10), rf(Vn, 5), rf(Xd, 0);
2224   }
2225 
2226   // We do not handle the 1Q arrangement.
2227   void pmull(FloatRegister Vd, SIMD_Arrangement Ta, FloatRegister Vn, FloatRegister Vm, SIMD_Arrangement Tb) {
2228     starti;
2229     assert(Ta == T8H && (Tb == T8B || Tb == T16B), "Invalid Size specifier");
2230     f(0, 31), f(Tb & 1, 30), f(0b001110001, 29, 21), rf(Vm, 16), f(0b111000, 15, 10);
2231     rf(Vn, 5), rf(Vd, 0);
2232   }
2233   void pmull2(FloatRegister Vd, SIMD_Arrangement Ta, FloatRegister Vn, FloatRegister Vm, SIMD_Arrangement Tb) {
2234     pmull(Vd, Ta, Vn, Vm, Tb);
2235   }
2236 
2237   void uqxtn(FloatRegister Vd, SIMD_Arrangement Tb, FloatRegister Vn, SIMD_Arrangement Ta) {
2238     starti;
2239     int size_b = (int)Tb >> 1;
2240     int size_a = (int)Ta >> 1;
2241     assert(size_b < 3 && size_b == size_a - 1, "Invalid size specifier");
2242     f(0, 31), f(Tb & 1, 30), f(0b101110, 29, 24), f(size_b, 23, 22);
2243     f(0b100001010010, 21, 10), rf(Vn, 5), rf(Vd, 0);
2244   }
2245 
2246   void rev32(FloatRegister Vd, SIMD_Arrangement T, FloatRegister Vn)
2247   {
2248     starti;
2249     assert(T <= T8H, "must be one of T8B, T16B, T4H, T8H");
2250     f(0, 31), f((int)T & 1, 30), f(0b101110, 29, 24);
2251     f(T <= T16B ? 0b00 : 0b01, 23, 22), f(0b100000000010, 21, 10);
2252     rf(Vn, 5), rf(Vd, 0);
2253   }
2254 
2255   void dup(FloatRegister Vd, SIMD_Arrangement T, Register Xs)
2256   {
2257     starti;
2258     assert(T != T1D, "reserved encoding");
2259     f(0,31), f((int)T & 1, 30), f(0b001110000, 29, 21);
2260     f((1 << (T >> 1)), 20, 16), f(0b000011, 15, 10), rf(Xs, 5), rf(Vd, 0);
2261   }
2262 
2263   void dup(FloatRegister Vd, SIMD_Arrangement T, FloatRegister Vn, int index = 0)
2264   {
2265     starti;
2266     assert(T != T1D, "reserved encoding");
2267     f(0, 31), f((int)T & 1, 30), f(0b001110000, 29, 21);
2268     f(((1 << (T >> 1)) | (index << ((T >> 1) + 1))), 20, 16);
2269     f(0b000001, 15, 10), rf(Vn, 5), rf(Vd, 0);
2270   }
2271 
2272   // CRC32 instructions
2273 #define INSN(NAME, c, sf, sz)                                             \
2274   void NAME(Register Rd, Register Rn, Register Rm) {                      \
2275     starti;                                                               \
2276     f(sf, 31), f(0b0011010110, 30, 21), f(0b010, 15, 13), f(c, 12);       \
2277     f(sz, 11, 10), rf(Rm, 16), rf(Rn, 5), rf(Rd, 0);                      \
2278   }
2279 
2280   INSN(crc32b,  0, 0, 0b00);
2281   INSN(crc32h,  0, 0, 0b01);
2282   INSN(crc32w,  0, 0, 0b10);
2283   INSN(crc32x,  0, 1, 0b11);
2284   INSN(crc32cb, 1, 0, 0b00);
2285   INSN(crc32ch, 1, 0, 0b01);
2286   INSN(crc32cw, 1, 0, 0b10);
2287   INSN(crc32cx, 1, 1, 0b11);
2288 
2289 #undef INSN
2290 
2291 
2292 /* Simulator extensions to the ISA
2293 
2294    haltsim
2295 
2296    takes no arguments, causes the sim to enter a debug break and then
2297    return from the simulator run() call with STATUS_HALT? The linking
2298    code will call fatal() when it sees STATUS_HALT.
2299 
2300    blrt Xn, Wm
2301    blrt Xn, #gpargs, #fpargs, #type
2302    Xn holds the 64 bit x86 branch_address
2303    call format is encoded either as immediate data in the call
2304    or in register Wm. In the latter case
2305      Wm[13..6] = #gpargs,
2306      Wm[5..2] = #fpargs,
2307      Wm[1,0] = #type
2308 
2309    calls the x86 code address 'branch_address' supplied in Xn passing
2310    arguments taken from the general and floating point registers according
2311    to the supplied counts 'gpargs' and 'fpargs'. may return a result in r0
2312    or v0 according to the the return type #type' where
2313 
2314    address branch_address;
2315    uimm4 gpargs;
2316    uimm4 fpargs;
2317    enum ReturnType type;
2318 
2319    enum ReturnType
2320      {
2321        void_ret = 0,
2322        int_ret = 1,
2323        long_ret = 1,
2324        obj_ret = 1, // i.e. same as long
2325        float_ret = 2,
2326        double_ret = 3
2327      }
2328 
2329    notify
2330 
2331    notifies the simulator of a transfer of control. instr[14:0]
2332    identifies the type of change of control.
2333 
2334    0 ==> initial entry to a method.
2335 
2336    1 ==> return into a method from a submethod call.
2337 
2338    2 ==> exit out of Java method code.
2339 
2340    3 ==> start execution for a new bytecode.
2341 
2342    in cases 1 and 2 the simulator is expected to use a JVM callback to
2343    identify the name of the specific method being executed. in case 4
2344    the simulator is expected to use a JVM callback to identify the
2345    bytecode index.
2346 
2347    Instruction encodings
2348    ---------------------
2349 
2350    These are encoded in the space with instr[28:25] = 00 which is
2351    unallocated. Encodings are
2352 
2353                      10987654321098765432109876543210
2354    PSEUDO_HALT   = 0x11100000000000000000000000000000
2355    PSEUDO_BLRT  = 0x11000000000000000_______________
2356    PSEUDO_BLRTR = 0x1100000000000000100000__________
2357    PSEUDO_NOTIFY = 0x10100000000000000_______________
2358 
2359    instr[31,29] = op1 : 111 ==> HALT, 110 ==> BLRT/BLRTR, 101 ==> NOTIFY
2360 
2361    for BLRT
2362      instr[14,11] = #gpargs, instr[10,7] = #fpargs
2363      instr[6,5] = #type, instr[4,0] = Rn
2364    for BLRTR
2365      instr[9,5] = Rm, instr[4,0] = Rn
2366    for NOTIFY
2367      instr[14:0] = type : 0 ==> entry, 1 ==> reentry, 2 ==> exit, 3 ==> bcstart
2368 */
2369 
2370   enum NotifyType { method_entry, method_reentry, method_exit, bytecode_start };
2371 
2372   virtual void notify(int type) {
2373     if (UseBuiltinSim) {
2374       starti;
2375       //  109
2376       f(0b101, 31, 29);
2377       //  87654321098765
2378       f(0b00000000000000, 28, 15);
2379       f(type, 14, 0);
2380     }
2381   }
2382 
2383   void blrt(Register Rn, int gpargs, int fpargs, int type) {
2384     if (UseBuiltinSim) {
2385       starti;
2386       f(0b110, 31 ,29);
2387       f(0b00, 28, 25);
2388       //  4321098765
2389       f(0b0000000000, 24, 15);
2390       f(gpargs, 14, 11);
2391       f(fpargs, 10, 7);
2392       f(type, 6, 5);
2393       rf(Rn, 0);
2394     } else {
2395       blr(Rn);
2396     }
2397   }
2398 
2399   void blrt(Register Rn, Register Rm) {
2400     if (UseBuiltinSim) {
2401       starti;
2402       f(0b110, 31 ,29);
2403       f(0b00, 28, 25);
2404       //  4321098765
2405       f(0b0000000001, 24, 15);
2406       //  43210
2407       f(0b00000, 14, 10);
2408       rf(Rm, 5);
2409       rf(Rn, 0);
2410     } else {
2411       blr(Rn);
2412     }
2413   }
2414 
2415   void haltsim() {
2416     starti;
2417     f(0b111, 31 ,29);
2418     f(0b00, 28, 27);
2419     //  654321098765432109876543210
2420     f(0b000000000000000000000000000, 26, 0);
2421   }
2422 
2423   Assembler(CodeBuffer* code) : AbstractAssembler(code) {
2424   }
2425 
2426   virtual RegisterOrConstant delayed_value_impl(intptr_t* delayed_value_addr,
2427                                                 Register tmp,
2428                                                 int offset) {
2429     ShouldNotCallThis();
2430     return RegisterOrConstant();
2431   }
2432 
2433   // Stack overflow checking
2434   virtual void bang_stack_with_offset(int offset);
2435 
2436   static bool operand_valid_for_logical_immediate(bool is32, uint64_t imm);
2437   static bool operand_valid_for_add_sub_immediate(long imm);
2438   static bool operand_valid_for_float_immediate(double imm);
2439 
2440   void emit_data64(jlong data, relocInfo::relocType rtype, int format = 0);
2441   void emit_data64(jlong data, RelocationHolder const& rspec, int format = 0);
2442 };
2443 
2444 inline Assembler::Membar_mask_bits operator|(Assembler::Membar_mask_bits a,
2445                                              Assembler::Membar_mask_bits b) {
2446   return Assembler::Membar_mask_bits(unsigned(a)|unsigned(b));
2447 }
2448 
2449 Instruction_aarch64::~Instruction_aarch64() {
2450   assem->emit();
2451 }
2452 
2453 #undef starti
2454 
2455 // Invert a condition
2456 inline const Assembler::Condition operator~(const Assembler::Condition cond) {
2457   return Assembler::Condition(int(cond) ^ 1);
2458 }
2459 
2460 class BiasedLockingCounters;
2461 
2462 extern "C" void das(uint64_t start, int len);
2463 
2464 #endif // CPU_AARCH64_VM_ASSEMBLER_AARCH64_HPP