1 /*
   2  * Copyright (c) 1997, 2016, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #include "precompiled.hpp"
  26 #include "asm/assembler.hpp"
  27 #include "asm/assembler.inline.hpp"
  28 #include "gc/shared/cardTableModRefBS.hpp"
  29 #include "gc/shared/collectedHeap.inline.hpp"
  30 #include "interpreter/interpreter.hpp"
  31 #include "memory/resourceArea.hpp"
  32 #include "prims/methodHandles.hpp"
  33 #include "runtime/biasedLocking.hpp"
  34 #include "runtime/interfaceSupport.hpp"
  35 #include "runtime/objectMonitor.hpp"
  36 #include "runtime/os.hpp"
  37 #include "runtime/sharedRuntime.hpp"
  38 #include "runtime/stubRoutines.hpp"
  39 #include "utilities/macros.hpp"
  40 #if INCLUDE_ALL_GCS
  41 #include "gc/g1/g1CollectedHeap.inline.hpp"
  42 #include "gc/g1/g1SATBCardTableModRefBS.hpp"
  43 #include "gc/g1/heapRegion.hpp"
  44 #endif // INCLUDE_ALL_GCS
  45 
  46 #ifdef PRODUCT
  47 #define BLOCK_COMMENT(str) /* nothing */
  48 #define STOP(error) stop(error)
  49 #else
  50 #define BLOCK_COMMENT(str) block_comment(str)
  51 #define STOP(error) block_comment(error); stop(error)
  52 #endif
  53 
  54 #define BIND(label) bind(label); BLOCK_COMMENT(#label ":")
  55 // Implementation of AddressLiteral
  56 
  57 // A 2-D table for managing compressed displacement(disp8) on EVEX enabled platforms.
  58 unsigned char tuple_table[Assembler::EVEX_ETUP + 1][Assembler::AVX_512bit + 1] = {
  59   // -----------------Table 4.5 -------------------- //
  60   16, 32, 64,  // EVEX_FV(0)
  61   4,  4,  4,   // EVEX_FV(1) - with Evex.b
  62   16, 32, 64,  // EVEX_FV(2) - with Evex.w
  63   8,  8,  8,   // EVEX_FV(3) - with Evex.w and Evex.b
  64   8,  16, 32,  // EVEX_HV(0)
  65   4,  4,  4,   // EVEX_HV(1) - with Evex.b
  66   // -----------------Table 4.6 -------------------- //
  67   16, 32, 64,  // EVEX_FVM(0)
  68   1,  1,  1,   // EVEX_T1S(0)
  69   2,  2,  2,   // EVEX_T1S(1)
  70   4,  4,  4,   // EVEX_T1S(2)
  71   8,  8,  8,   // EVEX_T1S(3)
  72   4,  4,  4,   // EVEX_T1F(0)
  73   8,  8,  8,   // EVEX_T1F(1)
  74   8,  8,  8,   // EVEX_T2(0)
  75   0,  16, 16,  // EVEX_T2(1)
  76   0,  16, 16,  // EVEX_T4(0)
  77   0,  0,  32,  // EVEX_T4(1)
  78   0,  0,  32,  // EVEX_T8(0)
  79   8,  16, 32,  // EVEX_HVM(0)
  80   4,  8,  16,  // EVEX_QVM(0)
  81   2,  4,  8,   // EVEX_OVM(0)
  82   16, 16, 16,  // EVEX_M128(0)
  83   8,  32, 64,  // EVEX_DUP(0)
  84   0,  0,  0    // EVEX_NTUP
  85 };
  86 
  87 AddressLiteral::AddressLiteral(address target, relocInfo::relocType rtype) {
  88   _is_lval = false;
  89   _target = target;
  90   switch (rtype) {
  91   case relocInfo::oop_type:
  92   case relocInfo::metadata_type:
  93     // Oops are a special case. Normally they would be their own section
  94     // but in cases like icBuffer they are literals in the code stream that
  95     // we don't have a section for. We use none so that we get a literal address
  96     // which is always patchable.
  97     break;
  98   case relocInfo::external_word_type:
  99     _rspec = external_word_Relocation::spec(target);
 100     break;
 101   case relocInfo::internal_word_type:
 102     _rspec = internal_word_Relocation::spec(target);
 103     break;
 104   case relocInfo::opt_virtual_call_type:
 105     _rspec = opt_virtual_call_Relocation::spec();
 106     break;
 107   case relocInfo::static_call_type:
 108     _rspec = static_call_Relocation::spec();
 109     break;
 110   case relocInfo::runtime_call_type:
 111     _rspec = runtime_call_Relocation::spec();
 112     break;
 113   case relocInfo::poll_type:
 114   case relocInfo::poll_return_type:
 115     _rspec = Relocation::spec_simple(rtype);
 116     break;
 117   case relocInfo::none:
 118     break;
 119   default:
 120     ShouldNotReachHere();
 121     break;
 122   }
 123 }
 124 
 125 // Implementation of Address
 126 
 127 #ifdef _LP64
 128 
 129 Address Address::make_array(ArrayAddress adr) {
 130   // Not implementable on 64bit machines
 131   // Should have been handled higher up the call chain.
 132   ShouldNotReachHere();
 133   return Address();
 134 }
 135 
 136 // exceedingly dangerous constructor
 137 Address::Address(int disp, address loc, relocInfo::relocType rtype) {
 138   _base  = noreg;
 139   _index = noreg;
 140   _scale = no_scale;
 141   _disp  = disp;
 142   switch (rtype) {
 143     case relocInfo::external_word_type:
 144       _rspec = external_word_Relocation::spec(loc);
 145       break;
 146     case relocInfo::internal_word_type:
 147       _rspec = internal_word_Relocation::spec(loc);
 148       break;
 149     case relocInfo::runtime_call_type:
 150       // HMM
 151       _rspec = runtime_call_Relocation::spec();
 152       break;
 153     case relocInfo::poll_type:
 154     case relocInfo::poll_return_type:
 155       _rspec = Relocation::spec_simple(rtype);
 156       break;
 157     case relocInfo::none:
 158       break;
 159     default:
 160       ShouldNotReachHere();
 161   }
 162 }
 163 #else // LP64
 164 
 165 Address Address::make_array(ArrayAddress adr) {
 166   AddressLiteral base = adr.base();
 167   Address index = adr.index();
 168   assert(index._disp == 0, "must not have disp"); // maybe it can?
 169   Address array(index._base, index._index, index._scale, (intptr_t) base.target());
 170   array._rspec = base._rspec;
 171   return array;
 172 }
 173 
 174 // exceedingly dangerous constructor
 175 Address::Address(address loc, RelocationHolder spec) {
 176   _base  = noreg;
 177   _index = noreg;
 178   _scale = no_scale;
 179   _disp  = (intptr_t) loc;
 180   _rspec = spec;
 181 }
 182 
 183 #endif // _LP64
 184 
 185 
 186 
 187 // Convert the raw encoding form into the form expected by the constructor for
 188 // Address.  An index of 4 (rsp) corresponds to having no index, so convert
 189 // that to noreg for the Address constructor.
 190 Address Address::make_raw(int base, int index, int scale, int disp, relocInfo::relocType disp_reloc) {
 191   RelocationHolder rspec;
 192   if (disp_reloc != relocInfo::none) {
 193     rspec = Relocation::spec_simple(disp_reloc);
 194   }
 195   bool valid_index = index != rsp->encoding();
 196   if (valid_index) {
 197     Address madr(as_Register(base), as_Register(index), (Address::ScaleFactor)scale, in_ByteSize(disp));
 198     madr._rspec = rspec;
 199     return madr;
 200   } else {
 201     Address madr(as_Register(base), noreg, Address::no_scale, in_ByteSize(disp));
 202     madr._rspec = rspec;
 203     return madr;
 204   }
 205 }
 206 
 207 // Implementation of Assembler
 208 
 209 int AbstractAssembler::code_fill_byte() {
 210   return (u_char)'\xF4'; // hlt
 211 }
 212 
 213 // make this go away someday
 214 void Assembler::emit_data(jint data, relocInfo::relocType rtype, int format) {
 215   if (rtype == relocInfo::none)
 216     emit_int32(data);
 217   else
 218     emit_data(data, Relocation::spec_simple(rtype), format);
 219 }
 220 
 221 void Assembler::emit_data(jint data, RelocationHolder const& rspec, int format) {
 222   assert(imm_operand == 0, "default format must be immediate in this file");
 223   assert(inst_mark() != NULL, "must be inside InstructionMark");
 224   if (rspec.type() !=  relocInfo::none) {
 225     #ifdef ASSERT
 226       check_relocation(rspec, format);
 227     #endif
 228     // Do not use AbstractAssembler::relocate, which is not intended for
 229     // embedded words.  Instead, relocate to the enclosing instruction.
 230 
 231     // hack. call32 is too wide for mask so use disp32
 232     if (format == call32_operand)
 233       code_section()->relocate(inst_mark(), rspec, disp32_operand);
 234     else
 235       code_section()->relocate(inst_mark(), rspec, format);
 236   }
 237   emit_int32(data);
 238 }
 239 
 240 static int encode(Register r) {
 241   int enc = r->encoding();
 242   if (enc >= 8) {
 243     enc -= 8;
 244   }
 245   return enc;
 246 }
 247 
 248 void Assembler::emit_arith_b(int op1, int op2, Register dst, int imm8) {
 249   assert(dst->has_byte_register(), "must have byte register");
 250   assert(isByte(op1) && isByte(op2), "wrong opcode");
 251   assert(isByte(imm8), "not a byte");
 252   assert((op1 & 0x01) == 0, "should be 8bit operation");
 253   emit_int8(op1);
 254   emit_int8(op2 | encode(dst));
 255   emit_int8(imm8);
 256 }
 257 
 258 
 259 void Assembler::emit_arith(int op1, int op2, Register dst, int32_t imm32) {
 260   assert(isByte(op1) && isByte(op2), "wrong opcode");
 261   assert((op1 & 0x01) == 1, "should be 32bit operation");
 262   assert((op1 & 0x02) == 0, "sign-extension bit should not be set");
 263   if (is8bit(imm32)) {
 264     emit_int8(op1 | 0x02); // set sign bit
 265     emit_int8(op2 | encode(dst));
 266     emit_int8(imm32 & 0xFF);
 267   } else {
 268     emit_int8(op1);
 269     emit_int8(op2 | encode(dst));
 270     emit_int32(imm32);
 271   }
 272 }
 273 
 274 // Force generation of a 4 byte immediate value even if it fits into 8bit
 275 void Assembler::emit_arith_imm32(int op1, int op2, Register dst, int32_t imm32) {
 276   assert(isByte(op1) && isByte(op2), "wrong opcode");
 277   assert((op1 & 0x01) == 1, "should be 32bit operation");
 278   assert((op1 & 0x02) == 0, "sign-extension bit should not be set");
 279   emit_int8(op1);
 280   emit_int8(op2 | encode(dst));
 281   emit_int32(imm32);
 282 }
 283 
 284 // immediate-to-memory forms
 285 void Assembler::emit_arith_operand(int op1, Register rm, Address adr, int32_t imm32) {
 286   assert((op1 & 0x01) == 1, "should be 32bit operation");
 287   assert((op1 & 0x02) == 0, "sign-extension bit should not be set");
 288   if (is8bit(imm32)) {
 289     emit_int8(op1 | 0x02); // set sign bit
 290     emit_operand(rm, adr, 1);
 291     emit_int8(imm32 & 0xFF);
 292   } else {
 293     emit_int8(op1);
 294     emit_operand(rm, adr, 4);
 295     emit_int32(imm32);
 296   }
 297 }
 298 
 299 
 300 void Assembler::emit_arith(int op1, int op2, Register dst, Register src) {
 301   assert(isByte(op1) && isByte(op2), "wrong opcode");
 302   emit_int8(op1);
 303   emit_int8(op2 | encode(dst) << 3 | encode(src));
 304 }
 305 
 306 
 307 bool Assembler::query_compressed_disp_byte(int disp, bool is_evex_inst, int vector_len,
 308                                            int cur_tuple_type, int in_size_in_bits, int cur_encoding) {
 309   int mod_idx = 0;
 310   // We will test if the displacement fits the compressed format and if so
 311   // apply the compression to the displacment iff the result is8bit.
 312   if (VM_Version::supports_evex() && is_evex_inst) {
 313     switch (cur_tuple_type) {
 314     case EVEX_FV:
 315       if ((cur_encoding & VEX_W) == VEX_W) {
 316         mod_idx = ((cur_encoding & EVEX_Rb) == EVEX_Rb) ? 3 : 2;
 317       } else {
 318         mod_idx = ((cur_encoding & EVEX_Rb) == EVEX_Rb) ? 1 : 0;
 319       }
 320       break;
 321 
 322     case EVEX_HV:
 323       mod_idx = ((cur_encoding & EVEX_Rb) == EVEX_Rb) ? 1 : 0;
 324       break;
 325 
 326     case EVEX_FVM:
 327       break;
 328 
 329     case EVEX_T1S:
 330       switch (in_size_in_bits) {
 331       case EVEX_8bit:
 332         break;
 333 
 334       case EVEX_16bit:
 335         mod_idx = 1;
 336         break;
 337 
 338       case EVEX_32bit:
 339         mod_idx = 2;
 340         break;
 341 
 342       case EVEX_64bit:
 343         mod_idx = 3;
 344         break;
 345       }
 346       break;
 347 
 348     case EVEX_T1F:
 349     case EVEX_T2:
 350     case EVEX_T4:
 351       mod_idx = (in_size_in_bits == EVEX_64bit) ? 1 : 0;
 352       break;
 353 
 354     case EVEX_T8:
 355       break;
 356 
 357     case EVEX_HVM:
 358       break;
 359 
 360     case EVEX_QVM:
 361       break;
 362 
 363     case EVEX_OVM:
 364       break;
 365 
 366     case EVEX_M128:
 367       break;
 368 
 369     case EVEX_DUP:
 370       break;
 371 
 372     default:
 373       assert(0, "no valid evex tuple_table entry");
 374       break;
 375     }
 376 
 377     if (vector_len >= AVX_128bit && vector_len <= AVX_512bit) {
 378       int disp_factor = tuple_table[cur_tuple_type + mod_idx][vector_len];
 379       if ((disp % disp_factor) == 0) {
 380         int new_disp = disp / disp_factor;
 381         if ((-0x80 <= new_disp && new_disp < 0x80)) {
 382           disp = new_disp;
 383         }
 384       } else {
 385         return false;
 386       }
 387     }
 388   }
 389   return (-0x80 <= disp && disp < 0x80);
 390 }
 391 
 392 
 393 bool Assembler::emit_compressed_disp_byte(int &disp) {
 394   int mod_idx = 0;
 395   // We will test if the displacement fits the compressed format and if so
 396   // apply the compression to the displacment iff the result is8bit.
 397   if (VM_Version::supports_evex() && _attributes && _attributes->is_evex_instruction()) {
 398     int evex_encoding = _attributes->get_evex_encoding();
 399     int tuple_type = _attributes->get_tuple_type();
 400     switch (tuple_type) {
 401     case EVEX_FV:
 402       if ((evex_encoding & VEX_W) == VEX_W) {
 403         mod_idx = ((evex_encoding & EVEX_Rb) == EVEX_Rb) ? 3 : 2;
 404       } else {
 405         mod_idx = ((evex_encoding & EVEX_Rb) == EVEX_Rb) ? 1 : 0;
 406       }
 407       break;
 408 
 409     case EVEX_HV:
 410       mod_idx = ((evex_encoding & EVEX_Rb) == EVEX_Rb) ? 1 : 0;
 411       break;
 412 
 413     case EVEX_FVM:
 414       break;
 415 
 416     case EVEX_T1S:
 417       switch (_attributes->get_input_size()) {
 418       case EVEX_8bit:
 419         break;
 420 
 421       case EVEX_16bit:
 422         mod_idx = 1;
 423         break;
 424 
 425       case EVEX_32bit:
 426         mod_idx = 2;
 427         break;
 428 
 429       case EVEX_64bit:
 430         mod_idx = 3;
 431         break;
 432       }
 433       break;
 434 
 435     case EVEX_T1F:
 436     case EVEX_T2:
 437     case EVEX_T4:
 438       mod_idx = (_attributes->get_input_size() == EVEX_64bit) ? 1 : 0;
 439       break;
 440 
 441     case EVEX_T8:
 442       break;
 443 
 444     case EVEX_HVM:
 445       break;
 446 
 447     case EVEX_QVM:
 448       break;
 449 
 450     case EVEX_OVM:
 451       break;
 452 
 453     case EVEX_M128:
 454       break;
 455 
 456     case EVEX_DUP:
 457       break;
 458 
 459     default:
 460       assert(0, "no valid evex tuple_table entry");
 461       break;
 462     }
 463 
 464     int vector_len = _attributes->get_vector_len();
 465     if (vector_len >= AVX_128bit && vector_len <= AVX_512bit) {
 466       int disp_factor = tuple_table[tuple_type + mod_idx][vector_len];
 467       if ((disp % disp_factor) == 0) {
 468         int new_disp = disp / disp_factor;
 469         if (is8bit(new_disp)) {
 470           disp = new_disp;
 471         }
 472       } else {
 473         return false;
 474       }
 475     }
 476   }
 477   return is8bit(disp);
 478 }
 479 
 480 
 481 void Assembler::emit_operand(Register reg, Register base, Register index,
 482                              Address::ScaleFactor scale, int disp,
 483                              RelocationHolder const& rspec,
 484                              int rip_relative_correction) {
 485   relocInfo::relocType rtype = (relocInfo::relocType) rspec.type();
 486 
 487   // Encode the registers as needed in the fields they are used in
 488 
 489   int regenc = encode(reg) << 3;
 490   int indexenc = index->is_valid() ? encode(index) << 3 : 0;
 491   int baseenc = base->is_valid() ? encode(base) : 0;
 492 
 493   if (base->is_valid()) {
 494     if (index->is_valid()) {
 495       assert(scale != Address::no_scale, "inconsistent address");
 496       // [base + index*scale + disp]
 497       if (disp == 0 && rtype == relocInfo::none  &&
 498           base != rbp LP64_ONLY(&& base != r13)) {
 499         // [base + index*scale]
 500         // [00 reg 100][ss index base]
 501         assert(index != rsp, "illegal addressing mode");
 502         emit_int8(0x04 | regenc);
 503         emit_int8(scale << 6 | indexenc | baseenc);
 504       } else if (emit_compressed_disp_byte(disp) && rtype == relocInfo::none) {
 505         // [base + index*scale + imm8]
 506         // [01 reg 100][ss index base] imm8
 507         assert(index != rsp, "illegal addressing mode");
 508         emit_int8(0x44 | regenc);
 509         emit_int8(scale << 6 | indexenc | baseenc);
 510         emit_int8(disp & 0xFF);
 511       } else {
 512         // [base + index*scale + disp32]
 513         // [10 reg 100][ss index base] disp32
 514         assert(index != rsp, "illegal addressing mode");
 515         emit_int8(0x84 | regenc);
 516         emit_int8(scale << 6 | indexenc | baseenc);
 517         emit_data(disp, rspec, disp32_operand);
 518       }
 519     } else if (base == rsp LP64_ONLY(|| base == r12)) {
 520       // [rsp + disp]
 521       if (disp == 0 && rtype == relocInfo::none) {
 522         // [rsp]
 523         // [00 reg 100][00 100 100]
 524         emit_int8(0x04 | regenc);
 525         emit_int8(0x24);
 526       } else if (emit_compressed_disp_byte(disp) && rtype == relocInfo::none) {
 527         // [rsp + imm8]
 528         // [01 reg 100][00 100 100] disp8
 529         emit_int8(0x44 | regenc);
 530         emit_int8(0x24);
 531         emit_int8(disp & 0xFF);
 532       } else {
 533         // [rsp + imm32]
 534         // [10 reg 100][00 100 100] disp32
 535         emit_int8(0x84 | regenc);
 536         emit_int8(0x24);
 537         emit_data(disp, rspec, disp32_operand);
 538       }
 539     } else {
 540       // [base + disp]
 541       assert(base != rsp LP64_ONLY(&& base != r12), "illegal addressing mode");
 542       if (disp == 0 && rtype == relocInfo::none &&
 543           base != rbp LP64_ONLY(&& base != r13)) {
 544         // [base]
 545         // [00 reg base]
 546         emit_int8(0x00 | regenc | baseenc);
 547       } else if (emit_compressed_disp_byte(disp) && rtype == relocInfo::none) {
 548         // [base + disp8]
 549         // [01 reg base] disp8
 550         emit_int8(0x40 | regenc | baseenc);
 551         emit_int8(disp & 0xFF);
 552       } else {
 553         // [base + disp32]
 554         // [10 reg base] disp32
 555         emit_int8(0x80 | regenc | baseenc);
 556         emit_data(disp, rspec, disp32_operand);
 557       }
 558     }
 559   } else {
 560     if (index->is_valid()) {
 561       assert(scale != Address::no_scale, "inconsistent address");
 562       // [index*scale + disp]
 563       // [00 reg 100][ss index 101] disp32
 564       assert(index != rsp, "illegal addressing mode");
 565       emit_int8(0x04 | regenc);
 566       emit_int8(scale << 6 | indexenc | 0x05);
 567       emit_data(disp, rspec, disp32_operand);
 568     } else if (rtype != relocInfo::none ) {
 569       // [disp] (64bit) RIP-RELATIVE (32bit) abs
 570       // [00 000 101] disp32
 571 
 572       emit_int8(0x05 | regenc);
 573       // Note that the RIP-rel. correction applies to the generated
 574       // disp field, but _not_ to the target address in the rspec.
 575 
 576       // disp was created by converting the target address minus the pc
 577       // at the start of the instruction. That needs more correction here.
 578       // intptr_t disp = target - next_ip;
 579       assert(inst_mark() != NULL, "must be inside InstructionMark");
 580       address next_ip = pc() + sizeof(int32_t) + rip_relative_correction;
 581       int64_t adjusted = disp;
 582       // Do rip-rel adjustment for 64bit
 583       LP64_ONLY(adjusted -=  (next_ip - inst_mark()));
 584       assert(is_simm32(adjusted),
 585              "must be 32bit offset (RIP relative address)");
 586       emit_data((int32_t) adjusted, rspec, disp32_operand);
 587 
 588     } else {
 589       // 32bit never did this, did everything as the rip-rel/disp code above
 590       // [disp] ABSOLUTE
 591       // [00 reg 100][00 100 101] disp32
 592       emit_int8(0x04 | regenc);
 593       emit_int8(0x25);
 594       emit_data(disp, rspec, disp32_operand);
 595     }
 596   }
 597 }
 598 
 599 void Assembler::emit_operand(XMMRegister reg, Register base, Register index,
 600                              Address::ScaleFactor scale, int disp,
 601                              RelocationHolder const& rspec) {
 602   if (UseAVX > 2) {
 603     int xreg_enc = reg->encoding();
 604     if (xreg_enc > 15) {
 605       XMMRegister new_reg = as_XMMRegister(xreg_enc & 0xf);
 606       emit_operand((Register)new_reg, base, index, scale, disp, rspec);
 607       return;
 608     }
 609   }
 610   emit_operand((Register)reg, base, index, scale, disp, rspec);
 611 }
 612 
 613 // Secret local extension to Assembler::WhichOperand:
 614 #define end_pc_operand (_WhichOperand_limit)
 615 
 616 address Assembler::locate_operand(address inst, WhichOperand which) {
 617   // Decode the given instruction, and return the address of
 618   // an embedded 32-bit operand word.
 619 
 620   // If "which" is disp32_operand, selects the displacement portion
 621   // of an effective address specifier.
 622   // If "which" is imm64_operand, selects the trailing immediate constant.
 623   // If "which" is call32_operand, selects the displacement of a call or jump.
 624   // Caller is responsible for ensuring that there is such an operand,
 625   // and that it is 32/64 bits wide.
 626 
 627   // If "which" is end_pc_operand, find the end of the instruction.
 628 
 629   address ip = inst;
 630   bool is_64bit = false;
 631 
 632   debug_only(bool has_disp32 = false);
 633   int tail_size = 0; // other random bytes (#32, #16, etc.) at end of insn
 634 
 635   again_after_prefix:
 636   switch (0xFF & *ip++) {
 637 
 638   // These convenience macros generate groups of "case" labels for the switch.
 639 #define REP4(x) (x)+0: case (x)+1: case (x)+2: case (x)+3
 640 #define REP8(x) (x)+0: case (x)+1: case (x)+2: case (x)+3: \
 641              case (x)+4: case (x)+5: case (x)+6: case (x)+7
 642 #define REP16(x) REP8((x)+0): \
 643               case REP8((x)+8)
 644 
 645   case CS_segment:
 646   case SS_segment:
 647   case DS_segment:
 648   case ES_segment:
 649   case FS_segment:
 650   case GS_segment:
 651     // Seems dubious
 652     LP64_ONLY(assert(false, "shouldn't have that prefix"));
 653     assert(ip == inst+1, "only one prefix allowed");
 654     goto again_after_prefix;
 655 
 656   case 0x67:
 657   case REX:
 658   case REX_B:
 659   case REX_X:
 660   case REX_XB:
 661   case REX_R:
 662   case REX_RB:
 663   case REX_RX:
 664   case REX_RXB:
 665     NOT_LP64(assert(false, "64bit prefixes"));
 666     goto again_after_prefix;
 667 
 668   case REX_W:
 669   case REX_WB:
 670   case REX_WX:
 671   case REX_WXB:
 672   case REX_WR:
 673   case REX_WRB:
 674   case REX_WRX:
 675   case REX_WRXB:
 676     NOT_LP64(assert(false, "64bit prefixes"));
 677     is_64bit = true;
 678     goto again_after_prefix;
 679 
 680   case 0xFF: // pushq a; decl a; incl a; call a; jmp a
 681   case 0x88: // movb a, r
 682   case 0x89: // movl a, r
 683   case 0x8A: // movb r, a
 684   case 0x8B: // movl r, a
 685   case 0x8F: // popl a
 686     debug_only(has_disp32 = true);
 687     break;
 688 
 689   case 0x68: // pushq #32
 690     if (which == end_pc_operand) {
 691       return ip + 4;
 692     }
 693     assert(which == imm_operand && !is_64bit, "pushl has no disp32 or 64bit immediate");
 694     return ip;                  // not produced by emit_operand
 695 
 696   case 0x66: // movw ... (size prefix)
 697     again_after_size_prefix2:
 698     switch (0xFF & *ip++) {
 699     case REX:
 700     case REX_B:
 701     case REX_X:
 702     case REX_XB:
 703     case REX_R:
 704     case REX_RB:
 705     case REX_RX:
 706     case REX_RXB:
 707     case REX_W:
 708     case REX_WB:
 709     case REX_WX:
 710     case REX_WXB:
 711     case REX_WR:
 712     case REX_WRB:
 713     case REX_WRX:
 714     case REX_WRXB:
 715       NOT_LP64(assert(false, "64bit prefix found"));
 716       goto again_after_size_prefix2;
 717     case 0x8B: // movw r, a
 718     case 0x89: // movw a, r
 719       debug_only(has_disp32 = true);
 720       break;
 721     case 0xC7: // movw a, #16
 722       debug_only(has_disp32 = true);
 723       tail_size = 2;  // the imm16
 724       break;
 725     case 0x0F: // several SSE/SSE2 variants
 726       ip--;    // reparse the 0x0F
 727       goto again_after_prefix;
 728     default:
 729       ShouldNotReachHere();
 730     }
 731     break;
 732 
 733   case REP8(0xB8): // movl/q r, #32/#64(oop?)
 734     if (which == end_pc_operand)  return ip + (is_64bit ? 8 : 4);
 735     // these asserts are somewhat nonsensical
 736 #ifndef _LP64
 737     assert(which == imm_operand || which == disp32_operand,
 738            "which %d is_64_bit %d ip " INTPTR_FORMAT, which, is_64bit, p2i(ip));
 739 #else
 740     assert((which == call32_operand || which == imm_operand) && is_64bit ||
 741            which == narrow_oop_operand && !is_64bit,
 742            "which %d is_64_bit %d ip " INTPTR_FORMAT, which, is_64bit, p2i(ip));
 743 #endif // _LP64
 744     return ip;
 745 
 746   case 0x69: // imul r, a, #32
 747   case 0xC7: // movl a, #32(oop?)
 748     tail_size = 4;
 749     debug_only(has_disp32 = true); // has both kinds of operands!
 750     break;
 751 
 752   case 0x0F: // movx..., etc.
 753     switch (0xFF & *ip++) {
 754     case 0x3A: // pcmpestri
 755       tail_size = 1;
 756     case 0x38: // ptest, pmovzxbw
 757       ip++; // skip opcode
 758       debug_only(has_disp32 = true); // has both kinds of operands!
 759       break;
 760 
 761     case 0x70: // pshufd r, r/a, #8
 762       debug_only(has_disp32 = true); // has both kinds of operands!
 763     case 0x73: // psrldq r, #8
 764       tail_size = 1;
 765       break;
 766 
 767     case 0x12: // movlps
 768     case 0x28: // movaps
 769     case 0x2E: // ucomiss
 770     case 0x2F: // comiss
 771     case 0x54: // andps
 772     case 0x55: // andnps
 773     case 0x56: // orps
 774     case 0x57: // xorps
 775     case 0x58: // addpd
 776     case 0x59: // mulpd
 777     case 0x6E: // movd
 778     case 0x7E: // movd
 779     case 0xAE: // ldmxcsr, stmxcsr, fxrstor, fxsave, clflush
 780     case 0xFE: // paddd
 781       debug_only(has_disp32 = true);
 782       break;
 783 
 784     case 0xAD: // shrd r, a, %cl
 785     case 0xAF: // imul r, a
 786     case 0xBE: // movsbl r, a (movsxb)
 787     case 0xBF: // movswl r, a (movsxw)
 788     case 0xB6: // movzbl r, a (movzxb)
 789     case 0xB7: // movzwl r, a (movzxw)
 790     case REP16(0x40): // cmovl cc, r, a
 791     case 0xB0: // cmpxchgb
 792     case 0xB1: // cmpxchg
 793     case 0xC1: // xaddl
 794     case 0xC7: // cmpxchg8
 795     case REP16(0x90): // setcc a
 796       debug_only(has_disp32 = true);
 797       // fall out of the switch to decode the address
 798       break;
 799 
 800     case 0xC4: // pinsrw r, a, #8
 801       debug_only(has_disp32 = true);
 802     case 0xC5: // pextrw r, r, #8
 803       tail_size = 1;  // the imm8
 804       break;
 805 
 806     case 0xAC: // shrd r, a, #8
 807       debug_only(has_disp32 = true);
 808       tail_size = 1;  // the imm8
 809       break;
 810 
 811     case REP16(0x80): // jcc rdisp32
 812       if (which == end_pc_operand)  return ip + 4;
 813       assert(which == call32_operand, "jcc has no disp32 or imm");
 814       return ip;
 815     default:
 816       ShouldNotReachHere();
 817     }
 818     break;
 819 
 820   case 0x81: // addl a, #32; addl r, #32
 821     // also: orl, adcl, sbbl, andl, subl, xorl, cmpl
 822     // on 32bit in the case of cmpl, the imm might be an oop
 823     tail_size = 4;
 824     debug_only(has_disp32 = true); // has both kinds of operands!
 825     break;
 826 
 827   case 0x83: // addl a, #8; addl r, #8
 828     // also: orl, adcl, sbbl, andl, subl, xorl, cmpl
 829     debug_only(has_disp32 = true); // has both kinds of operands!
 830     tail_size = 1;
 831     break;
 832 
 833   case 0x9B:
 834     switch (0xFF & *ip++) {
 835     case 0xD9: // fnstcw a
 836       debug_only(has_disp32 = true);
 837       break;
 838     default:
 839       ShouldNotReachHere();
 840     }
 841     break;
 842 
 843   case REP4(0x00): // addb a, r; addl a, r; addb r, a; addl r, a
 844   case REP4(0x10): // adc...
 845   case REP4(0x20): // and...
 846   case REP4(0x30): // xor...
 847   case REP4(0x08): // or...
 848   case REP4(0x18): // sbb...
 849   case REP4(0x28): // sub...
 850   case 0xF7: // mull a
 851   case 0x8D: // lea r, a
 852   case 0x87: // xchg r, a
 853   case REP4(0x38): // cmp...
 854   case 0x85: // test r, a
 855     debug_only(has_disp32 = true); // has both kinds of operands!
 856     break;
 857 
 858   case 0xC1: // sal a, #8; sar a, #8; shl a, #8; shr a, #8
 859   case 0xC6: // movb a, #8
 860   case 0x80: // cmpb a, #8
 861   case 0x6B: // imul r, a, #8
 862     debug_only(has_disp32 = true); // has both kinds of operands!
 863     tail_size = 1; // the imm8
 864     break;
 865 
 866   case 0xC4: // VEX_3bytes
 867   case 0xC5: // VEX_2bytes
 868     assert((UseAVX > 0), "shouldn't have VEX prefix");
 869     assert(ip == inst+1, "no prefixes allowed");
 870     // C4 and C5 are also used as opcodes for PINSRW and PEXTRW instructions
 871     // but they have prefix 0x0F and processed when 0x0F processed above.
 872     //
 873     // In 32-bit mode the VEX first byte C4 and C5 alias onto LDS and LES
 874     // instructions (these instructions are not supported in 64-bit mode).
 875     // To distinguish them bits [7:6] are set in the VEX second byte since
 876     // ModRM byte can not be of the form 11xxxxxx in 32-bit mode. To set
 877     // those VEX bits REX and vvvv bits are inverted.
 878     //
 879     // Fortunately C2 doesn't generate these instructions so we don't need
 880     // to check for them in product version.
 881 
 882     // Check second byte
 883     NOT_LP64(assert((0xC0 & *ip) == 0xC0, "shouldn't have LDS and LES instructions"));
 884 
 885     int vex_opcode;
 886     // First byte
 887     if ((0xFF & *inst) == VEX_3bytes) {
 888       vex_opcode = VEX_OPCODE_MASK & *ip;
 889       ip++; // third byte
 890       is_64bit = ((VEX_W & *ip) == VEX_W);
 891     } else {
 892       vex_opcode = VEX_OPCODE_0F;
 893     }
 894     ip++; // opcode
 895     // To find the end of instruction (which == end_pc_operand).
 896     switch (vex_opcode) {
 897       case VEX_OPCODE_0F:
 898         switch (0xFF & *ip) {
 899         case 0x70: // pshufd r, r/a, #8
 900         case 0x71: // ps[rl|ra|ll]w r, #8
 901         case 0x72: // ps[rl|ra|ll]d r, #8
 902         case 0x73: // ps[rl|ra|ll]q r, #8
 903         case 0xC2: // cmp[ps|pd|ss|sd] r, r, r/a, #8
 904         case 0xC4: // pinsrw r, r, r/a, #8
 905         case 0xC5: // pextrw r/a, r, #8
 906         case 0xC6: // shufp[s|d] r, r, r/a, #8
 907           tail_size = 1;  // the imm8
 908           break;
 909         }
 910         break;
 911       case VEX_OPCODE_0F_3A:
 912         tail_size = 1;
 913         break;
 914     }
 915     ip++; // skip opcode
 916     debug_only(has_disp32 = true); // has both kinds of operands!
 917     break;
 918 
 919   case 0x62: // EVEX_4bytes
 920     assert((UseAVX > 0), "shouldn't have EVEX prefix");
 921     assert(ip == inst+1, "no prefixes allowed");
 922     // no EVEX collisions, all instructions that have 0x62 opcodes
 923     // have EVEX versions and are subopcodes of 0x66
 924     ip++; // skip P0 and exmaine W in P1
 925     is_64bit = ((VEX_W & *ip) == VEX_W);
 926     ip++; // move to P2
 927     ip++; // skip P2, move to opcode
 928     // To find the end of instruction (which == end_pc_operand).
 929     switch (0xFF & *ip) {
 930     case 0x22: // pinsrd r, r/a, #8
 931     case 0x61: // pcmpestri r, r/a, #8
 932     case 0x70: // pshufd r, r/a, #8
 933     case 0x73: // psrldq r, #8
 934       tail_size = 1;  // the imm8
 935       break;
 936     default:
 937       break;
 938     }
 939     ip++; // skip opcode
 940     debug_only(has_disp32 = true); // has both kinds of operands!
 941     break;
 942 
 943   case 0xD1: // sal a, 1; sar a, 1; shl a, 1; shr a, 1
 944   case 0xD3: // sal a, %cl; sar a, %cl; shl a, %cl; shr a, %cl
 945   case 0xD9: // fld_s a; fst_s a; fstp_s a; fldcw a
 946   case 0xDD: // fld_d a; fst_d a; fstp_d a
 947   case 0xDB: // fild_s a; fistp_s a; fld_x a; fstp_x a
 948   case 0xDF: // fild_d a; fistp_d a
 949   case 0xD8: // fadd_s a; fsubr_s a; fmul_s a; fdivr_s a; fcomp_s a
 950   case 0xDC: // fadd_d a; fsubr_d a; fmul_d a; fdivr_d a; fcomp_d a
 951   case 0xDE: // faddp_d a; fsubrp_d a; fmulp_d a; fdivrp_d a; fcompp_d a
 952     debug_only(has_disp32 = true);
 953     break;
 954 
 955   case 0xE8: // call rdisp32
 956   case 0xE9: // jmp  rdisp32
 957     if (which == end_pc_operand)  return ip + 4;
 958     assert(which == call32_operand, "call has no disp32 or imm");
 959     return ip;
 960 
 961   case 0xF0:                    // Lock
 962     assert(os::is_MP(), "only on MP");
 963     goto again_after_prefix;
 964 
 965   case 0xF3:                    // For SSE
 966   case 0xF2:                    // For SSE2
 967     switch (0xFF & *ip++) {
 968     case REX:
 969     case REX_B:
 970     case REX_X:
 971     case REX_XB:
 972     case REX_R:
 973     case REX_RB:
 974     case REX_RX:
 975     case REX_RXB:
 976     case REX_W:
 977     case REX_WB:
 978     case REX_WX:
 979     case REX_WXB:
 980     case REX_WR:
 981     case REX_WRB:
 982     case REX_WRX:
 983     case REX_WRXB:
 984       NOT_LP64(assert(false, "found 64bit prefix"));
 985       ip++;
 986     default:
 987       ip++;
 988     }
 989     debug_only(has_disp32 = true); // has both kinds of operands!
 990     break;
 991 
 992   default:
 993     ShouldNotReachHere();
 994 
 995 #undef REP8
 996 #undef REP16
 997   }
 998 
 999   assert(which != call32_operand, "instruction is not a call, jmp, or jcc");
1000 #ifdef _LP64
1001   assert(which != imm_operand, "instruction is not a movq reg, imm64");
1002 #else
1003   // assert(which != imm_operand || has_imm32, "instruction has no imm32 field");
1004   assert(which != imm_operand || has_disp32, "instruction has no imm32 field");
1005 #endif // LP64
1006   assert(which != disp32_operand || has_disp32, "instruction has no disp32 field");
1007 
1008   // parse the output of emit_operand
1009   int op2 = 0xFF & *ip++;
1010   int base = op2 & 0x07;
1011   int op3 = -1;
1012   const int b100 = 4;
1013   const int b101 = 5;
1014   if (base == b100 && (op2 >> 6) != 3) {
1015     op3 = 0xFF & *ip++;
1016     base = op3 & 0x07;   // refetch the base
1017   }
1018   // now ip points at the disp (if any)
1019 
1020   switch (op2 >> 6) {
1021   case 0:
1022     // [00 reg  100][ss index base]
1023     // [00 reg  100][00   100  esp]
1024     // [00 reg base]
1025     // [00 reg  100][ss index  101][disp32]
1026     // [00 reg  101]               [disp32]
1027 
1028     if (base == b101) {
1029       if (which == disp32_operand)
1030         return ip;              // caller wants the disp32
1031       ip += 4;                  // skip the disp32
1032     }
1033     break;
1034 
1035   case 1:
1036     // [01 reg  100][ss index base][disp8]
1037     // [01 reg  100][00   100  esp][disp8]
1038     // [01 reg base]               [disp8]
1039     ip += 1;                    // skip the disp8
1040     break;
1041 
1042   case 2:
1043     // [10 reg  100][ss index base][disp32]
1044     // [10 reg  100][00   100  esp][disp32]
1045     // [10 reg base]               [disp32]
1046     if (which == disp32_operand)
1047       return ip;                // caller wants the disp32
1048     ip += 4;                    // skip the disp32
1049     break;
1050 
1051   case 3:
1052     // [11 reg base]  (not a memory addressing mode)
1053     break;
1054   }
1055 
1056   if (which == end_pc_operand) {
1057     return ip + tail_size;
1058   }
1059 
1060 #ifdef _LP64
1061   assert(which == narrow_oop_operand && !is_64bit, "instruction is not a movl adr, imm32");
1062 #else
1063   assert(which == imm_operand, "instruction has only an imm field");
1064 #endif // LP64
1065   return ip;
1066 }
1067 
1068 address Assembler::locate_next_instruction(address inst) {
1069   // Secretly share code with locate_operand:
1070   return locate_operand(inst, end_pc_operand);
1071 }
1072 
1073 
1074 #ifdef ASSERT
1075 void Assembler::check_relocation(RelocationHolder const& rspec, int format) {
1076   address inst = inst_mark();
1077   assert(inst != NULL && inst < pc(), "must point to beginning of instruction");
1078   address opnd;
1079 
1080   Relocation* r = rspec.reloc();
1081   if (r->type() == relocInfo::none) {
1082     return;
1083   } else if (r->is_call() || format == call32_operand) {
1084     // assert(format == imm32_operand, "cannot specify a nonzero format");
1085     opnd = locate_operand(inst, call32_operand);
1086   } else if (r->is_data()) {
1087     assert(format == imm_operand || format == disp32_operand
1088            LP64_ONLY(|| format == narrow_oop_operand), "format ok");
1089     opnd = locate_operand(inst, (WhichOperand)format);
1090   } else {
1091     assert(format == imm_operand, "cannot specify a format");
1092     return;
1093   }
1094   assert(opnd == pc(), "must put operand where relocs can find it");
1095 }
1096 #endif // ASSERT
1097 
1098 void Assembler::emit_operand32(Register reg, Address adr) {
1099   assert(reg->encoding() < 8, "no extended registers");
1100   assert(!adr.base_needs_rex() && !adr.index_needs_rex(), "no extended registers");
1101   emit_operand(reg, adr._base, adr._index, adr._scale, adr._disp,
1102                adr._rspec);
1103 }
1104 
1105 void Assembler::emit_operand(Register reg, Address adr,
1106                              int rip_relative_correction) {
1107   emit_operand(reg, adr._base, adr._index, adr._scale, adr._disp,
1108                adr._rspec,
1109                rip_relative_correction);
1110 }
1111 
1112 void Assembler::emit_operand(XMMRegister reg, Address adr) {
1113   emit_operand(reg, adr._base, adr._index, adr._scale, adr._disp,
1114                adr._rspec);
1115 }
1116 
1117 // MMX operations
1118 void Assembler::emit_operand(MMXRegister reg, Address adr) {
1119   assert(!adr.base_needs_rex() && !adr.index_needs_rex(), "no extended registers");
1120   emit_operand((Register)reg, adr._base, adr._index, adr._scale, adr._disp, adr._rspec);
1121 }
1122 
1123 // work around gcc (3.2.1-7a) bug
1124 void Assembler::emit_operand(Address adr, MMXRegister reg) {
1125   assert(!adr.base_needs_rex() && !adr.index_needs_rex(), "no extended registers");
1126   emit_operand((Register)reg, adr._base, adr._index, adr._scale, adr._disp, adr._rspec);
1127 }
1128 
1129 
1130 void Assembler::emit_farith(int b1, int b2, int i) {
1131   assert(isByte(b1) && isByte(b2), "wrong opcode");
1132   assert(0 <= i &&  i < 8, "illegal stack offset");
1133   emit_int8(b1);
1134   emit_int8(b2 + i);
1135 }
1136 
1137 
1138 // Now the Assembler instructions (identical for 32/64 bits)
1139 
1140 void Assembler::adcl(Address dst, int32_t imm32) {
1141   InstructionMark im(this);
1142   prefix(dst);
1143   emit_arith_operand(0x81, rdx, dst, imm32);
1144 }
1145 
1146 void Assembler::adcl(Address dst, Register src) {
1147   InstructionMark im(this);
1148   prefix(dst, src);
1149   emit_int8(0x11);
1150   emit_operand(src, dst);
1151 }
1152 
1153 void Assembler::adcl(Register dst, int32_t imm32) {
1154   prefix(dst);
1155   emit_arith(0x81, 0xD0, dst, imm32);
1156 }
1157 
1158 void Assembler::adcl(Register dst, Address src) {
1159   InstructionMark im(this);
1160   prefix(src, dst);
1161   emit_int8(0x13);
1162   emit_operand(dst, src);
1163 }
1164 
1165 void Assembler::adcl(Register dst, Register src) {
1166   (void) prefix_and_encode(dst->encoding(), src->encoding());
1167   emit_arith(0x13, 0xC0, dst, src);
1168 }
1169 
1170 void Assembler::addl(Address dst, int32_t imm32) {
1171   InstructionMark im(this);
1172   prefix(dst);
1173   emit_arith_operand(0x81, rax, dst, imm32);
1174 }
1175 
1176 void Assembler::addl(Address dst, Register src) {
1177   InstructionMark im(this);
1178   prefix(dst, src);
1179   emit_int8(0x01);
1180   emit_operand(src, dst);
1181 }
1182 
1183 void Assembler::addl(Register dst, int32_t imm32) {
1184   prefix(dst);
1185   emit_arith(0x81, 0xC0, dst, imm32);
1186 }
1187 
1188 void Assembler::addl(Register dst, Address src) {
1189   InstructionMark im(this);
1190   prefix(src, dst);
1191   emit_int8(0x03);
1192   emit_operand(dst, src);
1193 }
1194 
1195 void Assembler::addl(Register dst, Register src) {
1196   (void) prefix_and_encode(dst->encoding(), src->encoding());
1197   emit_arith(0x03, 0xC0, dst, src);
1198 }
1199 
1200 void Assembler::addr_nop_4() {
1201   assert(UseAddressNop, "no CPU support");
1202   // 4 bytes: NOP DWORD PTR [EAX+0]
1203   emit_int8(0x0F);
1204   emit_int8(0x1F);
1205   emit_int8(0x40); // emit_rm(cbuf, 0x1, EAX_enc, EAX_enc);
1206   emit_int8(0);    // 8-bits offset (1 byte)
1207 }
1208 
1209 void Assembler::addr_nop_5() {
1210   assert(UseAddressNop, "no CPU support");
1211   // 5 bytes: NOP DWORD PTR [EAX+EAX*0+0] 8-bits offset
1212   emit_int8(0x0F);
1213   emit_int8(0x1F);
1214   emit_int8(0x44); // emit_rm(cbuf, 0x1, EAX_enc, 0x4);
1215   emit_int8(0x00); // emit_rm(cbuf, 0x0, EAX_enc, EAX_enc);
1216   emit_int8(0);    // 8-bits offset (1 byte)
1217 }
1218 
1219 void Assembler::addr_nop_7() {
1220   assert(UseAddressNop, "no CPU support");
1221   // 7 bytes: NOP DWORD PTR [EAX+0] 32-bits offset
1222   emit_int8(0x0F);
1223   emit_int8(0x1F);
1224   emit_int8((unsigned char)0x80);
1225                    // emit_rm(cbuf, 0x2, EAX_enc, EAX_enc);
1226   emit_int32(0);   // 32-bits offset (4 bytes)
1227 }
1228 
1229 void Assembler::addr_nop_8() {
1230   assert(UseAddressNop, "no CPU support");
1231   // 8 bytes: NOP DWORD PTR [EAX+EAX*0+0] 32-bits offset
1232   emit_int8(0x0F);
1233   emit_int8(0x1F);
1234   emit_int8((unsigned char)0x84);
1235                    // emit_rm(cbuf, 0x2, EAX_enc, 0x4);
1236   emit_int8(0x00); // emit_rm(cbuf, 0x0, EAX_enc, EAX_enc);
1237   emit_int32(0);   // 32-bits offset (4 bytes)
1238 }
1239 
1240 void Assembler::addsd(XMMRegister dst, XMMRegister src) {
1241   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1242   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
1243   attributes.set_rex_vex_w_reverted();
1244   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
1245   emit_int8(0x58);
1246   emit_int8((unsigned char)(0xC0 | encode));
1247 }
1248 
1249 void Assembler::addsd(XMMRegister dst, Address src) {
1250   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1251   InstructionMark im(this);
1252   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
1253   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
1254   attributes.set_rex_vex_w_reverted();
1255   simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
1256   emit_int8(0x58);
1257   emit_operand(dst, src);
1258 }
1259 
1260 void Assembler::addss(XMMRegister dst, XMMRegister src) {
1261   NOT_LP64(assert(VM_Version::supports_sse(), ""));
1262   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
1263   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
1264   emit_int8(0x58);
1265   emit_int8((unsigned char)(0xC0 | encode));
1266 }
1267 
1268 void Assembler::addss(XMMRegister dst, Address src) {
1269   NOT_LP64(assert(VM_Version::supports_sse(), ""));
1270   InstructionMark im(this);
1271   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
1272   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
1273   simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
1274   emit_int8(0x58);
1275   emit_operand(dst, src);
1276 }
1277 
1278 void Assembler::aesdec(XMMRegister dst, Address src) {
1279   assert(VM_Version::supports_aes(), "");
1280   InstructionMark im(this);
1281   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
1282   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
1283   emit_int8((unsigned char)0xDE);
1284   emit_operand(dst, src);
1285 }
1286 
1287 void Assembler::aesdec(XMMRegister dst, XMMRegister src) {
1288   assert(VM_Version::supports_aes(), "");
1289   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
1290   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
1291   emit_int8((unsigned char)0xDE);
1292   emit_int8(0xC0 | encode);
1293 }
1294 
1295 void Assembler::aesdeclast(XMMRegister dst, Address src) {
1296   assert(VM_Version::supports_aes(), "");
1297   InstructionMark im(this);
1298   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
1299   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
1300   emit_int8((unsigned char)0xDF);
1301   emit_operand(dst, src);
1302 }
1303 
1304 void Assembler::aesdeclast(XMMRegister dst, XMMRegister src) {
1305   assert(VM_Version::supports_aes(), "");
1306   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
1307   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
1308   emit_int8((unsigned char)0xDF);
1309   emit_int8((unsigned char)(0xC0 | encode));
1310 }
1311 
1312 void Assembler::aesenc(XMMRegister dst, Address src) {
1313   assert(VM_Version::supports_aes(), "");
1314   InstructionMark im(this);
1315   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
1316   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
1317   emit_int8((unsigned char)0xDC);
1318   emit_operand(dst, src);
1319 }
1320 
1321 void Assembler::aesenc(XMMRegister dst, XMMRegister src) {
1322   assert(VM_Version::supports_aes(), "");
1323   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
1324   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
1325   emit_int8((unsigned char)0xDC);
1326   emit_int8(0xC0 | encode);
1327 }
1328 
1329 void Assembler::aesenclast(XMMRegister dst, Address src) {
1330   assert(VM_Version::supports_aes(), "");
1331   InstructionMark im(this);
1332   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
1333   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
1334   emit_int8((unsigned char)0xDD);
1335   emit_operand(dst, src);
1336 }
1337 
1338 void Assembler::aesenclast(XMMRegister dst, XMMRegister src) {
1339   assert(VM_Version::supports_aes(), "");
1340   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
1341   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
1342   emit_int8((unsigned char)0xDD);
1343   emit_int8((unsigned char)(0xC0 | encode));
1344 }
1345 
1346 void Assembler::andl(Address dst, int32_t imm32) {
1347   InstructionMark im(this);
1348   prefix(dst);
1349   emit_int8((unsigned char)0x81);
1350   emit_operand(rsp, dst, 4);
1351   emit_int32(imm32);
1352 }
1353 
1354 void Assembler::andl(Register dst, int32_t imm32) {
1355   prefix(dst);
1356   emit_arith(0x81, 0xE0, dst, imm32);
1357 }
1358 
1359 void Assembler::andl(Register dst, Address src) {
1360   InstructionMark im(this);
1361   prefix(src, dst);
1362   emit_int8(0x23);
1363   emit_operand(dst, src);
1364 }
1365 
1366 void Assembler::andl(Register dst, Register src) {
1367   (void) prefix_and_encode(dst->encoding(), src->encoding());
1368   emit_arith(0x23, 0xC0, dst, src);
1369 }
1370 
1371 void Assembler::andnl(Register dst, Register src1, Register src2) {
1372   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1373   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
1374   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
1375   emit_int8((unsigned char)0xF2);
1376   emit_int8((unsigned char)(0xC0 | encode));
1377 }
1378 
1379 void Assembler::andnl(Register dst, Register src1, Address src2) {
1380   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1381   InstructionMark im(this);
1382   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
1383   vex_prefix(src2, src1->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
1384   emit_int8((unsigned char)0xF2);
1385   emit_operand(dst, src2);
1386 }
1387 
1388 void Assembler::bsfl(Register dst, Register src) {
1389   int encode = prefix_and_encode(dst->encoding(), src->encoding());
1390   emit_int8(0x0F);
1391   emit_int8((unsigned char)0xBC);
1392   emit_int8((unsigned char)(0xC0 | encode));
1393 }
1394 
1395 void Assembler::bsrl(Register dst, Register src) {
1396   int encode = prefix_and_encode(dst->encoding(), src->encoding());
1397   emit_int8(0x0F);
1398   emit_int8((unsigned char)0xBD);
1399   emit_int8((unsigned char)(0xC0 | encode));
1400 }
1401 
1402 void Assembler::bswapl(Register reg) { // bswap
1403   int encode = prefix_and_encode(reg->encoding());
1404   emit_int8(0x0F);
1405   emit_int8((unsigned char)(0xC8 | encode));
1406 }
1407 
1408 void Assembler::blsil(Register dst, Register src) {
1409   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1410   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
1411   int encode = vex_prefix_and_encode(rbx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
1412   emit_int8((unsigned char)0xF3);
1413   emit_int8((unsigned char)(0xC0 | encode));
1414 }
1415 
1416 void Assembler::blsil(Register dst, Address src) {
1417   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1418   InstructionMark im(this);
1419   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
1420   vex_prefix(src, dst->encoding(), rbx->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
1421   emit_int8((unsigned char)0xF3);
1422   emit_operand(rbx, src);
1423 }
1424 
1425 void Assembler::blsmskl(Register dst, Register src) {
1426   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1427   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
1428   int encode = vex_prefix_and_encode(rdx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
1429   emit_int8((unsigned char)0xF3);
1430   emit_int8((unsigned char)(0xC0 | encode));
1431 }
1432 
1433 void Assembler::blsmskl(Register dst, Address src) {
1434   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1435   InstructionMark im(this);
1436   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
1437   vex_prefix(src, dst->encoding(), rdx->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
1438   emit_int8((unsigned char)0xF3);
1439   emit_operand(rdx, src);
1440 }
1441 
1442 void Assembler::blsrl(Register dst, Register src) {
1443   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1444   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
1445   int encode = vex_prefix_and_encode(rcx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
1446   emit_int8((unsigned char)0xF3);
1447   emit_int8((unsigned char)(0xC0 | encode));
1448 }
1449 
1450 void Assembler::blsrl(Register dst, Address src) {
1451   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1452   InstructionMark im(this);
1453   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
1454   vex_prefix(src, dst->encoding(), rcx->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
1455   emit_int8((unsigned char)0xF3);
1456   emit_operand(rcx, src);
1457 }
1458 
1459 void Assembler::call(Label& L, relocInfo::relocType rtype) {
1460   // suspect disp32 is always good
1461   int operand = LP64_ONLY(disp32_operand) NOT_LP64(imm_operand);
1462 
1463   if (L.is_bound()) {
1464     const int long_size = 5;
1465     int offs = (int)( target(L) - pc() );
1466     assert(offs <= 0, "assembler error");
1467     InstructionMark im(this);
1468     // 1110 1000 #32-bit disp
1469     emit_int8((unsigned char)0xE8);
1470     emit_data(offs - long_size, rtype, operand);
1471   } else {
1472     InstructionMark im(this);
1473     // 1110 1000 #32-bit disp
1474     L.add_patch_at(code(), locator());
1475 
1476     emit_int8((unsigned char)0xE8);
1477     emit_data(int(0), rtype, operand);
1478   }
1479 }
1480 
1481 void Assembler::call(Register dst) {
1482   int encode = prefix_and_encode(dst->encoding());
1483   emit_int8((unsigned char)0xFF);
1484   emit_int8((unsigned char)(0xD0 | encode));
1485 }
1486 
1487 
1488 void Assembler::call(Address adr) {
1489   InstructionMark im(this);
1490   prefix(adr);
1491   emit_int8((unsigned char)0xFF);
1492   emit_operand(rdx, adr);
1493 }
1494 
1495 void Assembler::call_literal(address entry, RelocationHolder const& rspec) {
1496   assert(entry != NULL, "call most probably wrong");
1497   InstructionMark im(this);
1498   emit_int8((unsigned char)0xE8);
1499   intptr_t disp = entry - (pc() + sizeof(int32_t));
1500   assert(is_simm32(disp), "must be 32bit offset (call2)");
1501   // Technically, should use call32_operand, but this format is
1502   // implied by the fact that we're emitting a call instruction.
1503 
1504   int operand = LP64_ONLY(disp32_operand) NOT_LP64(call32_operand);
1505   emit_data((int) disp, rspec, operand);
1506 }
1507 
1508 void Assembler::cdql() {
1509   emit_int8((unsigned char)0x99);
1510 }
1511 
1512 void Assembler::cld() {
1513   emit_int8((unsigned char)0xFC);
1514 }
1515 
1516 void Assembler::cmovl(Condition cc, Register dst, Register src) {
1517   NOT_LP64(guarantee(VM_Version::supports_cmov(), "illegal instruction"));
1518   int encode = prefix_and_encode(dst->encoding(), src->encoding());
1519   emit_int8(0x0F);
1520   emit_int8(0x40 | cc);
1521   emit_int8((unsigned char)(0xC0 | encode));
1522 }
1523 
1524 
1525 void Assembler::cmovl(Condition cc, Register dst, Address src) {
1526   NOT_LP64(guarantee(VM_Version::supports_cmov(), "illegal instruction"));
1527   prefix(src, dst);
1528   emit_int8(0x0F);
1529   emit_int8(0x40 | cc);
1530   emit_operand(dst, src);
1531 }
1532 
1533 void Assembler::cmpb(Address dst, int imm8) {
1534   InstructionMark im(this);
1535   prefix(dst);
1536   emit_int8((unsigned char)0x80);
1537   emit_operand(rdi, dst, 1);
1538   emit_int8(imm8);
1539 }
1540 
1541 void Assembler::cmpl(Address dst, int32_t imm32) {
1542   InstructionMark im(this);
1543   prefix(dst);
1544   emit_int8((unsigned char)0x81);
1545   emit_operand(rdi, dst, 4);
1546   emit_int32(imm32);
1547 }
1548 
1549 void Assembler::cmpl(Register dst, int32_t imm32) {
1550   prefix(dst);
1551   emit_arith(0x81, 0xF8, dst, imm32);
1552 }
1553 
1554 void Assembler::cmpl(Register dst, Register src) {
1555   (void) prefix_and_encode(dst->encoding(), src->encoding());
1556   emit_arith(0x3B, 0xC0, dst, src);
1557 }
1558 
1559 void Assembler::cmpl(Register dst, Address  src) {
1560   InstructionMark im(this);
1561   prefix(src, dst);
1562   emit_int8((unsigned char)0x3B);
1563   emit_operand(dst, src);
1564 }
1565 
1566 void Assembler::cmpw(Address dst, int imm16) {
1567   InstructionMark im(this);
1568   assert(!dst.base_needs_rex() && !dst.index_needs_rex(), "no extended registers");
1569   emit_int8(0x66);
1570   emit_int8((unsigned char)0x81);
1571   emit_operand(rdi, dst, 2);
1572   emit_int16(imm16);
1573 }
1574 
1575 // The 32-bit cmpxchg compares the value at adr with the contents of rax,
1576 // and stores reg into adr if so; otherwise, the value at adr is loaded into rax,.
1577 // The ZF is set if the compared values were equal, and cleared otherwise.
1578 void Assembler::cmpxchgl(Register reg, Address adr) { // cmpxchg
1579   InstructionMark im(this);
1580   prefix(adr, reg);
1581   emit_int8(0x0F);
1582   emit_int8((unsigned char)0xB1);
1583   emit_operand(reg, adr);
1584 }
1585 
1586 // The 8-bit cmpxchg compares the value at adr with the contents of rax,
1587 // and stores reg into adr if so; otherwise, the value at adr is loaded into rax,.
1588 // The ZF is set if the compared values were equal, and cleared otherwise.
1589 void Assembler::cmpxchgb(Register reg, Address adr) { // cmpxchg
1590   InstructionMark im(this);
1591   prefix(adr, reg, true);
1592   emit_int8(0x0F);
1593   emit_int8((unsigned char)0xB0);
1594   emit_operand(reg, adr);
1595 }
1596 
1597 void Assembler::comisd(XMMRegister dst, Address src) {
1598   // NOTE: dbx seems to decode this as comiss even though the
1599   // 0x66 is there. Strangly ucomisd comes out correct
1600   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1601   InstructionMark im(this);
1602   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);;
1603   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
1604   attributes.set_rex_vex_w_reverted();
1605   simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
1606   emit_int8(0x2F);
1607   emit_operand(dst, src);
1608 }
1609 
1610 void Assembler::comisd(XMMRegister dst, XMMRegister src) {
1611   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1612   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1613   attributes.set_rex_vex_w_reverted();
1614   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
1615   emit_int8(0x2F);
1616   emit_int8((unsigned char)(0xC0 | encode));
1617 }
1618 
1619 void Assembler::comiss(XMMRegister dst, Address src) {
1620   NOT_LP64(assert(VM_Version::supports_sse(), ""));
1621   InstructionMark im(this);
1622   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1623   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
1624   simd_prefix(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
1625   emit_int8(0x2F);
1626   emit_operand(dst, src);
1627 }
1628 
1629 void Assembler::comiss(XMMRegister dst, XMMRegister src) {
1630   NOT_LP64(assert(VM_Version::supports_sse(), ""));
1631   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1632   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
1633   emit_int8(0x2F);
1634   emit_int8((unsigned char)(0xC0 | encode));
1635 }
1636 
1637 void Assembler::cpuid() {
1638   emit_int8(0x0F);
1639   emit_int8((unsigned char)0xA2);
1640 }
1641 
1642 // Opcode / Instruction                      Op /  En  64 - Bit Mode     Compat / Leg Mode Description                  Implemented
1643 // F2 0F 38 F0 / r       CRC32 r32, r / m8   RM        Valid             Valid             Accumulate CRC32 on r / m8.  v
1644 // F2 REX 0F 38 F0 / r   CRC32 r32, r / m8*  RM        Valid             N.E.              Accumulate CRC32 on r / m8.  -
1645 // F2 REX.W 0F 38 F0 / r CRC32 r64, r / m8   RM        Valid             N.E.              Accumulate CRC32 on r / m8.  -
1646 //
1647 // F2 0F 38 F1 / r       CRC32 r32, r / m16  RM        Valid             Valid             Accumulate CRC32 on r / m16. v
1648 //
1649 // F2 0F 38 F1 / r       CRC32 r32, r / m32  RM        Valid             Valid             Accumulate CRC32 on r / m32. v
1650 //
1651 // F2 REX.W 0F 38 F1 / r CRC32 r64, r / m64  RM        Valid             N.E.              Accumulate CRC32 on r / m64. v
1652 void Assembler::crc32(Register crc, Register v, int8_t sizeInBytes) {
1653   assert(VM_Version::supports_sse4_2(), "");
1654   int8_t w = 0x01;
1655   Prefix p = Prefix_EMPTY;
1656 
1657   emit_int8((int8_t)0xF2);
1658   switch (sizeInBytes) {
1659   case 1:
1660     w = 0;
1661     break;
1662   case 2:
1663   case 4:
1664     break;
1665   LP64_ONLY(case 8:)
1666     // This instruction is not valid in 32 bits
1667     // Note:
1668     // http://www.intel.com/content/dam/www/public/us/en/documents/manuals/64-ia-32-architectures-software-developer-instruction-set-reference-manual-325383.pdf
1669     //
1670     // Page B - 72   Vol. 2C says
1671     // qwreg2 to qwreg            1111 0010 : 0100 1R0B : 0000 1111 : 0011 1000 : 1111 0000 : 11 qwreg1 qwreg2
1672     // mem64 to qwreg             1111 0010 : 0100 1R0B : 0000 1111 : 0011 1000 : 1111 0000 : mod qwreg r / m
1673     //                                                                            F0!!!
1674     // while 3 - 208 Vol. 2A
1675     // F2 REX.W 0F 38 F1 / r       CRC32 r64, r / m64             RM         Valid      N.E.Accumulate CRC32 on r / m64.
1676     //
1677     // the 0 on a last bit is reserved for a different flavor of this instruction :
1678     // F2 REX.W 0F 38 F0 / r       CRC32 r64, r / m8              RM         Valid      N.E.Accumulate CRC32 on r / m8.
1679     p = REX_W;
1680     break;
1681   default:
1682     assert(0, "Unsupported value for a sizeInBytes argument");
1683     break;
1684   }
1685   LP64_ONLY(prefix(crc, v, p);)
1686   emit_int8((int8_t)0x0F);
1687   emit_int8(0x38);
1688   emit_int8((int8_t)(0xF0 | w));
1689   emit_int8(0xC0 | ((crc->encoding() & 0x7) << 3) | (v->encoding() & 7));
1690 }
1691 
1692 void Assembler::crc32(Register crc, Address adr, int8_t sizeInBytes) {
1693   assert(VM_Version::supports_sse4_2(), "");
1694   InstructionMark im(this);
1695   int8_t w = 0x01;
1696   Prefix p = Prefix_EMPTY;
1697 
1698   emit_int8((int8_t)0xF2);
1699   switch (sizeInBytes) {
1700   case 1:
1701     w = 0;
1702     break;
1703   case 2:
1704   case 4:
1705     break;
1706   LP64_ONLY(case 8:)
1707     // This instruction is not valid in 32 bits
1708     p = REX_W;
1709     break;
1710   default:
1711     assert(0, "Unsupported value for a sizeInBytes argument");
1712     break;
1713   }
1714   LP64_ONLY(prefix(crc, adr, p);)
1715   emit_int8((int8_t)0x0F);
1716   emit_int8(0x38);
1717   emit_int8((int8_t)(0xF0 | w));
1718   emit_operand(crc, adr);
1719 }
1720 
1721 void Assembler::cvtdq2pd(XMMRegister dst, XMMRegister src) {
1722   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1723   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
1724   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
1725   emit_int8((unsigned char)0xE6);
1726   emit_int8((unsigned char)(0xC0 | encode));
1727 }
1728 
1729 void Assembler::cvtdq2ps(XMMRegister dst, XMMRegister src) {
1730   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1731   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
1732   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
1733   emit_int8(0x5B);
1734   emit_int8((unsigned char)(0xC0 | encode));
1735 }
1736 
1737 void Assembler::cvtsd2ss(XMMRegister dst, XMMRegister src) {
1738   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1739   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
1740   attributes.set_rex_vex_w_reverted();
1741   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
1742   emit_int8(0x5A);
1743   emit_int8((unsigned char)(0xC0 | encode));
1744 }
1745 
1746 void Assembler::cvtsd2ss(XMMRegister dst, Address src) {
1747   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1748   InstructionMark im(this);
1749   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
1750   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
1751   attributes.set_rex_vex_w_reverted();
1752   simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
1753   emit_int8(0x5A);
1754   emit_operand(dst, src);
1755 }
1756 
1757 void Assembler::cvtsi2sdl(XMMRegister dst, Register src) {
1758   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1759   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1760   int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
1761   emit_int8(0x2A);
1762   emit_int8((unsigned char)(0xC0 | encode));
1763 }
1764 
1765 void Assembler::cvtsi2sdl(XMMRegister dst, Address src) {
1766   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1767   InstructionMark im(this);
1768   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1769   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
1770   simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
1771   emit_int8(0x2A);
1772   emit_operand(dst, src);
1773 }
1774 
1775 void Assembler::cvtsi2ssl(XMMRegister dst, Register src) {
1776   NOT_LP64(assert(VM_Version::supports_sse(), ""));
1777   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1778   int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
1779   emit_int8(0x2A);
1780   emit_int8((unsigned char)(0xC0 | encode));
1781 }
1782 
1783 void Assembler::cvtsi2ssl(XMMRegister dst, Address src) {
1784   NOT_LP64(assert(VM_Version::supports_sse(), ""));
1785   InstructionMark im(this);
1786   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1787   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
1788   simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
1789   emit_int8(0x2A);
1790   emit_operand(dst, src);
1791 }
1792 
1793 void Assembler::cvtsi2ssq(XMMRegister dst, Register src) {
1794   NOT_LP64(assert(VM_Version::supports_sse(), ""));
1795   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1796   int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
1797   emit_int8(0x2A);
1798   emit_int8((unsigned char)(0xC0 | encode));
1799 }
1800 
1801 void Assembler::cvtss2sd(XMMRegister dst, XMMRegister src) {
1802   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1803   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
1804   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
1805   emit_int8(0x5A);
1806   emit_int8((unsigned char)(0xC0 | encode));
1807 }
1808 
1809 void Assembler::cvtss2sd(XMMRegister dst, Address src) {
1810   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1811   InstructionMark im(this);
1812   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
1813   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
1814   simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
1815   emit_int8(0x5A);
1816   emit_operand(dst, src);
1817 }
1818 
1819 
1820 void Assembler::cvttsd2sil(Register dst, XMMRegister src) {
1821   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1822   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1823   int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
1824   emit_int8(0x2C);
1825   emit_int8((unsigned char)(0xC0 | encode));
1826 }
1827 
1828 void Assembler::cvttss2sil(Register dst, XMMRegister src) {
1829   NOT_LP64(assert(VM_Version::supports_sse(), ""));
1830   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1831   int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
1832   emit_int8(0x2C);
1833   emit_int8((unsigned char)(0xC0 | encode));
1834 }
1835 
1836 void Assembler::cvttpd2dq(XMMRegister dst, XMMRegister src) {
1837   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1838   int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_128bit;
1839   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
1840   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
1841   emit_int8((unsigned char)0xE6);
1842   emit_int8((unsigned char)(0xC0 | encode));
1843 }
1844 
1845 void Assembler::decl(Address dst) {
1846   // Don't use it directly. Use MacroAssembler::decrement() instead.
1847   InstructionMark im(this);
1848   prefix(dst);
1849   emit_int8((unsigned char)0xFF);
1850   emit_operand(rcx, dst);
1851 }
1852 
1853 void Assembler::divsd(XMMRegister dst, Address src) {
1854   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1855   InstructionMark im(this);
1856   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
1857   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
1858   attributes.set_rex_vex_w_reverted();
1859   simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
1860   emit_int8(0x5E);
1861   emit_operand(dst, src);
1862 }
1863 
1864 void Assembler::divsd(XMMRegister dst, XMMRegister src) {
1865   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1866   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
1867   attributes.set_rex_vex_w_reverted();
1868   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
1869   emit_int8(0x5E);
1870   emit_int8((unsigned char)(0xC0 | encode));
1871 }
1872 
1873 void Assembler::divss(XMMRegister dst, Address src) {
1874   NOT_LP64(assert(VM_Version::supports_sse(), ""));
1875   InstructionMark im(this);
1876   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
1877   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
1878   simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
1879   emit_int8(0x5E);
1880   emit_operand(dst, src);
1881 }
1882 
1883 void Assembler::divss(XMMRegister dst, XMMRegister src) {
1884   NOT_LP64(assert(VM_Version::supports_sse(), ""));
1885   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
1886   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
1887   emit_int8(0x5E);
1888   emit_int8((unsigned char)(0xC0 | encode));
1889 }
1890 
1891 void Assembler::emms() {
1892   NOT_LP64(assert(VM_Version::supports_mmx(), ""));
1893   emit_int8(0x0F);
1894   emit_int8(0x77);
1895 }
1896 
1897 void Assembler::hlt() {
1898   emit_int8((unsigned char)0xF4);
1899 }
1900 
1901 void Assembler::idivl(Register src) {
1902   int encode = prefix_and_encode(src->encoding());
1903   emit_int8((unsigned char)0xF7);
1904   emit_int8((unsigned char)(0xF8 | encode));
1905 }
1906 
1907 void Assembler::divl(Register src) { // Unsigned
1908   int encode = prefix_and_encode(src->encoding());
1909   emit_int8((unsigned char)0xF7);
1910   emit_int8((unsigned char)(0xF0 | encode));
1911 }
1912 
1913 void Assembler::imull(Register src) {
1914   int encode = prefix_and_encode(src->encoding());
1915   emit_int8((unsigned char)0xF7);
1916   emit_int8((unsigned char)(0xE8 | encode));
1917 }
1918 
1919 void Assembler::imull(Register dst, Register src) {
1920   int encode = prefix_and_encode(dst->encoding(), src->encoding());
1921   emit_int8(0x0F);
1922   emit_int8((unsigned char)0xAF);
1923   emit_int8((unsigned char)(0xC0 | encode));
1924 }
1925 
1926 
1927 void Assembler::imull(Register dst, Register src, int value) {
1928   int encode = prefix_and_encode(dst->encoding(), src->encoding());
1929   if (is8bit(value)) {
1930     emit_int8(0x6B);
1931     emit_int8((unsigned char)(0xC0 | encode));
1932     emit_int8(value & 0xFF);
1933   } else {
1934     emit_int8(0x69);
1935     emit_int8((unsigned char)(0xC0 | encode));
1936     emit_int32(value);
1937   }
1938 }
1939 
1940 void Assembler::imull(Register dst, Address src) {
1941   InstructionMark im(this);
1942   prefix(src, dst);
1943   emit_int8(0x0F);
1944   emit_int8((unsigned char) 0xAF);
1945   emit_operand(dst, src);
1946 }
1947 
1948 
1949 void Assembler::incl(Address dst) {
1950   // Don't use it directly. Use MacroAssembler::increment() instead.
1951   InstructionMark im(this);
1952   prefix(dst);
1953   emit_int8((unsigned char)0xFF);
1954   emit_operand(rax, dst);
1955 }
1956 
1957 void Assembler::jcc(Condition cc, Label& L, bool maybe_short) {
1958   InstructionMark im(this);
1959   assert((0 <= cc) && (cc < 16), "illegal cc");
1960   if (L.is_bound()) {
1961     address dst = target(L);
1962     assert(dst != NULL, "jcc most probably wrong");
1963 
1964     const int short_size = 2;
1965     const int long_size = 6;
1966     intptr_t offs = (intptr_t)dst - (intptr_t)pc();
1967     if (maybe_short && is8bit(offs - short_size)) {
1968       // 0111 tttn #8-bit disp
1969       emit_int8(0x70 | cc);
1970       emit_int8((offs - short_size) & 0xFF);
1971     } else {
1972       // 0000 1111 1000 tttn #32-bit disp
1973       assert(is_simm32(offs - long_size),
1974              "must be 32bit offset (call4)");
1975       emit_int8(0x0F);
1976       emit_int8((unsigned char)(0x80 | cc));
1977       emit_int32(offs - long_size);
1978     }
1979   } else {
1980     // Note: could eliminate cond. jumps to this jump if condition
1981     //       is the same however, seems to be rather unlikely case.
1982     // Note: use jccb() if label to be bound is very close to get
1983     //       an 8-bit displacement
1984     L.add_patch_at(code(), locator());
1985     emit_int8(0x0F);
1986     emit_int8((unsigned char)(0x80 | cc));
1987     emit_int32(0);
1988   }
1989 }
1990 
1991 void Assembler::jccb(Condition cc, Label& L) {
1992   if (L.is_bound()) {
1993     const int short_size = 2;
1994     address entry = target(L);
1995 #ifdef ASSERT
1996     intptr_t dist = (intptr_t)entry - ((intptr_t)pc() + short_size);
1997     intptr_t delta = short_branch_delta();
1998     if (delta != 0) {
1999       dist += (dist < 0 ? (-delta) :delta);
2000     }
2001     assert(is8bit(dist), "Dispacement too large for a short jmp");
2002 #endif
2003     intptr_t offs = (intptr_t)entry - (intptr_t)pc();
2004     // 0111 tttn #8-bit disp
2005     emit_int8(0x70 | cc);
2006     emit_int8((offs - short_size) & 0xFF);
2007   } else {
2008     InstructionMark im(this);
2009     L.add_patch_at(code(), locator());
2010     emit_int8(0x70 | cc);
2011     emit_int8(0);
2012   }
2013 }
2014 
2015 void Assembler::jmp(Address adr) {
2016   InstructionMark im(this);
2017   prefix(adr);
2018   emit_int8((unsigned char)0xFF);
2019   emit_operand(rsp, adr);
2020 }
2021 
2022 void Assembler::jmp(Label& L, bool maybe_short) {
2023   if (L.is_bound()) {
2024     address entry = target(L);
2025     assert(entry != NULL, "jmp most probably wrong");
2026     InstructionMark im(this);
2027     const int short_size = 2;
2028     const int long_size = 5;
2029     intptr_t offs = entry - pc();
2030     if (maybe_short && is8bit(offs - short_size)) {
2031       emit_int8((unsigned char)0xEB);
2032       emit_int8((offs - short_size) & 0xFF);
2033     } else {
2034       emit_int8((unsigned char)0xE9);
2035       emit_int32(offs - long_size);
2036     }
2037   } else {
2038     // By default, forward jumps are always 32-bit displacements, since
2039     // we can't yet know where the label will be bound.  If you're sure that
2040     // the forward jump will not run beyond 256 bytes, use jmpb to
2041     // force an 8-bit displacement.
2042     InstructionMark im(this);
2043     L.add_patch_at(code(), locator());
2044     emit_int8((unsigned char)0xE9);
2045     emit_int32(0);
2046   }
2047 }
2048 
2049 void Assembler::jmp(Register entry) {
2050   int encode = prefix_and_encode(entry->encoding());
2051   emit_int8((unsigned char)0xFF);
2052   emit_int8((unsigned char)(0xE0 | encode));
2053 }
2054 
2055 void Assembler::jmp_literal(address dest, RelocationHolder const& rspec) {
2056   InstructionMark im(this);
2057   emit_int8((unsigned char)0xE9);
2058   assert(dest != NULL, "must have a target");
2059   intptr_t disp = dest - (pc() + sizeof(int32_t));
2060   assert(is_simm32(disp), "must be 32bit offset (jmp)");
2061   emit_data(disp, rspec.reloc(), call32_operand);
2062 }
2063 
2064 void Assembler::jmpb(Label& L) {
2065   if (L.is_bound()) {
2066     const int short_size = 2;
2067     address entry = target(L);
2068     assert(entry != NULL, "jmp most probably wrong");
2069 #ifdef ASSERT
2070     intptr_t dist = (intptr_t)entry - ((intptr_t)pc() + short_size);
2071     intptr_t delta = short_branch_delta();
2072     if (delta != 0) {
2073       dist += (dist < 0 ? (-delta) :delta);
2074     }
2075     assert(is8bit(dist), "Dispacement too large for a short jmp");
2076 #endif
2077     intptr_t offs = entry - pc();
2078     emit_int8((unsigned char)0xEB);
2079     emit_int8((offs - short_size) & 0xFF);
2080   } else {
2081     InstructionMark im(this);
2082     L.add_patch_at(code(), locator());
2083     emit_int8((unsigned char)0xEB);
2084     emit_int8(0);
2085   }
2086 }
2087 
2088 void Assembler::ldmxcsr( Address src) {
2089   NOT_LP64(assert(VM_Version::supports_sse(), ""));
2090   InstructionMark im(this);
2091   prefix(src);
2092   emit_int8(0x0F);
2093   emit_int8((unsigned char)0xAE);
2094   emit_operand(as_Register(2), src);
2095 }
2096 
2097 void Assembler::leal(Register dst, Address src) {
2098   InstructionMark im(this);
2099 #ifdef _LP64
2100   emit_int8(0x67); // addr32
2101   prefix(src, dst);
2102 #endif // LP64
2103   emit_int8((unsigned char)0x8D);
2104   emit_operand(dst, src);
2105 }
2106 
2107 void Assembler::lfence() {
2108   emit_int8(0x0F);
2109   emit_int8((unsigned char)0xAE);
2110   emit_int8((unsigned char)0xE8);
2111 }
2112 
2113 void Assembler::lock() {
2114   emit_int8((unsigned char)0xF0);
2115 }
2116 
2117 void Assembler::lzcntl(Register dst, Register src) {
2118   assert(VM_Version::supports_lzcnt(), "encoding is treated as BSR");
2119   emit_int8((unsigned char)0xF3);
2120   int encode = prefix_and_encode(dst->encoding(), src->encoding());
2121   emit_int8(0x0F);
2122   emit_int8((unsigned char)0xBD);
2123   emit_int8((unsigned char)(0xC0 | encode));
2124 }
2125 
2126 // Emit mfence instruction
2127 void Assembler::mfence() {
2128   NOT_LP64(assert(VM_Version::supports_sse2(), "unsupported");)
2129   emit_int8(0x0F);
2130   emit_int8((unsigned char)0xAE);
2131   emit_int8((unsigned char)0xF0);
2132 }
2133 
2134 void Assembler::mov(Register dst, Register src) {
2135   LP64_ONLY(movq(dst, src)) NOT_LP64(movl(dst, src));
2136 }
2137 
2138 void Assembler::movapd(XMMRegister dst, XMMRegister src) {
2139   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2140   int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_128bit;
2141   InstructionAttr attributes(vector_len, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
2142   attributes.set_rex_vex_w_reverted();
2143   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2144   emit_int8(0x28);
2145   emit_int8((unsigned char)(0xC0 | encode));
2146 }
2147 
2148 void Assembler::movaps(XMMRegister dst, XMMRegister src) {
2149   NOT_LP64(assert(VM_Version::supports_sse(), ""));
2150   int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_128bit;
2151   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
2152   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2153   emit_int8(0x28);
2154   emit_int8((unsigned char)(0xC0 | encode));
2155 }
2156 
2157 void Assembler::movlhps(XMMRegister dst, XMMRegister src) {
2158   NOT_LP64(assert(VM_Version::supports_sse(), ""));
2159   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2160   int encode = simd_prefix_and_encode(dst, src, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2161   emit_int8(0x16);
2162   emit_int8((unsigned char)(0xC0 | encode));
2163 }
2164 
2165 void Assembler::movb(Register dst, Address src) {
2166   NOT_LP64(assert(dst->has_byte_register(), "must have byte register"));
2167   InstructionMark im(this);
2168   prefix(src, dst, true);
2169   emit_int8((unsigned char)0x8A);
2170   emit_operand(dst, src);
2171 }
2172 
2173 void Assembler::movddup(XMMRegister dst, XMMRegister src) {
2174   NOT_LP64(assert(VM_Version::supports_sse3(), ""));
2175   int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_128bit;
2176   InstructionAttr attributes(vector_len, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2177   attributes.set_rex_vex_w_reverted();
2178   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2179   emit_int8(0x12);
2180   emit_int8(0xC0 | encode);
2181 }
2182 
2183 void Assembler::kmovbl(KRegister dst, Register src) {
2184   assert(VM_Version::supports_avx512dq(), "");
2185   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2186   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2187   emit_int8((unsigned char)0x92);
2188   emit_int8((unsigned char)(0xC0 | encode));
2189 }
2190 
2191 void Assembler::kmovbl(Register dst, KRegister src) {
2192   assert(VM_Version::supports_avx512dq(), "");
2193   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2194   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2195   emit_int8((unsigned char)0x93);
2196   emit_int8((unsigned char)(0xC0 | encode));
2197 }
2198 
2199 void Assembler::kmovwl(KRegister dst, Register src) {
2200   assert(VM_Version::supports_evex(), "");
2201   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2202   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2203   emit_int8((unsigned char)0x92);
2204   emit_int8((unsigned char)(0xC0 | encode));
2205 }
2206 
2207 void Assembler::kmovwl(Register dst, KRegister src) {
2208   assert(VM_Version::supports_evex(), "");
2209   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2210   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2211   emit_int8((unsigned char)0x93);
2212   emit_int8((unsigned char)(0xC0 | encode));
2213 }
2214 
2215 void Assembler::kmovwl(KRegister dst, Address src) {
2216   assert(VM_Version::supports_evex(), "");
2217   InstructionMark im(this);
2218   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2219   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2220   emit_int8((unsigned char)0x90);
2221   emit_operand((Register)dst, src);
2222 }
2223 
2224 void Assembler::kmovdl(KRegister dst, Register src) {
2225   assert(VM_Version::supports_avx512bw(), "");
2226   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2227   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2228   emit_int8((unsigned char)0x92);
2229   emit_int8((unsigned char)(0xC0 | encode));
2230 }
2231 
2232 void Assembler::kmovdl(Register dst, KRegister src) {
2233   assert(VM_Version::supports_avx512bw(), "");
2234   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2235   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2236   emit_int8((unsigned char)0x93);
2237   emit_int8((unsigned char)(0xC0 | encode));
2238 }
2239 
2240 void Assembler::kmovql(KRegister dst, KRegister src) {
2241   assert(VM_Version::supports_avx512bw(), "");
2242   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2243   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2244   emit_int8((unsigned char)0x90);
2245   emit_int8((unsigned char)(0xC0 | encode));
2246 }
2247 
2248 void Assembler::kmovql(KRegister dst, Address src) {
2249   assert(VM_Version::supports_avx512bw(), "");
2250   InstructionMark im(this);
2251   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2252   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2253   emit_int8((unsigned char)0x90);
2254   emit_operand((Register)dst, src);
2255 }
2256 
2257 void Assembler::kmovql(Address dst, KRegister src) {
2258   assert(VM_Version::supports_avx512bw(), "");
2259   InstructionMark im(this);
2260   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2261   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2262   emit_int8((unsigned char)0x90);
2263   emit_operand((Register)src, dst);
2264 }
2265 
2266 void Assembler::kmovql(KRegister dst, Register src) {
2267   assert(VM_Version::supports_avx512bw(), "");
2268   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2269   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2270   emit_int8((unsigned char)0x92);
2271   emit_int8((unsigned char)(0xC0 | encode));
2272 }
2273 
2274 void Assembler::kmovql(Register dst, KRegister src) {
2275   assert(VM_Version::supports_avx512bw(), "");
2276   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2277   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2278   emit_int8((unsigned char)0x93);
2279   emit_int8((unsigned char)(0xC0 | encode));
2280 }
2281 
2282 void Assembler::knotwl(KRegister dst, KRegister src) {
2283   assert(VM_Version::supports_evex(), "");
2284   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2285   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2286   emit_int8((unsigned char)0x44);
2287   emit_int8((unsigned char)(0xC0 | encode));
2288 }
2289 
2290 // This instruction produces ZF or CF flags
2291 void Assembler::kortestbl(KRegister src1, KRegister src2) {
2292   assert(VM_Version::supports_avx512dq(), "");
2293   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2294   int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2295   emit_int8((unsigned char)0x98);
2296   emit_int8((unsigned char)(0xC0 | encode));
2297 }
2298 
2299 // This instruction produces ZF or CF flags
2300 void Assembler::kortestwl(KRegister src1, KRegister src2) {
2301   assert(VM_Version::supports_evex(), "");
2302   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2303   int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2304   emit_int8((unsigned char)0x98);
2305   emit_int8((unsigned char)(0xC0 | encode));
2306 }
2307 
2308 // This instruction produces ZF or CF flags
2309 void Assembler::kortestdl(KRegister src1, KRegister src2) {
2310   assert(VM_Version::supports_avx512bw(), "");
2311   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2312   int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2313   emit_int8((unsigned char)0x98);
2314   emit_int8((unsigned char)(0xC0 | encode));
2315 }
2316 
2317 // This instruction produces ZF or CF flags
2318 void Assembler::kortestql(KRegister src1, KRegister src2) {
2319   assert(VM_Version::supports_avx512bw(), "");
2320   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2321   int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2322   emit_int8((unsigned char)0x98);
2323   emit_int8((unsigned char)(0xC0 | encode));
2324 }
2325 
2326 void Assembler::movb(Address dst, int imm8) {
2327   InstructionMark im(this);
2328    prefix(dst);
2329   emit_int8((unsigned char)0xC6);
2330   emit_operand(rax, dst, 1);
2331   emit_int8(imm8);
2332 }
2333 
2334 
2335 void Assembler::movb(Address dst, Register src) {
2336   assert(src->has_byte_register(), "must have byte register");
2337   InstructionMark im(this);
2338   prefix(dst, src, true);
2339   emit_int8((unsigned char)0x88);
2340   emit_operand(src, dst);
2341 }
2342 
2343 void Assembler::movdl(XMMRegister dst, Register src) {
2344   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2345   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2346   int encode = simd_prefix_and_encode(dst, xnoreg, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2347   emit_int8(0x6E);
2348   emit_int8((unsigned char)(0xC0 | encode));
2349 }
2350 
2351 void Assembler::movdl(Register dst, XMMRegister src) {
2352   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2353   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2354   // swap src/dst to get correct prefix
2355   int encode = simd_prefix_and_encode(src, xnoreg, as_XMMRegister(dst->encoding()), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2356   emit_int8(0x7E);
2357   emit_int8((unsigned char)(0xC0 | encode));
2358 }
2359 
2360 void Assembler::movdl(XMMRegister dst, Address src) {
2361   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2362   InstructionMark im(this);
2363   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2364   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
2365   simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2366   emit_int8(0x6E);
2367   emit_operand(dst, src);
2368 }
2369 
2370 void Assembler::movdl(Address dst, XMMRegister src) {
2371   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2372   InstructionMark im(this);
2373   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2374   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
2375   simd_prefix(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2376   emit_int8(0x7E);
2377   emit_operand(src, dst);
2378 }
2379 
2380 void Assembler::movdqa(XMMRegister dst, XMMRegister src) {
2381   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2382   int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_128bit;
2383   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2384   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2385   emit_int8(0x6F);
2386   emit_int8((unsigned char)(0xC0 | encode));
2387 }
2388 
2389 void Assembler::movdqa(XMMRegister dst, Address src) {
2390   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2391   InstructionMark im(this);
2392   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2393   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2394   simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2395   emit_int8(0x6F);
2396   emit_operand(dst, src);
2397 }
2398 
2399 void Assembler::movdqu(XMMRegister dst, Address src) {
2400   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2401   InstructionMark im(this);
2402   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2403   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2404   simd_prefix(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2405   emit_int8(0x6F);
2406   emit_operand(dst, src);
2407 }
2408 
2409 void Assembler::movdqu(XMMRegister dst, XMMRegister src) {
2410   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2411   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2412   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2413   emit_int8(0x6F);
2414   emit_int8((unsigned char)(0xC0 | encode));
2415 }
2416 
2417 void Assembler::movdqu(Address dst, XMMRegister src) {
2418   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2419   InstructionMark im(this);
2420   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2421   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2422   simd_prefix(src, xnoreg, dst, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2423   emit_int8(0x7F);
2424   emit_operand(src, dst);
2425 }
2426 
2427 // Move Unaligned 256bit Vector
2428 void Assembler::vmovdqu(XMMRegister dst, XMMRegister src) {
2429   assert(UseAVX > 0, "");
2430   InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2431   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2432   emit_int8(0x6F);
2433   emit_int8((unsigned char)(0xC0 | encode));
2434 }
2435 
2436 void Assembler::vmovdqu(XMMRegister dst, Address src) {
2437   assert(UseAVX > 0, "");
2438   InstructionMark im(this);
2439   InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2440   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2441   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2442   emit_int8(0x6F);
2443   emit_operand(dst, src);
2444 }
2445 
2446 void Assembler::vmovdqu(Address dst, XMMRegister src) {
2447   assert(UseAVX > 0, "");
2448   InstructionMark im(this);
2449   InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2450   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2451   // swap src<->dst for encoding
2452   assert(src != xnoreg, "sanity");
2453   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2454   emit_int8(0x7F);
2455   emit_operand(src, dst);
2456 }
2457 
2458 // Move Unaligned EVEX enabled Vector (programmable : 8,16,32,64)
2459 void Assembler::evmovdqub(XMMRegister dst, XMMRegister src, int vector_len) {
2460   assert(VM_Version::supports_evex(), "");
2461   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
2462   attributes.set_is_evex_instruction();
2463   int prefix = (_legacy_mode_bw) ? VEX_SIMD_F2 : VEX_SIMD_F3;
2464   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), (Assembler::VexSimdPrefix)prefix, VEX_OPCODE_0F, &attributes);
2465   emit_int8(0x6F);
2466   emit_int8((unsigned char)(0xC0 | encode));
2467 }
2468 
2469 void Assembler::evmovdqub(XMMRegister dst, Address src, int vector_len) {
2470   assert(VM_Version::supports_evex(), "");
2471   InstructionMark im(this);
2472   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
2473   int prefix = (_legacy_mode_bw) ? VEX_SIMD_F2 : VEX_SIMD_F3;
2474   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2475   attributes.set_is_evex_instruction();
2476   vex_prefix(src, 0, dst->encoding(), (Assembler::VexSimdPrefix)prefix, VEX_OPCODE_0F, &attributes);
2477   emit_int8(0x6F);
2478   emit_operand(dst, src);
2479 }
2480 
2481 void Assembler::evmovdqub(Address dst, XMMRegister src, int vector_len) {
2482   assert(VM_Version::supports_evex(), "");
2483   assert(src != xnoreg, "sanity");
2484   InstructionMark im(this);
2485   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
2486   int prefix = (_legacy_mode_bw) ? VEX_SIMD_F2 : VEX_SIMD_F3;
2487   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2488   attributes.set_is_evex_instruction();
2489   vex_prefix(dst, 0, src->encoding(), (Assembler::VexSimdPrefix)prefix, VEX_OPCODE_0F, &attributes);
2490   emit_int8(0x7F);
2491   emit_operand(src, dst);
2492 }
2493 
2494 void Assembler::evmovdquw(XMMRegister dst, XMMRegister src, int vector_len) {
2495   assert(VM_Version::supports_evex(), "");
2496   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
2497   attributes.set_is_evex_instruction();
2498   int prefix = (_legacy_mode_bw) ? VEX_SIMD_F2 : VEX_SIMD_F3;
2499   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), (Assembler::VexSimdPrefix)prefix, VEX_OPCODE_0F, &attributes);
2500   emit_int8(0x6F);
2501   emit_int8((unsigned char)(0xC0 | encode));
2502 }
2503 
2504 void Assembler::evmovdquw(XMMRegister dst, Address src, int vector_len) {
2505   assert(VM_Version::supports_evex(), "");
2506   InstructionMark im(this);
2507   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
2508   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2509   attributes.set_is_evex_instruction();
2510   int prefix = (_legacy_mode_bw) ? VEX_SIMD_F2 : VEX_SIMD_F3;
2511   vex_prefix(src, 0, dst->encoding(), (Assembler::VexSimdPrefix)prefix, VEX_OPCODE_0F, &attributes);
2512   emit_int8(0x6F);
2513   emit_operand(dst, src);
2514 }
2515 
2516 void Assembler::evmovdquw(Address dst, XMMRegister src, int vector_len) {
2517   assert(VM_Version::supports_evex(), "");
2518   assert(src != xnoreg, "sanity");
2519   InstructionMark im(this);
2520   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
2521   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2522   attributes.set_is_evex_instruction();
2523   int prefix = (_legacy_mode_bw) ? VEX_SIMD_F2 : VEX_SIMD_F3;
2524   vex_prefix(dst, 0, src->encoding(), (Assembler::VexSimdPrefix)prefix, VEX_OPCODE_0F, &attributes);
2525   emit_int8(0x7F);
2526   emit_operand(src, dst);
2527 }
2528 
2529 void Assembler::evmovdqul(XMMRegister dst, XMMRegister src, int vector_len) {
2530   assert(VM_Version::supports_evex(), "");
2531   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2532   attributes.set_is_evex_instruction();
2533   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2534   emit_int8(0x6F);
2535   emit_int8((unsigned char)(0xC0 | encode));
2536 }
2537 
2538 void Assembler::evmovdqul(XMMRegister dst, Address src, int vector_len) {
2539   assert(VM_Version::supports_evex(), "");
2540   InstructionMark im(this);
2541   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false , /* uses_vl */ true);
2542   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2543   attributes.set_is_evex_instruction();
2544   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2545   emit_int8(0x6F);
2546   emit_operand(dst, src);
2547 }
2548 
2549 void Assembler::evmovdqul(Address dst, XMMRegister src, int vector_len) {
2550   assert(VM_Version::supports_evex(), "");
2551   assert(src != xnoreg, "sanity");
2552   InstructionMark im(this);
2553   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2554   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2555   attributes.set_is_evex_instruction();
2556   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2557   emit_int8(0x7F);
2558   emit_operand(src, dst);
2559 }
2560 
2561 void Assembler::evmovdquq(XMMRegister dst, XMMRegister src, int vector_len) {
2562   assert(VM_Version::supports_evex(), "");
2563   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2564   attributes.set_is_evex_instruction();
2565   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2566   emit_int8(0x6F);
2567   emit_int8((unsigned char)(0xC0 | encode));
2568 }
2569 
2570 void Assembler::evmovdquq(XMMRegister dst, Address src, int vector_len) {
2571   assert(VM_Version::supports_evex(), "");
2572   InstructionMark im(this);
2573   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2574   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2575   attributes.set_is_evex_instruction();
2576   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2577   emit_int8(0x6F);
2578   emit_operand(dst, src);
2579 }
2580 
2581 void Assembler::evmovdquq(Address dst, XMMRegister src, int vector_len) {
2582   assert(VM_Version::supports_evex(), "");
2583   assert(src != xnoreg, "sanity");
2584   InstructionMark im(this);
2585   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2586   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2587   attributes.set_is_evex_instruction();
2588   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2589   emit_int8(0x7F);
2590   emit_operand(src, dst);
2591 }
2592 
2593 // Uses zero extension on 64bit
2594 
2595 void Assembler::movl(Register dst, int32_t imm32) {
2596   int encode = prefix_and_encode(dst->encoding());
2597   emit_int8((unsigned char)(0xB8 | encode));
2598   emit_int32(imm32);
2599 }
2600 
2601 void Assembler::movl(Register dst, Register src) {
2602   int encode = prefix_and_encode(dst->encoding(), src->encoding());
2603   emit_int8((unsigned char)0x8B);
2604   emit_int8((unsigned char)(0xC0 | encode));
2605 }
2606 
2607 void Assembler::movl(Register dst, Address src) {
2608   InstructionMark im(this);
2609   prefix(src, dst);
2610   emit_int8((unsigned char)0x8B);
2611   emit_operand(dst, src);
2612 }
2613 
2614 void Assembler::movl(Address dst, int32_t imm32) {
2615   InstructionMark im(this);
2616   prefix(dst);
2617   emit_int8((unsigned char)0xC7);
2618   emit_operand(rax, dst, 4);
2619   emit_int32(imm32);
2620 }
2621 
2622 void Assembler::movl(Address dst, Register src) {
2623   InstructionMark im(this);
2624   prefix(dst, src);
2625   emit_int8((unsigned char)0x89);
2626   emit_operand(src, dst);
2627 }
2628 
2629 // New cpus require to use movsd and movss to avoid partial register stall
2630 // when loading from memory. But for old Opteron use movlpd instead of movsd.
2631 // The selection is done in MacroAssembler::movdbl() and movflt().
2632 void Assembler::movlpd(XMMRegister dst, Address src) {
2633   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2634   InstructionMark im(this);
2635   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2636   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
2637   attributes.set_rex_vex_w_reverted();
2638   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2639   emit_int8(0x12);
2640   emit_operand(dst, src);
2641 }
2642 
2643 void Assembler::movq( MMXRegister dst, Address src ) {
2644   assert( VM_Version::supports_mmx(), "" );
2645   emit_int8(0x0F);
2646   emit_int8(0x6F);
2647   emit_operand(dst, src);
2648 }
2649 
2650 void Assembler::movq( Address dst, MMXRegister src ) {
2651   assert( VM_Version::supports_mmx(), "" );
2652   emit_int8(0x0F);
2653   emit_int8(0x7F);
2654   // workaround gcc (3.2.1-7a) bug
2655   // In that version of gcc with only an emit_operand(MMX, Address)
2656   // gcc will tail jump and try and reverse the parameters completely
2657   // obliterating dst in the process. By having a version available
2658   // that doesn't need to swap the args at the tail jump the bug is
2659   // avoided.
2660   emit_operand(dst, src);
2661 }
2662 
2663 void Assembler::movq(XMMRegister dst, Address src) {
2664   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2665   InstructionMark im(this);
2666   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2667   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
2668   attributes.set_rex_vex_w_reverted();
2669   simd_prefix(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2670   emit_int8(0x7E);
2671   emit_operand(dst, src);
2672 }
2673 
2674 void Assembler::movq(Address dst, XMMRegister src) {
2675   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2676   InstructionMark im(this);
2677   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2678   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
2679   attributes.set_rex_vex_w_reverted();
2680   simd_prefix(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2681   emit_int8((unsigned char)0xD6);
2682   emit_operand(src, dst);
2683 }
2684 
2685 void Assembler::movsbl(Register dst, Address src) { // movsxb
2686   InstructionMark im(this);
2687   prefix(src, dst);
2688   emit_int8(0x0F);
2689   emit_int8((unsigned char)0xBE);
2690   emit_operand(dst, src);
2691 }
2692 
2693 void Assembler::movsbl(Register dst, Register src) { // movsxb
2694   NOT_LP64(assert(src->has_byte_register(), "must have byte register"));
2695   int encode = prefix_and_encode(dst->encoding(), false, src->encoding(), true);
2696   emit_int8(0x0F);
2697   emit_int8((unsigned char)0xBE);
2698   emit_int8((unsigned char)(0xC0 | encode));
2699 }
2700 
2701 void Assembler::movsd(XMMRegister dst, XMMRegister src) {
2702   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2703   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
2704   attributes.set_rex_vex_w_reverted();
2705   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2706   emit_int8(0x10);
2707   emit_int8((unsigned char)(0xC0 | encode));
2708 }
2709 
2710 void Assembler::movsd(XMMRegister dst, Address src) {
2711   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2712   InstructionMark im(this);
2713   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
2714   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
2715   attributes.set_rex_vex_w_reverted();
2716   simd_prefix(dst, xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2717   emit_int8(0x10);
2718   emit_operand(dst, src);
2719 }
2720 
2721 void Assembler::movsd(Address dst, XMMRegister src) {
2722   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2723   InstructionMark im(this);
2724   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
2725   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
2726   attributes.set_rex_vex_w_reverted();
2727   simd_prefix(src, xnoreg, dst, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2728   emit_int8(0x11);
2729   emit_operand(src, dst);
2730 }
2731 
2732 void Assembler::movss(XMMRegister dst, XMMRegister src) {
2733   NOT_LP64(assert(VM_Version::supports_sse(), ""));
2734   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
2735   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2736   emit_int8(0x10);
2737   emit_int8((unsigned char)(0xC0 | encode));
2738 }
2739 
2740 void Assembler::movss(XMMRegister dst, Address src) {
2741   NOT_LP64(assert(VM_Version::supports_sse(), ""));
2742   InstructionMark im(this);
2743   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
2744   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
2745   simd_prefix(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2746   emit_int8(0x10);
2747   emit_operand(dst, src);
2748 }
2749 
2750 void Assembler::movss(Address dst, XMMRegister src) {
2751   NOT_LP64(assert(VM_Version::supports_sse(), ""));
2752   InstructionMark im(this);
2753   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
2754   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
2755   simd_prefix(src, xnoreg, dst, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2756   emit_int8(0x11);
2757   emit_operand(src, dst);
2758 }
2759 
2760 void Assembler::movswl(Register dst, Address src) { // movsxw
2761   InstructionMark im(this);
2762   prefix(src, dst);
2763   emit_int8(0x0F);
2764   emit_int8((unsigned char)0xBF);
2765   emit_operand(dst, src);
2766 }
2767 
2768 void Assembler::movswl(Register dst, Register src) { // movsxw
2769   int encode = prefix_and_encode(dst->encoding(), src->encoding());
2770   emit_int8(0x0F);
2771   emit_int8((unsigned char)0xBF);
2772   emit_int8((unsigned char)(0xC0 | encode));
2773 }
2774 
2775 void Assembler::movw(Address dst, int imm16) {
2776   InstructionMark im(this);
2777 
2778   emit_int8(0x66); // switch to 16-bit mode
2779   prefix(dst);
2780   emit_int8((unsigned char)0xC7);
2781   emit_operand(rax, dst, 2);
2782   emit_int16(imm16);
2783 }
2784 
2785 void Assembler::movw(Register dst, Address src) {
2786   InstructionMark im(this);
2787   emit_int8(0x66);
2788   prefix(src, dst);
2789   emit_int8((unsigned char)0x8B);
2790   emit_operand(dst, src);
2791 }
2792 
2793 void Assembler::movw(Address dst, Register src) {
2794   InstructionMark im(this);
2795   emit_int8(0x66);
2796   prefix(dst, src);
2797   emit_int8((unsigned char)0x89);
2798   emit_operand(src, dst);
2799 }
2800 
2801 void Assembler::movzbl(Register dst, Address src) { // movzxb
2802   InstructionMark im(this);
2803   prefix(src, dst);
2804   emit_int8(0x0F);
2805   emit_int8((unsigned char)0xB6);
2806   emit_operand(dst, src);
2807 }
2808 
2809 void Assembler::movzbl(Register dst, Register src) { // movzxb
2810   NOT_LP64(assert(src->has_byte_register(), "must have byte register"));
2811   int encode = prefix_and_encode(dst->encoding(), false, src->encoding(), true);
2812   emit_int8(0x0F);
2813   emit_int8((unsigned char)0xB6);
2814   emit_int8(0xC0 | encode);
2815 }
2816 
2817 void Assembler::movzwl(Register dst, Address src) { // movzxw
2818   InstructionMark im(this);
2819   prefix(src, dst);
2820   emit_int8(0x0F);
2821   emit_int8((unsigned char)0xB7);
2822   emit_operand(dst, src);
2823 }
2824 
2825 void Assembler::movzwl(Register dst, Register src) { // movzxw
2826   int encode = prefix_and_encode(dst->encoding(), src->encoding());
2827   emit_int8(0x0F);
2828   emit_int8((unsigned char)0xB7);
2829   emit_int8(0xC0 | encode);
2830 }
2831 
2832 void Assembler::mull(Address src) {
2833   InstructionMark im(this);
2834   prefix(src);
2835   emit_int8((unsigned char)0xF7);
2836   emit_operand(rsp, src);
2837 }
2838 
2839 void Assembler::mull(Register src) {
2840   int encode = prefix_and_encode(src->encoding());
2841   emit_int8((unsigned char)0xF7);
2842   emit_int8((unsigned char)(0xE0 | encode));
2843 }
2844 
2845 void Assembler::mulsd(XMMRegister dst, Address src) {
2846   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2847   InstructionMark im(this);
2848   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
2849   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
2850   attributes.set_rex_vex_w_reverted();
2851   simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2852   emit_int8(0x59);
2853   emit_operand(dst, src);
2854 }
2855 
2856 void Assembler::mulsd(XMMRegister dst, XMMRegister src) {
2857   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2858   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
2859   attributes.set_rex_vex_w_reverted();
2860   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2861   emit_int8(0x59);
2862   emit_int8((unsigned char)(0xC0 | encode));
2863 }
2864 
2865 void Assembler::mulss(XMMRegister dst, Address src) {
2866   NOT_LP64(assert(VM_Version::supports_sse(), ""));
2867   InstructionMark im(this);
2868   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
2869   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
2870   simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2871   emit_int8(0x59);
2872   emit_operand(dst, src);
2873 }
2874 
2875 void Assembler::mulss(XMMRegister dst, XMMRegister src) {
2876   NOT_LP64(assert(VM_Version::supports_sse(), ""));
2877   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
2878   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2879   emit_int8(0x59);
2880   emit_int8((unsigned char)(0xC0 | encode));
2881 }
2882 
2883 void Assembler::negl(Register dst) {
2884   int encode = prefix_and_encode(dst->encoding());
2885   emit_int8((unsigned char)0xF7);
2886   emit_int8((unsigned char)(0xD8 | encode));
2887 }
2888 
2889 void Assembler::nop(int i) {
2890 #ifdef ASSERT
2891   assert(i > 0, " ");
2892   // The fancy nops aren't currently recognized by debuggers making it a
2893   // pain to disassemble code while debugging. If asserts are on clearly
2894   // speed is not an issue so simply use the single byte traditional nop
2895   // to do alignment.
2896 
2897   for (; i > 0 ; i--) emit_int8((unsigned char)0x90);
2898   return;
2899 
2900 #endif // ASSERT
2901 
2902   if (UseAddressNop && VM_Version::is_intel()) {
2903     //
2904     // Using multi-bytes nops "0x0F 0x1F [address]" for Intel
2905     //  1: 0x90
2906     //  2: 0x66 0x90
2907     //  3: 0x66 0x66 0x90 (don't use "0x0F 0x1F 0x00" - need patching safe padding)
2908     //  4: 0x0F 0x1F 0x40 0x00
2909     //  5: 0x0F 0x1F 0x44 0x00 0x00
2910     //  6: 0x66 0x0F 0x1F 0x44 0x00 0x00
2911     //  7: 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00
2912     //  8: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
2913     //  9: 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
2914     // 10: 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
2915     // 11: 0x66 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
2916 
2917     // The rest coding is Intel specific - don't use consecutive address nops
2918 
2919     // 12: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
2920     // 13: 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
2921     // 14: 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
2922     // 15: 0x66 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
2923 
2924     while(i >= 15) {
2925       // For Intel don't generate consecutive addess nops (mix with regular nops)
2926       i -= 15;
2927       emit_int8(0x66);   // size prefix
2928       emit_int8(0x66);   // size prefix
2929       emit_int8(0x66);   // size prefix
2930       addr_nop_8();
2931       emit_int8(0x66);   // size prefix
2932       emit_int8(0x66);   // size prefix
2933       emit_int8(0x66);   // size prefix
2934       emit_int8((unsigned char)0x90);
2935                          // nop
2936     }
2937     switch (i) {
2938       case 14:
2939         emit_int8(0x66); // size prefix
2940       case 13:
2941         emit_int8(0x66); // size prefix
2942       case 12:
2943         addr_nop_8();
2944         emit_int8(0x66); // size prefix
2945         emit_int8(0x66); // size prefix
2946         emit_int8(0x66); // size prefix
2947         emit_int8((unsigned char)0x90);
2948                          // nop
2949         break;
2950       case 11:
2951         emit_int8(0x66); // size prefix
2952       case 10:
2953         emit_int8(0x66); // size prefix
2954       case 9:
2955         emit_int8(0x66); // size prefix
2956       case 8:
2957         addr_nop_8();
2958         break;
2959       case 7:
2960         addr_nop_7();
2961         break;
2962       case 6:
2963         emit_int8(0x66); // size prefix
2964       case 5:
2965         addr_nop_5();
2966         break;
2967       case 4:
2968         addr_nop_4();
2969         break;
2970       case 3:
2971         // Don't use "0x0F 0x1F 0x00" - need patching safe padding
2972         emit_int8(0x66); // size prefix
2973       case 2:
2974         emit_int8(0x66); // size prefix
2975       case 1:
2976         emit_int8((unsigned char)0x90);
2977                          // nop
2978         break;
2979       default:
2980         assert(i == 0, " ");
2981     }
2982     return;
2983   }
2984   if (UseAddressNop && VM_Version::is_amd()) {
2985     //
2986     // Using multi-bytes nops "0x0F 0x1F [address]" for AMD.
2987     //  1: 0x90
2988     //  2: 0x66 0x90
2989     //  3: 0x66 0x66 0x90 (don't use "0x0F 0x1F 0x00" - need patching safe padding)
2990     //  4: 0x0F 0x1F 0x40 0x00
2991     //  5: 0x0F 0x1F 0x44 0x00 0x00
2992     //  6: 0x66 0x0F 0x1F 0x44 0x00 0x00
2993     //  7: 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00
2994     //  8: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
2995     //  9: 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
2996     // 10: 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
2997     // 11: 0x66 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
2998 
2999     // The rest coding is AMD specific - use consecutive address nops
3000 
3001     // 12: 0x66 0x0F 0x1F 0x44 0x00 0x00 0x66 0x0F 0x1F 0x44 0x00 0x00
3002     // 13: 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00 0x66 0x0F 0x1F 0x44 0x00 0x00
3003     // 14: 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00
3004     // 15: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00
3005     // 16: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
3006     //     Size prefixes (0x66) are added for larger sizes
3007 
3008     while(i >= 22) {
3009       i -= 11;
3010       emit_int8(0x66); // size prefix
3011       emit_int8(0x66); // size prefix
3012       emit_int8(0x66); // size prefix
3013       addr_nop_8();
3014     }
3015     // Generate first nop for size between 21-12
3016     switch (i) {
3017       case 21:
3018         i -= 1;
3019         emit_int8(0x66); // size prefix
3020       case 20:
3021       case 19:
3022         i -= 1;
3023         emit_int8(0x66); // size prefix
3024       case 18:
3025       case 17:
3026         i -= 1;
3027         emit_int8(0x66); // size prefix
3028       case 16:
3029       case 15:
3030         i -= 8;
3031         addr_nop_8();
3032         break;
3033       case 14:
3034       case 13:
3035         i -= 7;
3036         addr_nop_7();
3037         break;
3038       case 12:
3039         i -= 6;
3040         emit_int8(0x66); // size prefix
3041         addr_nop_5();
3042         break;
3043       default:
3044         assert(i < 12, " ");
3045     }
3046 
3047     // Generate second nop for size between 11-1
3048     switch (i) {
3049       case 11:
3050         emit_int8(0x66); // size prefix
3051       case 10:
3052         emit_int8(0x66); // size prefix
3053       case 9:
3054         emit_int8(0x66); // size prefix
3055       case 8:
3056         addr_nop_8();
3057         break;
3058       case 7:
3059         addr_nop_7();
3060         break;
3061       case 6:
3062         emit_int8(0x66); // size prefix
3063       case 5:
3064         addr_nop_5();
3065         break;
3066       case 4:
3067         addr_nop_4();
3068         break;
3069       case 3:
3070         // Don't use "0x0F 0x1F 0x00" - need patching safe padding
3071         emit_int8(0x66); // size prefix
3072       case 2:
3073         emit_int8(0x66); // size prefix
3074       case 1:
3075         emit_int8((unsigned char)0x90);
3076                          // nop
3077         break;
3078       default:
3079         assert(i == 0, " ");
3080     }
3081     return;
3082   }
3083 
3084   // Using nops with size prefixes "0x66 0x90".
3085   // From AMD Optimization Guide:
3086   //  1: 0x90
3087   //  2: 0x66 0x90
3088   //  3: 0x66 0x66 0x90
3089   //  4: 0x66 0x66 0x66 0x90
3090   //  5: 0x66 0x66 0x90 0x66 0x90
3091   //  6: 0x66 0x66 0x90 0x66 0x66 0x90
3092   //  7: 0x66 0x66 0x66 0x90 0x66 0x66 0x90
3093   //  8: 0x66 0x66 0x66 0x90 0x66 0x66 0x66 0x90
3094   //  9: 0x66 0x66 0x90 0x66 0x66 0x90 0x66 0x66 0x90
3095   // 10: 0x66 0x66 0x66 0x90 0x66 0x66 0x90 0x66 0x66 0x90
3096   //
3097   while(i > 12) {
3098     i -= 4;
3099     emit_int8(0x66); // size prefix
3100     emit_int8(0x66);
3101     emit_int8(0x66);
3102     emit_int8((unsigned char)0x90);
3103                      // nop
3104   }
3105   // 1 - 12 nops
3106   if(i > 8) {
3107     if(i > 9) {
3108       i -= 1;
3109       emit_int8(0x66);
3110     }
3111     i -= 3;
3112     emit_int8(0x66);
3113     emit_int8(0x66);
3114     emit_int8((unsigned char)0x90);
3115   }
3116   // 1 - 8 nops
3117   if(i > 4) {
3118     if(i > 6) {
3119       i -= 1;
3120       emit_int8(0x66);
3121     }
3122     i -= 3;
3123     emit_int8(0x66);
3124     emit_int8(0x66);
3125     emit_int8((unsigned char)0x90);
3126   }
3127   switch (i) {
3128     case 4:
3129       emit_int8(0x66);
3130     case 3:
3131       emit_int8(0x66);
3132     case 2:
3133       emit_int8(0x66);
3134     case 1:
3135       emit_int8((unsigned char)0x90);
3136       break;
3137     default:
3138       assert(i == 0, " ");
3139   }
3140 }
3141 
3142 void Assembler::notl(Register dst) {
3143   int encode = prefix_and_encode(dst->encoding());
3144   emit_int8((unsigned char)0xF7);
3145   emit_int8((unsigned char)(0xD0 | encode));
3146 }
3147 
3148 void Assembler::orl(Address dst, int32_t imm32) {
3149   InstructionMark im(this);
3150   prefix(dst);
3151   emit_arith_operand(0x81, rcx, dst, imm32);
3152 }
3153 
3154 void Assembler::orl(Register dst, int32_t imm32) {
3155   prefix(dst);
3156   emit_arith(0x81, 0xC8, dst, imm32);
3157 }
3158 
3159 void Assembler::orl(Register dst, Address src) {
3160   InstructionMark im(this);
3161   prefix(src, dst);
3162   emit_int8(0x0B);
3163   emit_operand(dst, src);
3164 }
3165 
3166 void Assembler::orl(Register dst, Register src) {
3167   (void) prefix_and_encode(dst->encoding(), src->encoding());
3168   emit_arith(0x0B, 0xC0, dst, src);
3169 }
3170 
3171 void Assembler::orl(Address dst, Register src) {
3172   InstructionMark im(this);
3173   prefix(dst, src);
3174   emit_int8(0x09);
3175   emit_operand(src, dst);
3176 }
3177 
3178 void Assembler::packuswb(XMMRegister dst, Address src) {
3179   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3180   assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
3181   InstructionMark im(this);
3182   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
3183   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
3184   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3185   emit_int8(0x67);
3186   emit_operand(dst, src);
3187 }
3188 
3189 void Assembler::packuswb(XMMRegister dst, XMMRegister src) {
3190   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3191   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
3192   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3193   emit_int8(0x67);
3194   emit_int8((unsigned char)(0xC0 | encode));
3195 }
3196 
3197 void Assembler::vpackuswb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
3198   assert(UseAVX > 0, "some form of AVX must be enabled");
3199   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
3200   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3201   emit_int8(0x67);
3202   emit_int8((unsigned char)(0xC0 | encode));
3203 }
3204 
3205 void Assembler::vpermq(XMMRegister dst, XMMRegister src, int imm8, int vector_len) {
3206   assert(VM_Version::supports_avx2(), "");
3207   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
3208   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3209   emit_int8(0x00);
3210   emit_int8(0xC0 | encode);
3211   emit_int8(imm8);
3212 }
3213 
3214 void Assembler::pause() {
3215   emit_int8((unsigned char)0xF3);
3216   emit_int8((unsigned char)0x90);
3217 }
3218 
3219 void Assembler::pcmpestri(XMMRegister dst, Address src, int imm8) {
3220   assert(VM_Version::supports_sse4_2(), "");
3221   InstructionMark im(this);
3222   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3223   simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3224   emit_int8(0x61);
3225   emit_operand(dst, src);
3226   emit_int8(imm8);
3227 }
3228 
3229 void Assembler::pcmpestri(XMMRegister dst, XMMRegister src, int imm8) {
3230   assert(VM_Version::supports_sse4_2(), "");
3231   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3232   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3233   emit_int8(0x61);
3234   emit_int8((unsigned char)(0xC0 | encode));
3235   emit_int8(imm8);
3236 }
3237 
3238 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
3239 void Assembler::pcmpeqb(XMMRegister dst, XMMRegister src) {
3240   assert(VM_Version::supports_sse2(), "");
3241   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3242   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3243   emit_int8(0x74);
3244   emit_int8((unsigned char)(0xC0 | encode));
3245 }
3246 
3247 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
3248 void Assembler::vpcmpeqb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
3249   assert(VM_Version::supports_avx(), "");
3250   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3251   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3252   emit_int8(0x74);
3253   emit_int8((unsigned char)(0xC0 | encode));
3254 }
3255 
3256 // In this context, kdst is written the mask used to process the equal components
3257 void Assembler::evpcmpeqb(KRegister kdst, XMMRegister nds, XMMRegister src, int vector_len) {
3258   assert(VM_Version::supports_avx512bw(), "");
3259   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
3260   attributes.set_is_evex_instruction();
3261   int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3262   emit_int8(0x74);
3263   emit_int8((unsigned char)(0xC0 | encode));
3264 }
3265 
3266 void Assembler::evpcmpeqb(KRegister kdst, XMMRegister nds, Address src, int vector_len) {
3267   assert(VM_Version::supports_avx512bw(), "");
3268   InstructionMark im(this);
3269   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
3270   attributes.set_is_evex_instruction();
3271   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
3272   int dst_enc = kdst->encoding();
3273   vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3274   emit_int8(0x74);
3275   emit_operand(as_Register(dst_enc), src);
3276 }
3277 
3278 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
3279 void Assembler::pcmpeqw(XMMRegister dst, XMMRegister src) {
3280   assert(VM_Version::supports_sse2(), "");
3281   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3282   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3283   emit_int8(0x75);
3284   emit_int8((unsigned char)(0xC0 | encode));
3285 }
3286 
3287 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
3288 void Assembler::vpcmpeqw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
3289   assert(VM_Version::supports_avx(), "");
3290   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3291   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3292   emit_int8(0x75);
3293   emit_int8((unsigned char)(0xC0 | encode));
3294 }
3295 
3296 // In this context, kdst is written the mask used to process the equal components
3297 void Assembler::evpcmpeqw(KRegister kdst, XMMRegister nds, XMMRegister src, int vector_len) {
3298   assert(VM_Version::supports_avx512bw(), "");
3299   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
3300   attributes.set_is_evex_instruction();
3301   int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3302   emit_int8(0x75);
3303   emit_int8((unsigned char)(0xC0 | encode));
3304 }
3305 
3306 void Assembler::evpcmpeqw(KRegister kdst, XMMRegister nds, Address src, int vector_len) {
3307   assert(VM_Version::supports_avx512bw(), "");
3308   InstructionMark im(this);
3309   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
3310   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
3311   attributes.set_is_evex_instruction();
3312   int dst_enc = kdst->encoding();
3313   vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3314   emit_int8(0x75);
3315   emit_operand(as_Register(dst_enc), src);
3316 }
3317 
3318 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
3319 void Assembler::pcmpeqd(XMMRegister dst, XMMRegister src) {
3320   assert(VM_Version::supports_sse2(), "");
3321   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3322   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3323   emit_int8(0x76);
3324   emit_int8((unsigned char)(0xC0 | encode));
3325 }
3326 
3327 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
3328 void Assembler::vpcmpeqd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
3329   assert(VM_Version::supports_avx(), "");
3330   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3331   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3332   emit_int8(0x76);
3333   emit_int8((unsigned char)(0xC0 | encode));
3334 }
3335 
3336 // In this context, kdst is written the mask used to process the equal components
3337 void Assembler::evpcmpeqd(KRegister kdst, XMMRegister nds, XMMRegister src, int vector_len) {
3338   assert(VM_Version::supports_evex(), "");
3339   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
3340   attributes.set_is_evex_instruction();
3341   int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3342   emit_int8(0x76);
3343   emit_int8((unsigned char)(0xC0 | encode));
3344 }
3345 
3346 void Assembler::evpcmpeqd(KRegister kdst, XMMRegister nds, Address src, int vector_len) {
3347   assert(VM_Version::supports_evex(), "");
3348   InstructionMark im(this);
3349   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
3350   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
3351   attributes.set_is_evex_instruction();
3352   int dst_enc = kdst->encoding();
3353   vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3354   emit_int8(0x76);
3355   emit_operand(as_Register(dst_enc), src);
3356 }
3357 
3358 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
3359 void Assembler::pcmpeqq(XMMRegister dst, XMMRegister src) {
3360   assert(VM_Version::supports_sse4_1(), "");
3361   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3362   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3363   emit_int8(0x29);
3364   emit_int8((unsigned char)(0xC0 | encode));
3365 }
3366 
3367 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
3368 void Assembler::vpcmpeqq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
3369   assert(VM_Version::supports_avx(), "");
3370   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3371   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3372   emit_int8(0x29);
3373   emit_int8((unsigned char)(0xC0 | encode));
3374 }
3375 
3376 // In this context, kdst is written the mask used to process the equal components
3377 void Assembler::evpcmpeqq(KRegister kdst, XMMRegister nds, XMMRegister src, int vector_len) {
3378   assert(VM_Version::supports_evex(), "");
3379   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
3380   attributes.set_is_evex_instruction();
3381   int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3382   emit_int8(0x29);
3383   emit_int8((unsigned char)(0xC0 | encode));
3384 }
3385 
3386 // In this context, kdst is written the mask used to process the equal components
3387 void Assembler::evpcmpeqq(KRegister kdst, XMMRegister nds, Address src, int vector_len) {
3388   assert(VM_Version::supports_evex(), "");
3389   InstructionMark im(this);
3390   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
3391   attributes.set_is_evex_instruction();
3392   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
3393   int dst_enc = kdst->encoding();
3394   vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3395   emit_int8(0x29);
3396   emit_operand(as_Register(dst_enc), src);
3397 }
3398 
3399 void Assembler::pmovmskb(Register dst, XMMRegister src) {
3400   assert(VM_Version::supports_sse2(), "");
3401   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3402   int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3403   emit_int8((unsigned char)0xD7);
3404   emit_int8((unsigned char)(0xC0 | encode));
3405 }
3406 
3407 void Assembler::vpmovmskb(Register dst, XMMRegister src) {
3408   assert(VM_Version::supports_avx2(), "");
3409   InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3410   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3411   emit_int8((unsigned char)0xD7);
3412   emit_int8((unsigned char)(0xC0 | encode));
3413 }
3414 
3415 void Assembler::pextrd(Register dst, XMMRegister src, int imm8) {
3416   assert(VM_Version::supports_sse4_1(), "");
3417   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
3418   int encode = simd_prefix_and_encode(src, xnoreg, as_XMMRegister(dst->encoding()), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3419   emit_int8(0x16);
3420   emit_int8((unsigned char)(0xC0 | encode));
3421   emit_int8(imm8);
3422 }
3423 
3424 void Assembler::pextrd(Address dst, XMMRegister src, int imm8) {
3425   assert(VM_Version::supports_sse4_1(), "");
3426   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
3427   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
3428   simd_prefix(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3429   emit_int8(0x16);
3430   emit_operand(src, dst);
3431   emit_int8(imm8);
3432 }
3433 
3434 void Assembler::pextrq(Register dst, XMMRegister src, int imm8) {
3435   assert(VM_Version::supports_sse4_1(), "");
3436   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
3437   int encode = simd_prefix_and_encode(src, xnoreg, as_XMMRegister(dst->encoding()), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3438   emit_int8(0x16);
3439   emit_int8((unsigned char)(0xC0 | encode));
3440   emit_int8(imm8);
3441 }
3442 
3443 void Assembler::pextrq(Address dst, XMMRegister src, int imm8) {
3444   assert(VM_Version::supports_sse4_1(), "");
3445   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
3446   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
3447   simd_prefix(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3448   emit_int8(0x16);
3449   emit_operand(src, dst);
3450   emit_int8(imm8);
3451 }
3452 
3453 void Assembler::pextrw(Register dst, XMMRegister src, int imm8) {
3454   assert(VM_Version::supports_sse2(), "");
3455   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
3456   int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3457   emit_int8((unsigned char)0xC5);
3458   emit_int8((unsigned char)(0xC0 | encode));
3459   emit_int8(imm8);
3460 }
3461 
3462 void Assembler::pextrw(Address dst, XMMRegister src, int imm8) {
3463   assert(VM_Version::supports_sse4_1(), "");
3464   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
3465   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_16bit);
3466   simd_prefix(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3467   emit_int8((unsigned char)0x15);
3468   emit_operand(src, dst);
3469   emit_int8(imm8);
3470 }
3471 
3472 void Assembler::pextrb(Address dst, XMMRegister src, int imm8) {
3473   assert(VM_Version::supports_sse4_1(), "");
3474   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
3475   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_8bit);
3476   simd_prefix(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3477   emit_int8(0x14);
3478   emit_operand(src, dst);
3479   emit_int8(imm8);
3480 }
3481 
3482 void Assembler::pinsrd(XMMRegister dst, Register src, int imm8) {
3483   assert(VM_Version::supports_sse4_1(), "");
3484   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
3485   int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3486   emit_int8(0x22);
3487   emit_int8((unsigned char)(0xC0 | encode));
3488   emit_int8(imm8);
3489 }
3490 
3491 void Assembler::pinsrd(XMMRegister dst, Address src, int imm8) {
3492   assert(VM_Version::supports_sse4_1(), "");
3493   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
3494   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
3495   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3496   emit_int8(0x22);
3497   emit_operand(dst,src);
3498   emit_int8(imm8);
3499 }
3500 
3501 void Assembler::pinsrq(XMMRegister dst, Register src, int imm8) {
3502   assert(VM_Version::supports_sse4_1(), "");
3503   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
3504   int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3505   emit_int8(0x22);
3506   emit_int8((unsigned char)(0xC0 | encode));
3507   emit_int8(imm8);
3508 }
3509 
3510 void Assembler::pinsrq(XMMRegister dst, Address src, int imm8) {
3511   assert(VM_Version::supports_sse4_1(), "");
3512   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
3513   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
3514   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3515   emit_int8(0x22);
3516   emit_operand(dst, src);
3517   emit_int8(imm8);
3518 }
3519 
3520 void Assembler::pinsrw(XMMRegister dst, Register src, int imm8) {
3521   assert(VM_Version::supports_sse2(), "");
3522   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
3523   int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3524   emit_int8((unsigned char)0xC4);
3525   emit_int8((unsigned char)(0xC0 | encode));
3526   emit_int8(imm8);
3527 }
3528 
3529 void Assembler::pinsrw(XMMRegister dst, Address src, int imm8) {
3530   assert(VM_Version::supports_sse2(), "");
3531   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
3532   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_16bit);
3533   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3534   emit_int8((unsigned char)0xC4);
3535   emit_operand(dst, src);
3536   emit_int8(imm8);
3537 }
3538 
3539 void Assembler::pinsrb(XMMRegister dst, Address src, int imm8) {
3540   assert(VM_Version::supports_sse4_1(), "");
3541   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
3542   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_8bit);
3543   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3544   emit_int8(0x20);
3545   emit_operand(dst, src);
3546   emit_int8(imm8);
3547 }
3548 
3549 void Assembler::pmovzxbw(XMMRegister dst, Address src) {
3550   assert(VM_Version::supports_sse4_1(), "");
3551   InstructionMark im(this);
3552   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
3553   attributes.set_address_attributes(/* tuple_type */ EVEX_HVM, /* input_size_in_bits */ EVEX_NObit);
3554   simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3555   emit_int8(0x30);
3556   emit_operand(dst, src);
3557 }
3558 
3559 void Assembler::pmovzxbw(XMMRegister dst, XMMRegister src) {
3560   assert(VM_Version::supports_sse4_1(), "");
3561   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
3562   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3563   emit_int8(0x30);
3564   emit_int8((unsigned char)(0xC0 | encode));
3565 }
3566 
3567 void Assembler::vpmovzxbw(XMMRegister dst, Address src, int vector_len) {
3568   assert(VM_Version::supports_avx(), "");
3569   InstructionMark im(this);
3570   assert(dst != xnoreg, "sanity");
3571   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
3572   attributes.set_address_attributes(/* tuple_type */ EVEX_HVM, /* input_size_in_bits */ EVEX_NObit);
3573   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3574   emit_int8(0x30);
3575   emit_operand(dst, src);
3576 }
3577 
3578 // generic
3579 void Assembler::pop(Register dst) {
3580   int encode = prefix_and_encode(dst->encoding());
3581   emit_int8(0x58 | encode);
3582 }
3583 
3584 void Assembler::popcntl(Register dst, Address src) {
3585   assert(VM_Version::supports_popcnt(), "must support");
3586   InstructionMark im(this);
3587   emit_int8((unsigned char)0xF3);
3588   prefix(src, dst);
3589   emit_int8(0x0F);
3590   emit_int8((unsigned char)0xB8);
3591   emit_operand(dst, src);
3592 }
3593 
3594 void Assembler::popcntl(Register dst, Register src) {
3595   assert(VM_Version::supports_popcnt(), "must support");
3596   emit_int8((unsigned char)0xF3);
3597   int encode = prefix_and_encode(dst->encoding(), src->encoding());
3598   emit_int8(0x0F);
3599   emit_int8((unsigned char)0xB8);
3600   emit_int8((unsigned char)(0xC0 | encode));
3601 }
3602 
3603 void Assembler::popf() {
3604   emit_int8((unsigned char)0x9D);
3605 }
3606 
3607 #ifndef _LP64 // no 32bit push/pop on amd64
3608 void Assembler::popl(Address dst) {
3609   // NOTE: this will adjust stack by 8byte on 64bits
3610   InstructionMark im(this);
3611   prefix(dst);
3612   emit_int8((unsigned char)0x8F);
3613   emit_operand(rax, dst);
3614 }
3615 #endif
3616 
3617 void Assembler::prefetch_prefix(Address src) {
3618   prefix(src);
3619   emit_int8(0x0F);
3620 }
3621 
3622 void Assembler::prefetchnta(Address src) {
3623   NOT_LP64(assert(VM_Version::supports_sse(), "must support"));
3624   InstructionMark im(this);
3625   prefetch_prefix(src);
3626   emit_int8(0x18);
3627   emit_operand(rax, src); // 0, src
3628 }
3629 
3630 void Assembler::prefetchr(Address src) {
3631   assert(VM_Version::supports_3dnow_prefetch(), "must support");
3632   InstructionMark im(this);
3633   prefetch_prefix(src);
3634   emit_int8(0x0D);
3635   emit_operand(rax, src); // 0, src
3636 }
3637 
3638 void Assembler::prefetcht0(Address src) {
3639   NOT_LP64(assert(VM_Version::supports_sse(), "must support"));
3640   InstructionMark im(this);
3641   prefetch_prefix(src);
3642   emit_int8(0x18);
3643   emit_operand(rcx, src); // 1, src
3644 }
3645 
3646 void Assembler::prefetcht1(Address src) {
3647   NOT_LP64(assert(VM_Version::supports_sse(), "must support"));
3648   InstructionMark im(this);
3649   prefetch_prefix(src);
3650   emit_int8(0x18);
3651   emit_operand(rdx, src); // 2, src
3652 }
3653 
3654 void Assembler::prefetcht2(Address src) {
3655   NOT_LP64(assert(VM_Version::supports_sse(), "must support"));
3656   InstructionMark im(this);
3657   prefetch_prefix(src);
3658   emit_int8(0x18);
3659   emit_operand(rbx, src); // 3, src
3660 }
3661 
3662 void Assembler::prefetchw(Address src) {
3663   assert(VM_Version::supports_3dnow_prefetch(), "must support");
3664   InstructionMark im(this);
3665   prefetch_prefix(src);
3666   emit_int8(0x0D);
3667   emit_operand(rcx, src); // 1, src
3668 }
3669 
3670 void Assembler::prefix(Prefix p) {
3671   emit_int8(p);
3672 }
3673 
3674 void Assembler::pshufb(XMMRegister dst, XMMRegister src) {
3675   assert(VM_Version::supports_ssse3(), "");
3676   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
3677   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3678   emit_int8(0x00);
3679   emit_int8((unsigned char)(0xC0 | encode));
3680 }
3681 
3682 void Assembler::pshufb(XMMRegister dst, Address src) {
3683   assert(VM_Version::supports_ssse3(), "");
3684   InstructionMark im(this);
3685   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
3686   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
3687   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3688   emit_int8(0x00);
3689   emit_operand(dst, src);
3690 }
3691 
3692 void Assembler::pshufd(XMMRegister dst, XMMRegister src, int mode) {
3693   assert(isByte(mode), "invalid value");
3694   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3695   int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_128bit;
3696   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
3697   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3698   emit_int8(0x70);
3699   emit_int8((unsigned char)(0xC0 | encode));
3700   emit_int8(mode & 0xFF);
3701 }
3702 
3703 void Assembler::pshufd(XMMRegister dst, Address src, int mode) {
3704   assert(isByte(mode), "invalid value");
3705   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3706   assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
3707   InstructionMark im(this);
3708   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
3709   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
3710   simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3711   emit_int8(0x70);
3712   emit_operand(dst, src);
3713   emit_int8(mode & 0xFF);
3714 }
3715 
3716 void Assembler::pshuflw(XMMRegister dst, XMMRegister src, int mode) {
3717   assert(isByte(mode), "invalid value");
3718   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3719   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
3720   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
3721   emit_int8(0x70);
3722   emit_int8((unsigned char)(0xC0 | encode));
3723   emit_int8(mode & 0xFF);
3724 }
3725 
3726 void Assembler::pshuflw(XMMRegister dst, Address src, int mode) {
3727   assert(isByte(mode), "invalid value");
3728   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3729   assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
3730   InstructionMark im(this);
3731   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
3732   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
3733   simd_prefix(dst, xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
3734   emit_int8(0x70);
3735   emit_operand(dst, src);
3736   emit_int8(mode & 0xFF);
3737 }
3738 
3739 void Assembler::psrldq(XMMRegister dst, int shift) {
3740   // Shift left 128 bit value in dst XMMRegister by shift number of bytes.
3741   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3742   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
3743   // XMM3 is for /3 encoding: 66 0F 73 /3 ib
3744   int encode = simd_prefix_and_encode(xmm3, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3745   emit_int8(0x73);
3746   emit_int8((unsigned char)(0xC0 | encode));
3747   emit_int8(shift);
3748 }
3749 
3750 void Assembler::pslldq(XMMRegister dst, int shift) {
3751   // Shift left 128 bit value in dst XMMRegister by shift number of bytes.
3752   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3753   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
3754   // XMM7 is for /7 encoding: 66 0F 73 /7 ib
3755   int encode = simd_prefix_and_encode(xmm7, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3756   emit_int8(0x73);
3757   emit_int8((unsigned char)(0xC0 | encode));
3758   emit_int8(shift);
3759 }
3760 
3761 void Assembler::ptest(XMMRegister dst, Address src) {
3762   assert(VM_Version::supports_sse4_1(), "");
3763   assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
3764   InstructionMark im(this);
3765   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3766   simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3767   emit_int8(0x17);
3768   emit_operand(dst, src);
3769 }
3770 
3771 void Assembler::ptest(XMMRegister dst, XMMRegister src) {
3772   assert(VM_Version::supports_sse4_1(), "");
3773   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3774   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3775   emit_int8(0x17);
3776   emit_int8((unsigned char)(0xC0 | encode));
3777 }
3778 
3779 void Assembler::vptest(XMMRegister dst, Address src) {
3780   assert(VM_Version::supports_avx(), "");
3781   InstructionMark im(this);
3782   InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3783   assert(dst != xnoreg, "sanity");
3784   // swap src<->dst for encoding
3785   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3786   emit_int8(0x17);
3787   emit_operand(dst, src);
3788 }
3789 
3790 void Assembler::vptest(XMMRegister dst, XMMRegister src) {
3791   assert(VM_Version::supports_avx(), "");
3792   InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3793   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3794   emit_int8(0x17);
3795   emit_int8((unsigned char)(0xC0 | encode));
3796 }
3797 
3798 void Assembler::punpcklbw(XMMRegister dst, Address src) {
3799   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3800   assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
3801   InstructionMark im(this);
3802   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_vlbw, /* no_mask_reg */ false, /* uses_vl */ true);
3803   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
3804   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3805   emit_int8(0x60);
3806   emit_operand(dst, src);
3807 }
3808 
3809 void Assembler::punpcklbw(XMMRegister dst, XMMRegister src) {
3810   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3811   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_vlbw, /* no_mask_reg */ false, /* uses_vl */ true);
3812   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3813   emit_int8(0x60);
3814   emit_int8((unsigned char)(0xC0 | encode));
3815 }
3816 
3817 void Assembler::punpckldq(XMMRegister dst, Address src) {
3818   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3819   assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
3820   InstructionMark im(this);
3821   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
3822   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
3823   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3824   emit_int8(0x62);
3825   emit_operand(dst, src);
3826 }
3827 
3828 void Assembler::punpckldq(XMMRegister dst, XMMRegister src) {
3829   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3830   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
3831   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3832   emit_int8(0x62);
3833   emit_int8((unsigned char)(0xC0 | encode));
3834 }
3835 
3836 void Assembler::punpcklqdq(XMMRegister dst, XMMRegister src) {
3837   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3838   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
3839   attributes.set_rex_vex_w_reverted();
3840   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3841   emit_int8(0x6C);
3842   emit_int8((unsigned char)(0xC0 | encode));
3843 }
3844 
3845 void Assembler::push(int32_t imm32) {
3846   // in 64bits we push 64bits onto the stack but only
3847   // take a 32bit immediate
3848   emit_int8(0x68);
3849   emit_int32(imm32);
3850 }
3851 
3852 void Assembler::push(Register src) {
3853   int encode = prefix_and_encode(src->encoding());
3854 
3855   emit_int8(0x50 | encode);
3856 }
3857 
3858 void Assembler::pushf() {
3859   emit_int8((unsigned char)0x9C);
3860 }
3861 
3862 #ifndef _LP64 // no 32bit push/pop on amd64
3863 void Assembler::pushl(Address src) {
3864   // Note this will push 64bit on 64bit
3865   InstructionMark im(this);
3866   prefix(src);
3867   emit_int8((unsigned char)0xFF);
3868   emit_operand(rsi, src);
3869 }
3870 #endif
3871 
3872 void Assembler::rcll(Register dst, int imm8) {
3873   assert(isShiftCount(imm8), "illegal shift count");
3874   int encode = prefix_and_encode(dst->encoding());
3875   if (imm8 == 1) {
3876     emit_int8((unsigned char)0xD1);
3877     emit_int8((unsigned char)(0xD0 | encode));
3878   } else {
3879     emit_int8((unsigned char)0xC1);
3880     emit_int8((unsigned char)0xD0 | encode);
3881     emit_int8(imm8);
3882   }
3883 }
3884 
3885 void Assembler::rcpps(XMMRegister dst, XMMRegister src) {
3886   NOT_LP64(assert(VM_Version::supports_sse(), ""));
3887   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3888   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
3889   emit_int8(0x53);
3890   emit_int8((unsigned char)(0xC0 | encode));
3891 }
3892 
3893 void Assembler::rcpss(XMMRegister dst, XMMRegister src) {
3894   NOT_LP64(assert(VM_Version::supports_sse(), ""));
3895   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3896   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
3897   emit_int8(0x53);
3898   emit_int8((unsigned char)(0xC0 | encode));
3899 }
3900 
3901 void Assembler::rdtsc() {
3902   emit_int8((unsigned char)0x0F);
3903   emit_int8((unsigned char)0x31);
3904 }
3905 
3906 // copies data from [esi] to [edi] using rcx pointer sized words
3907 // generic
3908 void Assembler::rep_mov() {
3909   emit_int8((unsigned char)0xF3);
3910   // MOVSQ
3911   LP64_ONLY(prefix(REX_W));
3912   emit_int8((unsigned char)0xA5);
3913 }
3914 
3915 // sets rcx bytes with rax, value at [edi]
3916 void Assembler::rep_stosb() {
3917   emit_int8((unsigned char)0xF3); // REP
3918   LP64_ONLY(prefix(REX_W));
3919   emit_int8((unsigned char)0xAA); // STOSB
3920 }
3921 
3922 // sets rcx pointer sized words with rax, value at [edi]
3923 // generic
3924 void Assembler::rep_stos() {
3925   emit_int8((unsigned char)0xF3); // REP
3926   LP64_ONLY(prefix(REX_W));       // LP64:STOSQ, LP32:STOSD
3927   emit_int8((unsigned char)0xAB);
3928 }
3929 
3930 // scans rcx pointer sized words at [edi] for occurance of rax,
3931 // generic
3932 void Assembler::repne_scan() { // repne_scan
3933   emit_int8((unsigned char)0xF2);
3934   // SCASQ
3935   LP64_ONLY(prefix(REX_W));
3936   emit_int8((unsigned char)0xAF);
3937 }
3938 
3939 #ifdef _LP64
3940 // scans rcx 4 byte words at [edi] for occurance of rax,
3941 // generic
3942 void Assembler::repne_scanl() { // repne_scan
3943   emit_int8((unsigned char)0xF2);
3944   // SCASL
3945   emit_int8((unsigned char)0xAF);
3946 }
3947 #endif
3948 
3949 void Assembler::ret(int imm16) {
3950   if (imm16 == 0) {
3951     emit_int8((unsigned char)0xC3);
3952   } else {
3953     emit_int8((unsigned char)0xC2);
3954     emit_int16(imm16);
3955   }
3956 }
3957 
3958 void Assembler::sahf() {
3959 #ifdef _LP64
3960   // Not supported in 64bit mode
3961   ShouldNotReachHere();
3962 #endif
3963   emit_int8((unsigned char)0x9E);
3964 }
3965 
3966 void Assembler::sarl(Register dst, int imm8) {
3967   int encode = prefix_and_encode(dst->encoding());
3968   assert(isShiftCount(imm8), "illegal shift count");
3969   if (imm8 == 1) {
3970     emit_int8((unsigned char)0xD1);
3971     emit_int8((unsigned char)(0xF8 | encode));
3972   } else {
3973     emit_int8((unsigned char)0xC1);
3974     emit_int8((unsigned char)(0xF8 | encode));
3975     emit_int8(imm8);
3976   }
3977 }
3978 
3979 void Assembler::sarl(Register dst) {
3980   int encode = prefix_and_encode(dst->encoding());
3981   emit_int8((unsigned char)0xD3);
3982   emit_int8((unsigned char)(0xF8 | encode));
3983 }
3984 
3985 void Assembler::sbbl(Address dst, int32_t imm32) {
3986   InstructionMark im(this);
3987   prefix(dst);
3988   emit_arith_operand(0x81, rbx, dst, imm32);
3989 }
3990 
3991 void Assembler::sbbl(Register dst, int32_t imm32) {
3992   prefix(dst);
3993   emit_arith(0x81, 0xD8, dst, imm32);
3994 }
3995 
3996 
3997 void Assembler::sbbl(Register dst, Address src) {
3998   InstructionMark im(this);
3999   prefix(src, dst);
4000   emit_int8(0x1B);
4001   emit_operand(dst, src);
4002 }
4003 
4004 void Assembler::sbbl(Register dst, Register src) {
4005   (void) prefix_and_encode(dst->encoding(), src->encoding());
4006   emit_arith(0x1B, 0xC0, dst, src);
4007 }
4008 
4009 void Assembler::setb(Condition cc, Register dst) {
4010   assert(0 <= cc && cc < 16, "illegal cc");
4011   int encode = prefix_and_encode(dst->encoding(), true);
4012   emit_int8(0x0F);
4013   emit_int8((unsigned char)0x90 | cc);
4014   emit_int8((unsigned char)(0xC0 | encode));
4015 }
4016 
4017 void Assembler::palignr(XMMRegister dst, XMMRegister src, int imm8) {
4018   assert(VM_Version::supports_ssse3(), "");
4019   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ false);
4020   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
4021   emit_int8((unsigned char)0x0F);
4022   emit_int8((unsigned char)(0xC0 | encode));
4023   emit_int8(imm8);
4024 }
4025 
4026 void Assembler::pblendw(XMMRegister dst, XMMRegister src, int imm8) {
4027   assert(VM_Version::supports_sse4_1(), "");
4028   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
4029   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
4030   emit_int8((unsigned char)0x0E);
4031   emit_int8((unsigned char)(0xC0 | encode));
4032   emit_int8(imm8);
4033 }
4034 
4035 void Assembler::sha1rnds4(XMMRegister dst, XMMRegister src, int imm8) {
4036   assert(VM_Version::supports_sha(), "");
4037   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
4038   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F_3A, &attributes);
4039   emit_int8((unsigned char)0xCC);
4040   emit_int8((unsigned char)(0xC0 | encode));
4041   emit_int8((unsigned char)imm8);
4042 }
4043 
4044 void Assembler::sha1nexte(XMMRegister dst, XMMRegister src) {
4045   assert(VM_Version::supports_sha(), "");
4046   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
4047   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
4048   emit_int8((unsigned char)0xC8);
4049   emit_int8((unsigned char)(0xC0 | encode));
4050 }
4051 
4052 void Assembler::sha1msg1(XMMRegister dst, XMMRegister src) {
4053   assert(VM_Version::supports_sha(), "");
4054   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
4055   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
4056   emit_int8((unsigned char)0xC9);
4057   emit_int8((unsigned char)(0xC0 | encode));
4058 }
4059 
4060 void Assembler::sha1msg2(XMMRegister dst, XMMRegister src) {
4061   assert(VM_Version::supports_sha(), "");
4062   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
4063   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
4064   emit_int8((unsigned char)0xCA);
4065   emit_int8((unsigned char)(0xC0 | encode));
4066 }
4067 
4068 // xmm0 is implicit additional source to this instruction.
4069 void Assembler::sha256rnds2(XMMRegister dst, XMMRegister src) {
4070   assert(VM_Version::supports_sha(), "");
4071   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
4072   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
4073   emit_int8((unsigned char)0xCB);
4074   emit_int8((unsigned char)(0xC0 | encode));
4075 }
4076 
4077 void Assembler::sha256msg1(XMMRegister dst, XMMRegister src) {
4078   assert(VM_Version::supports_sha(), "");
4079   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
4080   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
4081   emit_int8((unsigned char)0xCC);
4082   emit_int8((unsigned char)(0xC0 | encode));
4083 }
4084 
4085 void Assembler::sha256msg2(XMMRegister dst, XMMRegister src) {
4086   assert(VM_Version::supports_sha(), "");
4087   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
4088   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
4089   emit_int8((unsigned char)0xCD);
4090   emit_int8((unsigned char)(0xC0 | encode));
4091 }
4092 
4093 
4094 void Assembler::shll(Register dst, int imm8) {
4095   assert(isShiftCount(imm8), "illegal shift count");
4096   int encode = prefix_and_encode(dst->encoding());
4097   if (imm8 == 1 ) {
4098     emit_int8((unsigned char)0xD1);
4099     emit_int8((unsigned char)(0xE0 | encode));
4100   } else {
4101     emit_int8((unsigned char)0xC1);
4102     emit_int8((unsigned char)(0xE0 | encode));
4103     emit_int8(imm8);
4104   }
4105 }
4106 
4107 void Assembler::shll(Register dst) {
4108   int encode = prefix_and_encode(dst->encoding());
4109   emit_int8((unsigned char)0xD3);
4110   emit_int8((unsigned char)(0xE0 | encode));
4111 }
4112 
4113 void Assembler::shrl(Register dst, int imm8) {
4114   assert(isShiftCount(imm8), "illegal shift count");
4115   int encode = prefix_and_encode(dst->encoding());
4116   emit_int8((unsigned char)0xC1);
4117   emit_int8((unsigned char)(0xE8 | encode));
4118   emit_int8(imm8);
4119 }
4120 
4121 void Assembler::shrl(Register dst) {
4122   int encode = prefix_and_encode(dst->encoding());
4123   emit_int8((unsigned char)0xD3);
4124   emit_int8((unsigned char)(0xE8 | encode));
4125 }
4126 
4127 // copies a single word from [esi] to [edi]
4128 void Assembler::smovl() {
4129   emit_int8((unsigned char)0xA5);
4130 }
4131 
4132 void Assembler::sqrtsd(XMMRegister dst, XMMRegister src) {
4133   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4134   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
4135   attributes.set_rex_vex_w_reverted();
4136   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
4137   emit_int8(0x51);
4138   emit_int8((unsigned char)(0xC0 | encode));
4139 }
4140 
4141 void Assembler::sqrtsd(XMMRegister dst, Address src) {
4142   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4143   InstructionMark im(this);
4144   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
4145   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
4146   attributes.set_rex_vex_w_reverted();
4147   simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
4148   emit_int8(0x51);
4149   emit_operand(dst, src);
4150 }
4151 
4152 void Assembler::sqrtss(XMMRegister dst, XMMRegister src) {
4153   NOT_LP64(assert(VM_Version::supports_sse(), ""));
4154   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
4155   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
4156   emit_int8(0x51);
4157   emit_int8((unsigned char)(0xC0 | encode));
4158 }
4159 
4160 void Assembler::std() {
4161   emit_int8((unsigned char)0xFD);
4162 }
4163 
4164 void Assembler::sqrtss(XMMRegister dst, Address src) {
4165   NOT_LP64(assert(VM_Version::supports_sse(), ""));
4166   InstructionMark im(this);
4167   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
4168   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
4169   simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
4170   emit_int8(0x51);
4171   emit_operand(dst, src);
4172 }
4173 
4174 void Assembler::stmxcsr( Address dst) {
4175   NOT_LP64(assert(VM_Version::supports_sse(), ""));
4176   InstructionMark im(this);
4177   prefix(dst);
4178   emit_int8(0x0F);
4179   emit_int8((unsigned char)0xAE);
4180   emit_operand(as_Register(3), dst);
4181 }
4182 
4183 void Assembler::subl(Address dst, int32_t imm32) {
4184   InstructionMark im(this);
4185   prefix(dst);
4186   emit_arith_operand(0x81, rbp, dst, imm32);
4187 }
4188 
4189 void Assembler::subl(Address dst, Register src) {
4190   InstructionMark im(this);
4191   prefix(dst, src);
4192   emit_int8(0x29);
4193   emit_operand(src, dst);
4194 }
4195 
4196 void Assembler::subl(Register dst, int32_t imm32) {
4197   prefix(dst);
4198   emit_arith(0x81, 0xE8, dst, imm32);
4199 }
4200 
4201 // Force generation of a 4 byte immediate value even if it fits into 8bit
4202 void Assembler::subl_imm32(Register dst, int32_t imm32) {
4203   prefix(dst);
4204   emit_arith_imm32(0x81, 0xE8, dst, imm32);
4205 }
4206 
4207 void Assembler::subl(Register dst, Address src) {
4208   InstructionMark im(this);
4209   prefix(src, dst);
4210   emit_int8(0x2B);
4211   emit_operand(dst, src);
4212 }
4213 
4214 void Assembler::subl(Register dst, Register src) {
4215   (void) prefix_and_encode(dst->encoding(), src->encoding());
4216   emit_arith(0x2B, 0xC0, dst, src);
4217 }
4218 
4219 void Assembler::subsd(XMMRegister dst, XMMRegister src) {
4220   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4221   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
4222   attributes.set_rex_vex_w_reverted();
4223   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
4224   emit_int8(0x5C);
4225   emit_int8((unsigned char)(0xC0 | encode));
4226 }
4227 
4228 void Assembler::subsd(XMMRegister dst, Address src) {
4229   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4230   InstructionMark im(this);
4231   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
4232   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
4233   attributes.set_rex_vex_w_reverted();
4234   simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
4235   emit_int8(0x5C);
4236   emit_operand(dst, src);
4237 }
4238 
4239 void Assembler::subss(XMMRegister dst, XMMRegister src) {
4240   NOT_LP64(assert(VM_Version::supports_sse(), ""));
4241   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false , /* uses_vl */ false);
4242   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
4243   emit_int8(0x5C);
4244   emit_int8((unsigned char)(0xC0 | encode));
4245 }
4246 
4247 void Assembler::subss(XMMRegister dst, Address src) {
4248   NOT_LP64(assert(VM_Version::supports_sse(), ""));
4249   InstructionMark im(this);
4250   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
4251   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
4252   simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
4253   emit_int8(0x5C);
4254   emit_operand(dst, src);
4255 }
4256 
4257 void Assembler::testb(Register dst, int imm8) {
4258   NOT_LP64(assert(dst->has_byte_register(), "must have byte register"));
4259   (void) prefix_and_encode(dst->encoding(), true);
4260   emit_arith_b(0xF6, 0xC0, dst, imm8);
4261 }
4262 
4263 void Assembler::testb(Address dst, int imm8) {
4264   InstructionMark im(this);
4265   prefix(dst);
4266   emit_int8((unsigned char)0xF6);
4267   emit_operand(rax, dst, 1);
4268   emit_int8(imm8);
4269 }
4270 
4271 void Assembler::testl(Register dst, int32_t imm32) {
4272   // not using emit_arith because test
4273   // doesn't support sign-extension of
4274   // 8bit operands
4275   int encode = dst->encoding();
4276   if (encode == 0) {
4277     emit_int8((unsigned char)0xA9);
4278   } else {
4279     encode = prefix_and_encode(encode);
4280     emit_int8((unsigned char)0xF7);
4281     emit_int8((unsigned char)(0xC0 | encode));
4282   }
4283   emit_int32(imm32);
4284 }
4285 
4286 void Assembler::testl(Register dst, Register src) {
4287   (void) prefix_and_encode(dst->encoding(), src->encoding());
4288   emit_arith(0x85, 0xC0, dst, src);
4289 }
4290 
4291 void Assembler::testl(Register dst, Address src) {
4292   InstructionMark im(this);
4293   prefix(src, dst);
4294   emit_int8((unsigned char)0x85);
4295   emit_operand(dst, src);
4296 }
4297 
4298 void Assembler::tzcntl(Register dst, Register src) {
4299   assert(VM_Version::supports_bmi1(), "tzcnt instruction not supported");
4300   emit_int8((unsigned char)0xF3);
4301   int encode = prefix_and_encode(dst->encoding(), src->encoding());
4302   emit_int8(0x0F);
4303   emit_int8((unsigned char)0xBC);
4304   emit_int8((unsigned char)0xC0 | encode);
4305 }
4306 
4307 void Assembler::tzcntq(Register dst, Register src) {
4308   assert(VM_Version::supports_bmi1(), "tzcnt instruction not supported");
4309   emit_int8((unsigned char)0xF3);
4310   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
4311   emit_int8(0x0F);
4312   emit_int8((unsigned char)0xBC);
4313   emit_int8((unsigned char)(0xC0 | encode));
4314 }
4315 
4316 void Assembler::ucomisd(XMMRegister dst, Address src) {
4317   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4318   InstructionMark im(this);
4319   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4320   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
4321   attributes.set_rex_vex_w_reverted();
4322   simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4323   emit_int8(0x2E);
4324   emit_operand(dst, src);
4325 }
4326 
4327 void Assembler::ucomisd(XMMRegister dst, XMMRegister src) {
4328   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4329   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4330   attributes.set_rex_vex_w_reverted();
4331   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4332   emit_int8(0x2E);
4333   emit_int8((unsigned char)(0xC0 | encode));
4334 }
4335 
4336 void Assembler::ucomiss(XMMRegister dst, Address src) {
4337   NOT_LP64(assert(VM_Version::supports_sse(), ""));
4338   InstructionMark im(this);
4339   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4340   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
4341   simd_prefix(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
4342   emit_int8(0x2E);
4343   emit_operand(dst, src);
4344 }
4345 
4346 void Assembler::ucomiss(XMMRegister dst, XMMRegister src) {
4347   NOT_LP64(assert(VM_Version::supports_sse(), ""));
4348   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4349   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
4350   emit_int8(0x2E);
4351   emit_int8((unsigned char)(0xC0 | encode));
4352 }
4353 
4354 void Assembler::xabort(int8_t imm8) {
4355   emit_int8((unsigned char)0xC6);
4356   emit_int8((unsigned char)0xF8);
4357   emit_int8((unsigned char)(imm8 & 0xFF));
4358 }
4359 
4360 void Assembler::xaddl(Address dst, Register src) {
4361   InstructionMark im(this);
4362   prefix(dst, src);
4363   emit_int8(0x0F);
4364   emit_int8((unsigned char)0xC1);
4365   emit_operand(src, dst);
4366 }
4367 
4368 void Assembler::xbegin(Label& abort, relocInfo::relocType rtype) {
4369   InstructionMark im(this);
4370   relocate(rtype);
4371   if (abort.is_bound()) {
4372     address entry = target(abort);
4373     assert(entry != NULL, "abort entry NULL");
4374     intptr_t offset = entry - pc();
4375     emit_int8((unsigned char)0xC7);
4376     emit_int8((unsigned char)0xF8);
4377     emit_int32(offset - 6); // 2 opcode + 4 address
4378   } else {
4379     abort.add_patch_at(code(), locator());
4380     emit_int8((unsigned char)0xC7);
4381     emit_int8((unsigned char)0xF8);
4382     emit_int32(0);
4383   }
4384 }
4385 
4386 void Assembler::xchgl(Register dst, Address src) { // xchg
4387   InstructionMark im(this);
4388   prefix(src, dst);
4389   emit_int8((unsigned char)0x87);
4390   emit_operand(dst, src);
4391 }
4392 
4393 void Assembler::xchgl(Register dst, Register src) {
4394   int encode = prefix_and_encode(dst->encoding(), src->encoding());
4395   emit_int8((unsigned char)0x87);
4396   emit_int8((unsigned char)(0xC0 | encode));
4397 }
4398 
4399 void Assembler::xend() {
4400   emit_int8((unsigned char)0x0F);
4401   emit_int8((unsigned char)0x01);
4402   emit_int8((unsigned char)0xD5);
4403 }
4404 
4405 void Assembler::xgetbv() {
4406   emit_int8(0x0F);
4407   emit_int8(0x01);
4408   emit_int8((unsigned char)0xD0);
4409 }
4410 
4411 void Assembler::xorl(Register dst, int32_t imm32) {
4412   prefix(dst);
4413   emit_arith(0x81, 0xF0, dst, imm32);
4414 }
4415 
4416 void Assembler::xorl(Register dst, Address src) {
4417   InstructionMark im(this);
4418   prefix(src, dst);
4419   emit_int8(0x33);
4420   emit_operand(dst, src);
4421 }
4422 
4423 void Assembler::xorl(Register dst, Register src) {
4424   (void) prefix_and_encode(dst->encoding(), src->encoding());
4425   emit_arith(0x33, 0xC0, dst, src);
4426 }
4427 
4428 void Assembler::xorb(Register dst, Address src) {
4429   InstructionMark im(this);
4430   prefix(src, dst);
4431   emit_int8(0x32);
4432   emit_operand(dst, src);
4433 }
4434 
4435 // AVX 3-operands scalar float-point arithmetic instructions
4436 
4437 void Assembler::vaddsd(XMMRegister dst, XMMRegister nds, Address src) {
4438   assert(VM_Version::supports_avx(), "");
4439   InstructionMark im(this);
4440   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
4441   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
4442   attributes.set_rex_vex_w_reverted();
4443   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
4444   emit_int8(0x58);
4445   emit_operand(dst, src);
4446 }
4447 
4448 void Assembler::vaddsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
4449   assert(VM_Version::supports_avx(), "");
4450   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
4451   attributes.set_rex_vex_w_reverted();
4452   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
4453   emit_int8(0x58);
4454   emit_int8((unsigned char)(0xC0 | encode));
4455 }
4456 
4457 void Assembler::vaddss(XMMRegister dst, XMMRegister nds, Address src) {
4458   assert(VM_Version::supports_avx(), "");
4459   InstructionMark im(this);
4460   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
4461   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
4462   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
4463   emit_int8(0x58);
4464   emit_operand(dst, src);
4465 }
4466 
4467 void Assembler::vaddss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
4468   assert(VM_Version::supports_avx(), "");
4469   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
4470   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
4471   emit_int8(0x58);
4472   emit_int8((unsigned char)(0xC0 | encode));
4473 }
4474 
4475 void Assembler::vdivsd(XMMRegister dst, XMMRegister nds, Address src) {
4476   assert(VM_Version::supports_avx(), "");
4477   InstructionMark im(this);
4478   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
4479   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
4480   attributes.set_rex_vex_w_reverted();
4481   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
4482   emit_int8(0x5E);
4483   emit_operand(dst, src);
4484 }
4485 
4486 void Assembler::vdivsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
4487   assert(VM_Version::supports_avx(), "");
4488   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
4489   attributes.set_rex_vex_w_reverted();
4490   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
4491   emit_int8(0x5E);
4492   emit_int8((unsigned char)(0xC0 | encode));
4493 }
4494 
4495 void Assembler::vdivss(XMMRegister dst, XMMRegister nds, Address src) {
4496   assert(VM_Version::supports_avx(), "");
4497   InstructionMark im(this);
4498   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
4499   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
4500   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
4501   emit_int8(0x5E);
4502   emit_operand(dst, src);
4503 }
4504 
4505 void Assembler::vdivss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
4506   assert(VM_Version::supports_avx(), "");
4507   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
4508   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
4509   emit_int8(0x5E);
4510   emit_int8((unsigned char)(0xC0 | encode));
4511 }
4512 
4513 void Assembler::vmulsd(XMMRegister dst, XMMRegister nds, Address src) {
4514   assert(VM_Version::supports_avx(), "");
4515   InstructionMark im(this);
4516   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
4517   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
4518   attributes.set_rex_vex_w_reverted();
4519   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
4520   emit_int8(0x59);
4521   emit_operand(dst, src);
4522 }
4523 
4524 void Assembler::vmulsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
4525   assert(VM_Version::supports_avx(), "");
4526   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
4527   attributes.set_rex_vex_w_reverted();
4528   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
4529   emit_int8(0x59);
4530   emit_int8((unsigned char)(0xC0 | encode));
4531 }
4532 
4533 void Assembler::vmulss(XMMRegister dst, XMMRegister nds, Address src) {
4534   assert(VM_Version::supports_avx(), "");
4535   InstructionMark im(this);
4536   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
4537   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
4538   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
4539   emit_int8(0x59);
4540   emit_operand(dst, src);
4541 }
4542 
4543 void Assembler::vmulss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
4544   assert(VM_Version::supports_avx(), "");
4545   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
4546   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
4547   emit_int8(0x59);
4548   emit_int8((unsigned char)(0xC0 | encode));
4549 }
4550 
4551 void Assembler::vsubsd(XMMRegister dst, XMMRegister nds, Address src) {
4552   assert(VM_Version::supports_avx(), "");
4553   InstructionMark im(this);
4554   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
4555   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
4556   attributes.set_rex_vex_w_reverted();
4557   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
4558   emit_int8(0x5C);
4559   emit_operand(dst, src);
4560 }
4561 
4562 void Assembler::vsubsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
4563   assert(VM_Version::supports_avx(), "");
4564   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
4565   attributes.set_rex_vex_w_reverted();
4566   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
4567   emit_int8(0x5C);
4568   emit_int8((unsigned char)(0xC0 | encode));
4569 }
4570 
4571 void Assembler::vsubss(XMMRegister dst, XMMRegister nds, Address src) {
4572   assert(VM_Version::supports_avx(), "");
4573   InstructionMark im(this);
4574   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
4575   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
4576   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
4577   emit_int8(0x5C);
4578   emit_operand(dst, src);
4579 }
4580 
4581 void Assembler::vsubss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
4582   assert(VM_Version::supports_avx(), "");
4583   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
4584   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
4585   emit_int8(0x5C);
4586   emit_int8((unsigned char)(0xC0 | encode));
4587 }
4588 
4589 //====================VECTOR ARITHMETIC=====================================
4590 
4591 // Float-point vector arithmetic
4592 
4593 void Assembler::addpd(XMMRegister dst, XMMRegister src) {
4594   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4595   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4596   attributes.set_rex_vex_w_reverted();
4597   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4598   emit_int8(0x58);
4599   emit_int8((unsigned char)(0xC0 | encode));
4600 }
4601 
4602 void Assembler::addpd(XMMRegister dst, Address src) {
4603   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4604   InstructionMark im(this);
4605   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4606   attributes.set_rex_vex_w_reverted();
4607   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
4608   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4609   emit_int8(0x58);
4610   emit_operand(dst, src);
4611 }
4612 
4613 
4614 void Assembler::addps(XMMRegister dst, XMMRegister src) {
4615   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4616   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4617   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
4618   emit_int8(0x58);
4619   emit_int8((unsigned char)(0xC0 | encode));
4620 }
4621 
4622 void Assembler::vaddpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4623   assert(VM_Version::supports_avx(), "");
4624   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4625   attributes.set_rex_vex_w_reverted();
4626   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4627   emit_int8(0x58);
4628   emit_int8((unsigned char)(0xC0 | encode));
4629 }
4630 
4631 void Assembler::vaddps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4632   assert(VM_Version::supports_avx(), "");
4633   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4634   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
4635   emit_int8(0x58);
4636   emit_int8((unsigned char)(0xC0 | encode));
4637 }
4638 
4639 void Assembler::vaddpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4640   assert(VM_Version::supports_avx(), "");
4641   InstructionMark im(this);
4642   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4643   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
4644   attributes.set_rex_vex_w_reverted();
4645   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4646   emit_int8(0x58);
4647   emit_operand(dst, src);
4648 }
4649 
4650 void Assembler::vaddps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4651   assert(VM_Version::supports_avx(), "");
4652   InstructionMark im(this);
4653   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4654   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
4655   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
4656   emit_int8(0x58);
4657   emit_operand(dst, src);
4658 }
4659 
4660 void Assembler::subpd(XMMRegister dst, XMMRegister src) {
4661   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4662   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4663   attributes.set_rex_vex_w_reverted();
4664   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4665   emit_int8(0x5C);
4666   emit_int8((unsigned char)(0xC0 | encode));
4667 }
4668 
4669 void Assembler::subps(XMMRegister dst, XMMRegister src) {
4670   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4671   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4672   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
4673   emit_int8(0x5C);
4674   emit_int8((unsigned char)(0xC0 | encode));
4675 }
4676 
4677 void Assembler::vsubpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4678   assert(VM_Version::supports_avx(), "");
4679   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4680   attributes.set_rex_vex_w_reverted();
4681   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4682   emit_int8(0x5C);
4683   emit_int8((unsigned char)(0xC0 | encode));
4684 }
4685 
4686 void Assembler::vsubps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4687   assert(VM_Version::supports_avx(), "");
4688   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4689   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
4690   emit_int8(0x5C);
4691   emit_int8((unsigned char)(0xC0 | encode));
4692 }
4693 
4694 void Assembler::vsubpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4695   assert(VM_Version::supports_avx(), "");
4696   InstructionMark im(this);
4697   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4698   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
4699   attributes.set_rex_vex_w_reverted();
4700   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4701   emit_int8(0x5C);
4702   emit_operand(dst, src);
4703 }
4704 
4705 void Assembler::vsubps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4706   assert(VM_Version::supports_avx(), "");
4707   InstructionMark im(this);
4708   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4709   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
4710   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
4711   emit_int8(0x5C);
4712   emit_operand(dst, src);
4713 }
4714 
4715 void Assembler::mulpd(XMMRegister dst, XMMRegister src) {
4716   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4717   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4718   attributes.set_rex_vex_w_reverted();
4719   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4720   emit_int8(0x59);
4721   emit_int8((unsigned char)(0xC0 | encode));
4722 }
4723 
4724 void Assembler::mulpd(XMMRegister dst, Address src) {
4725   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4726   InstructionMark im(this);
4727   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4728   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
4729   attributes.set_rex_vex_w_reverted();
4730   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4731   emit_int8(0x59);
4732   emit_operand(dst, src);
4733 }
4734 
4735 void Assembler::mulps(XMMRegister dst, XMMRegister src) {
4736   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4737   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4738   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
4739   emit_int8(0x59);
4740   emit_int8((unsigned char)(0xC0 | encode));
4741 }
4742 
4743 void Assembler::vmulpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4744   assert(VM_Version::supports_avx(), "");
4745   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4746   attributes.set_rex_vex_w_reverted();
4747   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4748   emit_int8(0x59);
4749   emit_int8((unsigned char)(0xC0 | encode));
4750 }
4751 
4752 void Assembler::vmulps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4753   assert(VM_Version::supports_avx(), "");
4754   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4755   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
4756   emit_int8(0x59);
4757   emit_int8((unsigned char)(0xC0 | encode));
4758 }
4759 
4760 void Assembler::vmulpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4761   assert(VM_Version::supports_avx(), "");
4762   InstructionMark im(this);
4763   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4764   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
4765   attributes.set_rex_vex_w_reverted();
4766   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4767   emit_int8(0x59);
4768   emit_operand(dst, src);
4769 }
4770 
4771 void Assembler::vmulps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4772   assert(VM_Version::supports_avx(), "");
4773   InstructionMark im(this);
4774   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4775   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
4776   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
4777   emit_int8(0x59);
4778   emit_operand(dst, src);
4779 }
4780 
4781 void Assembler::divpd(XMMRegister dst, XMMRegister src) {
4782   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4783   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4784   attributes.set_rex_vex_w_reverted();
4785   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4786   emit_int8(0x5E);
4787   emit_int8((unsigned char)(0xC0 | encode));
4788 }
4789 
4790 void Assembler::divps(XMMRegister dst, XMMRegister src) {
4791   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4792   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4793   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
4794   emit_int8(0x5E);
4795   emit_int8((unsigned char)(0xC0 | encode));
4796 }
4797 
4798 void Assembler::vdivpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4799   assert(VM_Version::supports_avx(), "");
4800   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4801   attributes.set_rex_vex_w_reverted();
4802   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4803   emit_int8(0x5E);
4804   emit_int8((unsigned char)(0xC0 | encode));
4805 }
4806 
4807 void Assembler::vdivps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4808   assert(VM_Version::supports_avx(), "");
4809   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4810   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
4811   emit_int8(0x5E);
4812   emit_int8((unsigned char)(0xC0 | encode));
4813 }
4814 
4815 void Assembler::vdivpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4816   assert(VM_Version::supports_avx(), "");
4817   InstructionMark im(this);
4818   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4819   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
4820   attributes.set_rex_vex_w_reverted();
4821   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4822   emit_int8(0x5E);
4823   emit_operand(dst, src);
4824 }
4825 
4826 void Assembler::vdivps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4827   assert(VM_Version::supports_avx(), "");
4828   InstructionMark im(this);
4829   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4830   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
4831   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
4832   emit_int8(0x5E);
4833   emit_operand(dst, src);
4834 }
4835 
4836 void Assembler::vsqrtpd(XMMRegister dst, XMMRegister src, int vector_len) {
4837   assert(VM_Version::supports_avx(), "");
4838   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4839   attributes.set_rex_vex_w_reverted();
4840   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4841   emit_int8(0x51);
4842   emit_int8((unsigned char)(0xC0 | encode));
4843 }
4844 
4845 void Assembler::vsqrtpd(XMMRegister dst, Address src, int vector_len) {
4846   assert(VM_Version::supports_avx(), "");
4847   InstructionMark im(this);
4848   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4849   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
4850   attributes.set_rex_vex_w_reverted();
4851   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4852   emit_int8(0x51);
4853   emit_operand(dst, src);
4854 }
4855 
4856 void Assembler::andpd(XMMRegister dst, XMMRegister src) {
4857   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4858   InstructionAttr attributes(AVX_128bit, /* rex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
4859   attributes.set_rex_vex_w_reverted();
4860   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4861   emit_int8(0x54);
4862   emit_int8((unsigned char)(0xC0 | encode));
4863 }
4864 
4865 void Assembler::andps(XMMRegister dst, XMMRegister src) {
4866   NOT_LP64(assert(VM_Version::supports_sse(), ""));
4867   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
4868   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
4869   emit_int8(0x54);
4870   emit_int8((unsigned char)(0xC0 | encode));
4871 }
4872 
4873 void Assembler::andps(XMMRegister dst, Address src) {
4874   NOT_LP64(assert(VM_Version::supports_sse(), ""));
4875   InstructionMark im(this);
4876   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
4877   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
4878   simd_prefix(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
4879   emit_int8(0x54);
4880   emit_operand(dst, src);
4881 }
4882 
4883 void Assembler::andpd(XMMRegister dst, Address src) {
4884   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4885   InstructionMark im(this);
4886   InstructionAttr attributes(AVX_128bit, /* rex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
4887   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
4888   attributes.set_rex_vex_w_reverted();
4889   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4890   emit_int8(0x54);
4891   emit_operand(dst, src);
4892 }
4893 
4894 void Assembler::vandpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4895   assert(VM_Version::supports_avx(), "");
4896   InstructionAttr attributes(vector_len, /* vex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
4897   attributes.set_rex_vex_w_reverted();
4898   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4899   emit_int8(0x54);
4900   emit_int8((unsigned char)(0xC0 | encode));
4901 }
4902 
4903 void Assembler::vandps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4904   assert(VM_Version::supports_avx(), "");
4905   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
4906   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
4907   emit_int8(0x54);
4908   emit_int8((unsigned char)(0xC0 | encode));
4909 }
4910 
4911 void Assembler::vandpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4912   assert(VM_Version::supports_avx(), "");
4913   InstructionMark im(this);
4914   InstructionAttr attributes(vector_len, /* vex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
4915   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
4916   attributes.set_rex_vex_w_reverted();
4917   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4918   emit_int8(0x54);
4919   emit_operand(dst, src);
4920 }
4921 
4922 void Assembler::vandps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4923   assert(VM_Version::supports_avx(), "");
4924   InstructionMark im(this);
4925   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
4926   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
4927   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
4928   emit_int8(0x54);
4929   emit_operand(dst, src);
4930 }
4931 
4932 void Assembler::unpckhpd(XMMRegister dst, XMMRegister src) {
4933   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4934   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4935   attributes.set_rex_vex_w_reverted();
4936   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4937   emit_int8(0x15);
4938   emit_int8((unsigned char)(0xC0 | encode));
4939 }
4940 
4941 void Assembler::unpcklpd(XMMRegister dst, XMMRegister src) {
4942   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4943   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4944   attributes.set_rex_vex_w_reverted();
4945   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4946   emit_int8(0x14);
4947   emit_int8((unsigned char)(0xC0 | encode));
4948 }
4949 
4950 void Assembler::xorpd(XMMRegister dst, XMMRegister src) {
4951   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4952   InstructionAttr attributes(AVX_128bit, /* rex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
4953   attributes.set_rex_vex_w_reverted();
4954   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4955   emit_int8(0x57);
4956   emit_int8((unsigned char)(0xC0 | encode));
4957 }
4958 
4959 void Assembler::xorps(XMMRegister dst, XMMRegister src) {
4960   NOT_LP64(assert(VM_Version::supports_sse(), ""));
4961   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
4962   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
4963   emit_int8(0x57);
4964   emit_int8((unsigned char)(0xC0 | encode));
4965 }
4966 
4967 void Assembler::xorpd(XMMRegister dst, Address src) {
4968   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4969   InstructionMark im(this);
4970   InstructionAttr attributes(AVX_128bit, /* rex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
4971   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
4972   attributes.set_rex_vex_w_reverted();
4973   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4974   emit_int8(0x57);
4975   emit_operand(dst, src);
4976 }
4977 
4978 void Assembler::xorps(XMMRegister dst, Address src) {
4979   NOT_LP64(assert(VM_Version::supports_sse(), ""));
4980   InstructionMark im(this);
4981   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
4982   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
4983   simd_prefix(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
4984   emit_int8(0x57);
4985   emit_operand(dst, src);
4986 }
4987 
4988 void Assembler::vxorpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4989   assert(VM_Version::supports_avx(), "");
4990   InstructionAttr attributes(vector_len, /* vex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
4991   attributes.set_rex_vex_w_reverted();
4992   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4993   emit_int8(0x57);
4994   emit_int8((unsigned char)(0xC0 | encode));
4995 }
4996 
4997 void Assembler::vxorps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4998   assert(VM_Version::supports_avx(), "");
4999   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
5000   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
5001   emit_int8(0x57);
5002   emit_int8((unsigned char)(0xC0 | encode));
5003 }
5004 
5005 void Assembler::vxorpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5006   assert(VM_Version::supports_avx(), "");
5007   InstructionMark im(this);
5008   InstructionAttr attributes(vector_len, /* vex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
5009   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
5010   attributes.set_rex_vex_w_reverted();
5011   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5012   emit_int8(0x57);
5013   emit_operand(dst, src);
5014 }
5015 
5016 void Assembler::vxorps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5017   assert(VM_Version::supports_avx(), "");
5018   InstructionMark im(this);
5019   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
5020   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
5021   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
5022   emit_int8(0x57);
5023   emit_operand(dst, src);
5024 }
5025 
5026 // Integer vector arithmetic
5027 void Assembler::vphaddw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5028   assert(VM_Version::supports_avx() && (vector_len == 0) ||
5029          VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
5030   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
5031   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5032   emit_int8(0x01);
5033   emit_int8((unsigned char)(0xC0 | encode));
5034 }
5035 
5036 void Assembler::vphaddd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5037   assert(VM_Version::supports_avx() && (vector_len == 0) ||
5038          VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
5039   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
5040   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5041   emit_int8(0x02);
5042   emit_int8((unsigned char)(0xC0 | encode));
5043 }
5044 
5045 void Assembler::paddb(XMMRegister dst, XMMRegister src) {
5046   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5047   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5048   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5049   emit_int8((unsigned char)0xFC);
5050   emit_int8((unsigned char)(0xC0 | encode));
5051 }
5052 
5053 void Assembler::paddw(XMMRegister dst, XMMRegister src) {
5054   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5055   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5056   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5057   emit_int8((unsigned char)0xFD);
5058   emit_int8((unsigned char)(0xC0 | encode));
5059 }
5060 
5061 void Assembler::paddd(XMMRegister dst, XMMRegister src) {
5062   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5063   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5064   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5065   emit_int8((unsigned char)0xFE);
5066   emit_int8((unsigned char)(0xC0 | encode));
5067 }
5068 
5069 void Assembler::paddd(XMMRegister dst, Address src) {
5070   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5071   InstructionMark im(this);
5072   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5073   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5074   emit_int8((unsigned char)0xFE);
5075   emit_operand(dst, src);
5076 }
5077 
5078 void Assembler::paddq(XMMRegister dst, XMMRegister src) {
5079   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5080   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5081   attributes.set_rex_vex_w_reverted();
5082   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5083   emit_int8((unsigned char)0xD4);
5084   emit_int8((unsigned char)(0xC0 | encode));
5085 }
5086 
5087 void Assembler::phaddw(XMMRegister dst, XMMRegister src) {
5088   assert(VM_Version::supports_sse3(), "");
5089   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
5090   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5091   emit_int8(0x01);
5092   emit_int8((unsigned char)(0xC0 | encode));
5093 }
5094 
5095 void Assembler::phaddd(XMMRegister dst, XMMRegister src) {
5096   assert(VM_Version::supports_sse3(), "");
5097   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
5098   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5099   emit_int8(0x02);
5100   emit_int8((unsigned char)(0xC0 | encode));
5101 }
5102 
5103 void Assembler::vpaddb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5104   assert(UseAVX > 0, "requires some form of AVX");
5105   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5106   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5107   emit_int8((unsigned char)0xFC);
5108   emit_int8((unsigned char)(0xC0 | encode));
5109 }
5110 
5111 void Assembler::vpaddw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5112   assert(UseAVX > 0, "requires some form of AVX");
5113   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5114   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5115   emit_int8((unsigned char)0xFD);
5116   emit_int8((unsigned char)(0xC0 | encode));
5117 }
5118 
5119 void Assembler::vpaddd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5120   assert(UseAVX > 0, "requires some form of AVX");
5121   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5122   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5123   emit_int8((unsigned char)0xFE);
5124   emit_int8((unsigned char)(0xC0 | encode));
5125 }
5126 
5127 void Assembler::vpaddq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5128   assert(UseAVX > 0, "requires some form of AVX");
5129   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5130   attributes.set_rex_vex_w_reverted();
5131   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5132   emit_int8((unsigned char)0xD4);
5133   emit_int8((unsigned char)(0xC0 | encode));
5134 }
5135 
5136 void Assembler::vpaddb(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5137   assert(UseAVX > 0, "requires some form of AVX");
5138   InstructionMark im(this);
5139   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5140   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
5141   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5142   emit_int8((unsigned char)0xFC);
5143   emit_operand(dst, src);
5144 }
5145 
5146 void Assembler::vpaddw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5147   assert(UseAVX > 0, "requires some form of AVX");
5148   InstructionMark im(this);
5149   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5150   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
5151   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5152   emit_int8((unsigned char)0xFD);
5153   emit_operand(dst, src);
5154 }
5155 
5156 void Assembler::vpaddd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5157   assert(UseAVX > 0, "requires some form of AVX");
5158   InstructionMark im(this);
5159   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5160   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
5161   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5162   emit_int8((unsigned char)0xFE);
5163   emit_operand(dst, src);
5164 }
5165 
5166 void Assembler::vpaddq(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5167   assert(UseAVX > 0, "requires some form of AVX");
5168   InstructionMark im(this);
5169   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5170   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
5171   attributes.set_rex_vex_w_reverted();
5172   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5173   emit_int8((unsigned char)0xD4);
5174   emit_operand(dst, src);
5175 }
5176 
5177 void Assembler::psubb(XMMRegister dst, XMMRegister src) {
5178   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5179   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5180   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5181   emit_int8((unsigned char)0xF8);
5182   emit_int8((unsigned char)(0xC0 | encode));
5183 }
5184 
5185 void Assembler::psubw(XMMRegister dst, XMMRegister src) {
5186   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5187   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5188   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5189   emit_int8((unsigned char)0xF9);
5190   emit_int8((unsigned char)(0xC0 | encode));
5191 }
5192 
5193 void Assembler::psubd(XMMRegister dst, XMMRegister src) {
5194   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5195   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5196   emit_int8((unsigned char)0xFA);
5197   emit_int8((unsigned char)(0xC0 | encode));
5198 }
5199 
5200 void Assembler::psubq(XMMRegister dst, XMMRegister src) {
5201   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5202   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5203   attributes.set_rex_vex_w_reverted();
5204   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5205   emit_int8((unsigned char)0xFB);
5206   emit_int8((unsigned char)(0xC0 | encode));
5207 }
5208 
5209 void Assembler::vpsubb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5210   assert(UseAVX > 0, "requires some form of AVX");
5211   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5212   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5213   emit_int8((unsigned char)0xF8);
5214   emit_int8((unsigned char)(0xC0 | encode));
5215 }
5216 
5217 void Assembler::vpsubw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5218   assert(UseAVX > 0, "requires some form of AVX");
5219   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5220   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5221   emit_int8((unsigned char)0xF9);
5222   emit_int8((unsigned char)(0xC0 | encode));
5223 }
5224 
5225 void Assembler::vpsubd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5226   assert(UseAVX > 0, "requires some form of AVX");
5227   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5228   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5229   emit_int8((unsigned char)0xFA);
5230   emit_int8((unsigned char)(0xC0 | encode));
5231 }
5232 
5233 void Assembler::vpsubq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5234   assert(UseAVX > 0, "requires some form of AVX");
5235   InstructionAttr attributes(vector_len, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5236   attributes.set_rex_vex_w_reverted();
5237   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5238   emit_int8((unsigned char)0xFB);
5239   emit_int8((unsigned char)(0xC0 | encode));
5240 }
5241 
5242 void Assembler::vpsubb(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5243   assert(UseAVX > 0, "requires some form of AVX");
5244   InstructionMark im(this);
5245   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5246   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
5247   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5248   emit_int8((unsigned char)0xF8);
5249   emit_operand(dst, src);
5250 }
5251 
5252 void Assembler::vpsubw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5253   assert(UseAVX > 0, "requires some form of AVX");
5254   InstructionMark im(this);
5255   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5256   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
5257   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5258   emit_int8((unsigned char)0xF9);
5259   emit_operand(dst, src);
5260 }
5261 
5262 void Assembler::vpsubd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5263   assert(UseAVX > 0, "requires some form of AVX");
5264   InstructionMark im(this);
5265   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5266   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
5267   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5268   emit_int8((unsigned char)0xFA);
5269   emit_operand(dst, src);
5270 }
5271 
5272 void Assembler::vpsubq(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5273   assert(UseAVX > 0, "requires some form of AVX");
5274   InstructionMark im(this);
5275   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5276   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
5277   attributes.set_rex_vex_w_reverted();
5278   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5279   emit_int8((unsigned char)0xFB);
5280   emit_operand(dst, src);
5281 }
5282 
5283 void Assembler::pmullw(XMMRegister dst, XMMRegister src) {
5284   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5285   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5286   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5287   emit_int8((unsigned char)0xD5);
5288   emit_int8((unsigned char)(0xC0 | encode));
5289 }
5290 
5291 void Assembler::pmulld(XMMRegister dst, XMMRegister src) {
5292   assert(VM_Version::supports_sse4_1(), "");
5293   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5294   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5295   emit_int8(0x40);
5296   emit_int8((unsigned char)(0xC0 | encode));
5297 }
5298 
5299 void Assembler::vpmullw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5300   assert(UseAVX > 0, "requires some form of AVX");
5301   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5302   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5303   emit_int8((unsigned char)0xD5);
5304   emit_int8((unsigned char)(0xC0 | encode));
5305 }
5306 
5307 void Assembler::vpmulld(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5308   assert(UseAVX > 0, "requires some form of AVX");
5309   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5310   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5311   emit_int8(0x40);
5312   emit_int8((unsigned char)(0xC0 | encode));
5313 }
5314 
5315 void Assembler::vpmullq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5316   assert(UseAVX > 2, "requires some form of EVEX");
5317   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
5318   attributes.set_is_evex_instruction();
5319   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5320   emit_int8(0x40);
5321   emit_int8((unsigned char)(0xC0 | encode));
5322 }
5323 
5324 void Assembler::vpmullw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5325   assert(UseAVX > 0, "requires some form of AVX");
5326   InstructionMark im(this);
5327   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5328   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
5329   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5330   emit_int8((unsigned char)0xD5);
5331   emit_operand(dst, src);
5332 }
5333 
5334 void Assembler::vpmulld(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5335   assert(UseAVX > 0, "requires some form of AVX");
5336   InstructionMark im(this);
5337   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5338   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
5339   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5340   emit_int8(0x40);
5341   emit_operand(dst, src);
5342 }
5343 
5344 void Assembler::vpmullq(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5345   assert(UseAVX > 2, "requires some form of EVEX");
5346   InstructionMark im(this);
5347   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
5348   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
5349   attributes.set_is_evex_instruction();
5350   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5351   emit_int8(0x40);
5352   emit_operand(dst, src);
5353 }
5354 
5355 // Shift packed integers left by specified number of bits.
5356 void Assembler::psllw(XMMRegister dst, int shift) {
5357   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5358   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5359   // XMM6 is for /6 encoding: 66 0F 71 /6 ib
5360   int encode = simd_prefix_and_encode(xmm6, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5361   emit_int8(0x71);
5362   emit_int8((unsigned char)(0xC0 | encode));
5363   emit_int8(shift & 0xFF);
5364 }
5365 
5366 void Assembler::pslld(XMMRegister dst, int shift) {
5367   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5368   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5369   // XMM6 is for /6 encoding: 66 0F 72 /6 ib
5370   int encode = simd_prefix_and_encode(xmm6, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5371   emit_int8(0x72);
5372   emit_int8((unsigned char)(0xC0 | encode));
5373   emit_int8(shift & 0xFF);
5374 }
5375 
5376 void Assembler::psllq(XMMRegister dst, int shift) {
5377   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5378   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5379   // XMM6 is for /6 encoding: 66 0F 73 /6 ib
5380   int encode = simd_prefix_and_encode(xmm6, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5381   emit_int8(0x73);
5382   emit_int8((unsigned char)(0xC0 | encode));
5383   emit_int8(shift & 0xFF);
5384 }
5385 
5386 void Assembler::psllw(XMMRegister dst, XMMRegister shift) {
5387   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5388   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5389   int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5390   emit_int8((unsigned char)0xF1);
5391   emit_int8((unsigned char)(0xC0 | encode));
5392 }
5393 
5394 void Assembler::pslld(XMMRegister dst, XMMRegister shift) {
5395   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5396   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5397   int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5398   emit_int8((unsigned char)0xF2);
5399   emit_int8((unsigned char)(0xC0 | encode));
5400 }
5401 
5402 void Assembler::psllq(XMMRegister dst, XMMRegister shift) {
5403   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5404   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5405   attributes.set_rex_vex_w_reverted();
5406   int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5407   emit_int8((unsigned char)0xF3);
5408   emit_int8((unsigned char)(0xC0 | encode));
5409 }
5410 
5411 void Assembler::vpsllw(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
5412   assert(UseAVX > 0, "requires some form of AVX");
5413   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5414   // XMM6 is for /6 encoding: 66 0F 71 /6 ib
5415   int encode = vex_prefix_and_encode(xmm6->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5416   emit_int8(0x71);
5417   emit_int8((unsigned char)(0xC0 | encode));
5418   emit_int8(shift & 0xFF);
5419 }
5420 
5421 void Assembler::vpslld(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
5422   assert(UseAVX > 0, "requires some form of AVX");
5423   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5424   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5425   // XMM6 is for /6 encoding: 66 0F 72 /6 ib
5426   int encode = vex_prefix_and_encode(xmm6->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5427   emit_int8(0x72);
5428   emit_int8((unsigned char)(0xC0 | encode));
5429   emit_int8(shift & 0xFF);
5430 }
5431 
5432 void Assembler::vpsllq(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
5433   assert(UseAVX > 0, "requires some form of AVX");
5434   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5435   attributes.set_rex_vex_w_reverted();
5436   // XMM6 is for /6 encoding: 66 0F 73 /6 ib
5437   int encode = vex_prefix_and_encode(xmm6->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5438   emit_int8(0x73);
5439   emit_int8((unsigned char)(0xC0 | encode));
5440   emit_int8(shift & 0xFF);
5441 }
5442 
5443 void Assembler::vpsllw(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
5444   assert(UseAVX > 0, "requires some form of AVX");
5445   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5446   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5447   emit_int8((unsigned char)0xF1);
5448   emit_int8((unsigned char)(0xC0 | encode));
5449 }
5450 
5451 void Assembler::vpslld(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
5452   assert(UseAVX > 0, "requires some form of AVX");
5453   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5454   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5455   emit_int8((unsigned char)0xF2);
5456   emit_int8((unsigned char)(0xC0 | encode));
5457 }
5458 
5459 void Assembler::vpsllq(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
5460   assert(UseAVX > 0, "requires some form of AVX");
5461   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5462   attributes.set_rex_vex_w_reverted();
5463   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5464   emit_int8((unsigned char)0xF3);
5465   emit_int8((unsigned char)(0xC0 | encode));
5466 }
5467 
5468 // Shift packed integers logically right by specified number of bits.
5469 void Assembler::psrlw(XMMRegister dst, int shift) {
5470   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5471   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5472   // XMM2 is for /2 encoding: 66 0F 71 /2 ib
5473   int encode = simd_prefix_and_encode(xmm2, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5474   emit_int8(0x71);
5475   emit_int8((unsigned char)(0xC0 | encode));
5476   emit_int8(shift & 0xFF);
5477 }
5478 
5479 void Assembler::psrld(XMMRegister dst, int shift) {
5480   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5481   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5482   // XMM2 is for /2 encoding: 66 0F 72 /2 ib
5483   int encode = simd_prefix_and_encode(xmm2, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5484   emit_int8(0x72);
5485   emit_int8((unsigned char)(0xC0 | encode));
5486   emit_int8(shift & 0xFF);
5487 }
5488 
5489 void Assembler::psrlq(XMMRegister dst, int shift) {
5490   // Do not confuse it with psrldq SSE2 instruction which
5491   // shifts 128 bit value in xmm register by number of bytes.
5492   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5493   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5494   attributes.set_rex_vex_w_reverted();
5495   // XMM2 is for /2 encoding: 66 0F 73 /2 ib
5496   int encode = simd_prefix_and_encode(xmm2, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5497   emit_int8(0x73);
5498   emit_int8((unsigned char)(0xC0 | encode));
5499   emit_int8(shift & 0xFF);
5500 }
5501 
5502 void Assembler::psrlw(XMMRegister dst, XMMRegister shift) {
5503   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5504   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5505   int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5506   emit_int8((unsigned char)0xD1);
5507   emit_int8((unsigned char)(0xC0 | encode));
5508 }
5509 
5510 void Assembler::psrld(XMMRegister dst, XMMRegister shift) {
5511   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5512   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5513   int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5514   emit_int8((unsigned char)0xD2);
5515   emit_int8((unsigned char)(0xC0 | encode));
5516 }
5517 
5518 void Assembler::psrlq(XMMRegister dst, XMMRegister shift) {
5519   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5520   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5521   attributes.set_rex_vex_w_reverted();
5522   int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5523   emit_int8((unsigned char)0xD3);
5524   emit_int8((unsigned char)(0xC0 | encode));
5525 }
5526 
5527 void Assembler::vpsrlw(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
5528   assert(UseAVX > 0, "requires some form of AVX");
5529   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5530   // XMM2 is for /2 encoding: 66 0F 71 /2 ib
5531   int encode = vex_prefix_and_encode(xmm2->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5532   emit_int8(0x71);
5533   emit_int8((unsigned char)(0xC0 | encode));
5534   emit_int8(shift & 0xFF);
5535 }
5536 
5537 void Assembler::vpsrld(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
5538   assert(UseAVX > 0, "requires some form of AVX");
5539   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5540   // XMM2 is for /2 encoding: 66 0F 72 /2 ib
5541   int encode = vex_prefix_and_encode(xmm2->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5542   emit_int8(0x72);
5543   emit_int8((unsigned char)(0xC0 | encode));
5544   emit_int8(shift & 0xFF);
5545 }
5546 
5547 void Assembler::vpsrlq(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
5548   assert(UseAVX > 0, "requires some form of AVX");
5549   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5550   attributes.set_rex_vex_w_reverted();
5551   // XMM2 is for /2 encoding: 66 0F 73 /2 ib
5552   int encode = vex_prefix_and_encode(xmm2->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5553   emit_int8(0x73);
5554   emit_int8((unsigned char)(0xC0 | encode));
5555   emit_int8(shift & 0xFF);
5556 }
5557 
5558 void Assembler::vpsrlw(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
5559   assert(UseAVX > 0, "requires some form of AVX");
5560   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5561   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5562   emit_int8((unsigned char)0xD1);
5563   emit_int8((unsigned char)(0xC0 | encode));
5564 }
5565 
5566 void Assembler::vpsrld(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
5567   assert(UseAVX > 0, "requires some form of AVX");
5568   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5569   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5570   emit_int8((unsigned char)0xD2);
5571   emit_int8((unsigned char)(0xC0 | encode));
5572 }
5573 
5574 void Assembler::vpsrlq(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
5575   assert(UseAVX > 0, "requires some form of AVX");
5576   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5577   attributes.set_rex_vex_w_reverted();
5578   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5579   emit_int8((unsigned char)0xD3);
5580   emit_int8((unsigned char)(0xC0 | encode));
5581 }
5582 
5583 // Shift packed integers arithmetically right by specified number of bits.
5584 void Assembler::psraw(XMMRegister dst, int shift) {
5585   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5586   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5587   // XMM4 is for /4 encoding: 66 0F 71 /4 ib
5588   int encode = simd_prefix_and_encode(xmm4, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5589   emit_int8(0x71);
5590   emit_int8((unsigned char)(0xC0 | encode));
5591   emit_int8(shift & 0xFF);
5592 }
5593 
5594 void Assembler::psrad(XMMRegister dst, int shift) {
5595   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5596   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5597   // XMM4 is for /4 encoding: 66 0F 72 /4 ib
5598   int encode = simd_prefix_and_encode(xmm4, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5599   emit_int8(0x72);
5600   emit_int8((unsigned char)(0xC0 | encode));
5601   emit_int8(shift & 0xFF);
5602 }
5603 
5604 void Assembler::psraw(XMMRegister dst, XMMRegister shift) {
5605   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5606   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5607   int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5608   emit_int8((unsigned char)0xE1);
5609   emit_int8((unsigned char)(0xC0 | encode));
5610 }
5611 
5612 void Assembler::psrad(XMMRegister dst, XMMRegister shift) {
5613   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5614   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5615   int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5616   emit_int8((unsigned char)0xE2);
5617   emit_int8((unsigned char)(0xC0 | encode));
5618 }
5619 
5620 void Assembler::vpsraw(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
5621   assert(UseAVX > 0, "requires some form of AVX");
5622   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5623   // XMM4 is for /4 encoding: 66 0F 71 /4 ib
5624   int encode = vex_prefix_and_encode(xmm4->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5625   emit_int8(0x71);
5626   emit_int8((unsigned char)(0xC0 | encode));
5627   emit_int8(shift & 0xFF);
5628 }
5629 
5630 void Assembler::vpsrad(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
5631   assert(UseAVX > 0, "requires some form of AVX");
5632   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5633   // XMM4 is for /4 encoding: 66 0F 71 /4 ib
5634   int encode = vex_prefix_and_encode(xmm4->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5635   emit_int8(0x72);
5636   emit_int8((unsigned char)(0xC0 | encode));
5637   emit_int8(shift & 0xFF);
5638 }
5639 
5640 void Assembler::vpsraw(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
5641   assert(UseAVX > 0, "requires some form of AVX");
5642   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5643   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5644   emit_int8((unsigned char)0xE1);
5645   emit_int8((unsigned char)(0xC0 | encode));
5646 }
5647 
5648 void Assembler::vpsrad(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
5649   assert(UseAVX > 0, "requires some form of AVX");
5650   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5651   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5652   emit_int8((unsigned char)0xE2);
5653   emit_int8((unsigned char)(0xC0 | encode));
5654 }
5655 
5656 
5657 // logical operations packed integers
5658 void Assembler::pand(XMMRegister dst, XMMRegister src) {
5659   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5660   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5661   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5662   emit_int8((unsigned char)0xDB);
5663   emit_int8((unsigned char)(0xC0 | encode));
5664 }
5665 
5666 void Assembler::vpand(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5667   assert(UseAVX > 0, "requires some form of AVX");
5668   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5669   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5670   emit_int8((unsigned char)0xDB);
5671   emit_int8((unsigned char)(0xC0 | encode));
5672 }
5673 
5674 void Assembler::vpand(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5675   assert(UseAVX > 0, "requires some form of AVX");
5676   InstructionMark im(this);
5677   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5678   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
5679   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5680   emit_int8((unsigned char)0xDB);
5681   emit_operand(dst, src);
5682 }
5683 
5684 void Assembler::pandn(XMMRegister dst, XMMRegister src) {
5685   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5686   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5687   attributes.set_rex_vex_w_reverted();
5688   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5689   emit_int8((unsigned char)0xDF);
5690   emit_int8((unsigned char)(0xC0 | encode));
5691 }
5692 
5693 void Assembler::por(XMMRegister dst, XMMRegister src) {
5694   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5695   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5696   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5697   emit_int8((unsigned char)0xEB);
5698   emit_int8((unsigned char)(0xC0 | encode));
5699 }
5700 
5701 void Assembler::vpor(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5702   assert(UseAVX > 0, "requires some form of AVX");
5703   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5704   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5705   emit_int8((unsigned char)0xEB);
5706   emit_int8((unsigned char)(0xC0 | encode));
5707 }
5708 
5709 void Assembler::vpor(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5710   assert(UseAVX > 0, "requires some form of AVX");
5711   InstructionMark im(this);
5712   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5713   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
5714   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5715   emit_int8((unsigned char)0xEB);
5716   emit_operand(dst, src);
5717 }
5718 
5719 void Assembler::pxor(XMMRegister dst, XMMRegister src) {
5720   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5721   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5722   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5723   emit_int8((unsigned char)0xEF);
5724   emit_int8((unsigned char)(0xC0 | encode));
5725 }
5726 
5727 void Assembler::vpxor(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5728   assert(UseAVX > 0, "requires some form of AVX");
5729   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5730   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5731   emit_int8((unsigned char)0xEF);
5732   emit_int8((unsigned char)(0xC0 | encode));
5733 }
5734 
5735 void Assembler::vpxor(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5736   assert(UseAVX > 0, "requires some form of AVX");
5737   InstructionMark im(this);
5738   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5739   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
5740   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5741   emit_int8((unsigned char)0xEF);
5742   emit_operand(dst, src);
5743 }
5744 
5745 
5746 // vinserti forms
5747 
5748 void Assembler::vinserti128(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) {
5749   assert(VM_Version::supports_avx2(), "");
5750   assert(imm8 <= 0x01, "imm8: %u", imm8);
5751   int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_256bit;
5752   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
5753   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
5754   emit_int8(0x38);
5755   emit_int8((unsigned char)(0xC0 | encode));
5756   // 0x00 - insert into lower 128 bits
5757   // 0x01 - insert into upper 128 bits
5758   emit_int8(imm8 & 0x01);
5759 }
5760 
5761 void Assembler::vinserti128(XMMRegister dst, XMMRegister nds, Address src, uint8_t imm8) {
5762   assert(VM_Version::supports_avx2(), "");
5763   assert(dst != xnoreg, "sanity");
5764   assert(imm8 <= 0x01, "imm8: %u", imm8);
5765   int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_256bit;
5766   InstructionMark im(this);
5767   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
5768   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
5769   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
5770   emit_int8(0x38);
5771   emit_operand(dst, src);
5772   // 0x00 - insert into lower 128 bits
5773   // 0x01 - insert into upper 128 bits
5774   emit_int8(imm8 & 0x01);
5775 }
5776 
5777 void Assembler::vinserti32x4(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) {
5778   assert(VM_Version::supports_evex(), "");
5779   assert(imm8 <= 0x03, "imm8: %u", imm8);
5780   InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
5781   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
5782   emit_int8(0x38);
5783   emit_int8((unsigned char)(0xC0 | encode));
5784   // 0x00 - insert into q0 128 bits (0..127)
5785   // 0x01 - insert into q1 128 bits (128..255)
5786   // 0x02 - insert into q2 128 bits (256..383)
5787   // 0x03 - insert into q3 128 bits (384..511)
5788   emit_int8(imm8 & 0x03);
5789 }
5790 
5791 void Assembler::vinserti32x4(XMMRegister dst, XMMRegister nds, Address src, uint8_t imm8) {
5792   assert(VM_Version::supports_avx(), "");
5793   assert(dst != xnoreg, "sanity");
5794   assert(imm8 <= 0x03, "imm8: %u", imm8);
5795   int vector_len = VM_Version::supports_evex() ? AVX_512bit : AVX_256bit;
5796   InstructionMark im(this);
5797   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
5798   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
5799   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
5800   emit_int8(0x18);
5801   emit_operand(dst, src);
5802   // 0x00 - insert into q0 128 bits (0..127)
5803   // 0x01 - insert into q1 128 bits (128..255)
5804   // 0x02 - insert into q2 128 bits (256..383)
5805   // 0x03 - insert into q3 128 bits (384..511)
5806   emit_int8(imm8 & 0x03);
5807 }
5808 
5809 void Assembler::vinserti64x4(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) {
5810   assert(VM_Version::supports_evex(), "");
5811   assert(imm8 <= 0x01, "imm8: %u", imm8);
5812   InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
5813   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
5814   emit_int8(0x38);
5815   emit_int8((unsigned char)(0xC0 | encode));
5816   // 0x00 - insert into lower 256 bits
5817   // 0x01 - insert into upper 256 bits
5818   emit_int8(imm8 & 0x01);
5819 }
5820 
5821 
5822 // vinsertf forms
5823 
5824 void Assembler::vinsertf128(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) {
5825   assert(VM_Version::supports_avx(), "");
5826   assert(imm8 <= 0x01, "imm8: %u", imm8);
5827   int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_256bit;
5828   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
5829   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
5830   emit_int8(0x18);
5831   emit_int8((unsigned char)(0xC0 | encode));
5832   // 0x00 - insert into lower 128 bits
5833   // 0x01 - insert into upper 128 bits
5834   emit_int8(imm8 & 0x01);
5835 }
5836 
5837 void Assembler::vinsertf128(XMMRegister dst, XMMRegister nds, Address src, uint8_t imm8) {
5838   assert(VM_Version::supports_avx(), "");
5839   assert(dst != xnoreg, "sanity");
5840   assert(imm8 <= 0x01, "imm8: %u", imm8);
5841   int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_256bit;
5842   InstructionMark im(this);
5843   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
5844   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
5845   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
5846   emit_int8(0x18);
5847   emit_operand(dst, src);
5848   // 0x00 - insert into lower 128 bits
5849   // 0x01 - insert into upper 128 bits
5850   emit_int8(imm8 & 0x01);
5851 }
5852 
5853 void Assembler::vinsertf32x4(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) {
5854   assert(VM_Version::supports_evex(), "");
5855   assert(imm8 <= 0x03, "imm8: %u", imm8);
5856   InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
5857   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
5858   emit_int8(0x18);
5859   emit_int8((unsigned char)(0xC0 | encode));
5860   // 0x00 - insert into q0 128 bits (0..127)
5861   // 0x01 - insert into q1 128 bits (128..255)
5862   // 0x02 - insert into q2 128 bits (256..383)
5863   // 0x03 - insert into q3 128 bits (384..511)
5864   emit_int8(imm8 & 0x03);
5865 }
5866 
5867 void Assembler::vinsertf32x4(XMMRegister dst, XMMRegister nds, Address src, uint8_t imm8) {
5868   assert(VM_Version::supports_avx(), "");
5869   assert(dst != xnoreg, "sanity");
5870   assert(imm8 <= 0x03, "imm8: %u", imm8);
5871   int vector_len = VM_Version::supports_evex() ? AVX_512bit : AVX_256bit;
5872   InstructionMark im(this);
5873   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
5874   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
5875   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
5876   emit_int8(0x18);
5877   emit_operand(dst, src);
5878   // 0x00 - insert into q0 128 bits (0..127)
5879   // 0x01 - insert into q1 128 bits (128..255)
5880   // 0x02 - insert into q2 128 bits (256..383)
5881   // 0x03 - insert into q3 128 bits (384..511)
5882   emit_int8(imm8 & 0x03);
5883 }
5884 
5885 void Assembler::vinsertf64x4(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) {
5886   assert(VM_Version::supports_evex(), "");
5887   assert(imm8 <= 0x01, "imm8: %u", imm8);
5888   InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
5889   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
5890   emit_int8(0x1A);
5891   emit_int8((unsigned char)(0xC0 | encode));
5892   // 0x00 - insert into lower 256 bits
5893   // 0x01 - insert into upper 256 bits
5894   emit_int8(imm8 & 0x01);
5895 }
5896 
5897 void Assembler::vinsertf64x4(XMMRegister dst, XMMRegister nds, Address src, uint8_t imm8) {
5898   assert(VM_Version::supports_evex(), "");
5899   assert(dst != xnoreg, "sanity");
5900   assert(imm8 <= 0x01, "imm8: %u", imm8);
5901   InstructionMark im(this);
5902   InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
5903   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_64bit);
5904   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
5905   emit_int8(0x1A);
5906   emit_operand(dst, src);
5907   // 0x00 - insert into lower 256 bits
5908   // 0x01 - insert into upper 256 bits
5909   emit_int8(imm8 & 0x01);
5910 }
5911 
5912 
5913 // vextracti forms
5914 
5915 void Assembler::vextracti128(XMMRegister dst, XMMRegister src, uint8_t imm8) {
5916   assert(VM_Version::supports_avx(), "");
5917   assert(imm8 <= 0x01, "imm8: %u", imm8);
5918   int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_256bit;
5919   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
5920   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
5921   emit_int8(0x39);
5922   emit_int8((unsigned char)(0xC0 | encode));
5923   // 0x00 - extract from lower 128 bits
5924   // 0x01 - extract from upper 128 bits
5925   emit_int8(imm8 & 0x01);
5926 }
5927 
5928 void Assembler::vextracti128(Address dst, XMMRegister src, uint8_t imm8) {
5929   assert(VM_Version::supports_avx2(), "");
5930   assert(src != xnoreg, "sanity");
5931   assert(imm8 <= 0x01, "imm8: %u", imm8);
5932   int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_256bit;
5933   InstructionMark im(this);
5934   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
5935   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
5936   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
5937   emit_int8(0x39);
5938   emit_operand(src, dst);
5939   // 0x00 - extract from lower 128 bits
5940   // 0x01 - extract from upper 128 bits
5941   emit_int8(imm8 & 0x01);
5942 }
5943 
5944 void Assembler::vextracti32x4(XMMRegister dst, XMMRegister src, uint8_t imm8) {
5945   assert(VM_Version::supports_avx(), "");
5946   assert(imm8 <= 0x03, "imm8: %u", imm8);
5947   int vector_len = VM_Version::supports_evex() ? AVX_512bit : AVX_256bit;
5948   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
5949   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
5950   emit_int8(0x39);
5951   emit_int8((unsigned char)(0xC0 | encode));
5952   // 0x00 - extract from bits 127:0
5953   // 0x01 - extract from bits 255:128
5954   // 0x02 - extract from bits 383:256
5955   // 0x03 - extract from bits 511:384
5956   emit_int8(imm8 & 0x03);
5957 }
5958 
5959 void Assembler::vextracti32x4(Address dst, XMMRegister src, uint8_t imm8) {
5960   assert(VM_Version::supports_evex(), "");
5961   assert(src != xnoreg, "sanity");
5962   assert(imm8 <= 0x03, "imm8: %u", imm8);
5963   InstructionMark im(this);
5964   InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
5965   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
5966   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
5967   emit_int8(0x39);
5968   emit_operand(src, dst);
5969   // 0x00 - extract from bits 127:0
5970   // 0x01 - extract from bits 255:128
5971   // 0x02 - extract from bits 383:256
5972   // 0x03 - extract from bits 511:384
5973   emit_int8(imm8 & 0x03);
5974 }
5975 
5976 void Assembler::vextracti64x2(XMMRegister dst, XMMRegister src, uint8_t imm8) {
5977   assert(VM_Version::supports_avx512dq(), "");
5978   assert(imm8 <= 0x03, "imm8: %u", imm8);
5979   InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
5980   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
5981   emit_int8(0x39);
5982   emit_int8((unsigned char)(0xC0 | encode));
5983   // 0x00 - extract from bits 127:0
5984   // 0x01 - extract from bits 255:128
5985   // 0x02 - extract from bits 383:256
5986   // 0x03 - extract from bits 511:384
5987   emit_int8(imm8 & 0x03);
5988 }
5989 
5990 void Assembler::vextracti64x4(XMMRegister dst, XMMRegister src, uint8_t imm8) {
5991   assert(VM_Version::supports_evex(), "");
5992   assert(imm8 <= 0x01, "imm8: %u", imm8);
5993   InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
5994   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
5995   emit_int8(0x3B);
5996   emit_int8((unsigned char)(0xC0 | encode));
5997   // 0x00 - extract from lower 256 bits
5998   // 0x01 - extract from upper 256 bits
5999   emit_int8(imm8 & 0x01);
6000 }
6001 
6002 
6003 // vextractf forms
6004 
6005 void Assembler::vextractf128(XMMRegister dst, XMMRegister src, uint8_t imm8) {
6006   assert(VM_Version::supports_avx(), "");
6007   assert(imm8 <= 0x01, "imm8: %u", imm8);
6008   int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_256bit;
6009   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6010   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6011   emit_int8(0x19);
6012   emit_int8((unsigned char)(0xC0 | encode));
6013   // 0x00 - extract from lower 128 bits
6014   // 0x01 - extract from upper 128 bits
6015   emit_int8(imm8 & 0x01);
6016 }
6017 
6018 void Assembler::vextractf128(Address dst, XMMRegister src, uint8_t imm8) {
6019   assert(VM_Version::supports_avx(), "");
6020   assert(src != xnoreg, "sanity");
6021   assert(imm8 <= 0x01, "imm8: %u", imm8);
6022   int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_256bit;
6023   InstructionMark im(this);
6024   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6025   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
6026   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6027   emit_int8(0x19);
6028   emit_operand(src, dst);
6029   // 0x00 - extract from lower 128 bits
6030   // 0x01 - extract from upper 128 bits
6031   emit_int8(imm8 & 0x01);
6032 }
6033 
6034 void Assembler::vextractf32x4(XMMRegister dst, XMMRegister src, uint8_t imm8) {
6035   assert(VM_Version::supports_avx(), "");
6036   assert(imm8 <= 0x03, "imm8: %u", imm8);
6037   int vector_len = VM_Version::supports_evex() ? AVX_512bit : AVX_256bit;
6038   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6039   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6040   emit_int8(0x19);
6041   emit_int8((unsigned char)(0xC0 | encode));
6042   // 0x00 - extract from bits 127:0
6043   // 0x01 - extract from bits 255:128
6044   // 0x02 - extract from bits 383:256
6045   // 0x03 - extract from bits 511:384
6046   emit_int8(imm8 & 0x03);
6047 }
6048 
6049 void Assembler::vextractf32x4(Address dst, XMMRegister src, uint8_t imm8) {
6050   assert(VM_Version::supports_evex(), "");
6051   assert(src != xnoreg, "sanity");
6052   assert(imm8 <= 0x03, "imm8: %u", imm8);
6053   InstructionMark im(this);
6054   InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6055   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
6056   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6057   emit_int8(0x19);
6058   emit_operand(src, dst);
6059   // 0x00 - extract from bits 127:0
6060   // 0x01 - extract from bits 255:128
6061   // 0x02 - extract from bits 383:256
6062   // 0x03 - extract from bits 511:384
6063   emit_int8(imm8 & 0x03);
6064 }
6065 
6066 void Assembler::vextractf64x2(XMMRegister dst, XMMRegister src, uint8_t imm8) {
6067   assert(VM_Version::supports_avx512dq(), "");
6068   assert(imm8 <= 0x03, "imm8: %u", imm8);
6069   InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6070   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6071   emit_int8(0x19);
6072   emit_int8((unsigned char)(0xC0 | encode));
6073   // 0x00 - extract from bits 127:0
6074   // 0x01 - extract from bits 255:128
6075   // 0x02 - extract from bits 383:256
6076   // 0x03 - extract from bits 511:384
6077   emit_int8(imm8 & 0x03);
6078 }
6079 
6080 void Assembler::vextractf64x4(XMMRegister dst, XMMRegister src, uint8_t imm8) {
6081   assert(VM_Version::supports_evex(), "");
6082   assert(imm8 <= 0x01, "imm8: %u", imm8);
6083   InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6084   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6085   emit_int8(0x1B);
6086   emit_int8((unsigned char)(0xC0 | encode));
6087   // 0x00 - extract from lower 256 bits
6088   // 0x01 - extract from upper 256 bits
6089   emit_int8(imm8 & 0x01);
6090 }
6091 
6092 void Assembler::vextractf64x4(Address dst, XMMRegister src, uint8_t imm8) {
6093   assert(VM_Version::supports_evex(), "");
6094   assert(src != xnoreg, "sanity");
6095   assert(imm8 <= 0x01, "imm8: %u", imm8);
6096   InstructionMark im(this);
6097   InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6098   attributes.set_address_attributes(/* tuple_type */ EVEX_T4,/* input_size_in_bits */  EVEX_64bit);
6099   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6100   emit_int8(0x1B);
6101   emit_operand(src, dst);
6102   // 0x00 - extract from lower 256 bits
6103   // 0x01 - extract from upper 256 bits
6104   emit_int8(imm8 & 0x01);
6105 }
6106 
6107 
6108 // legacy word/dword replicate
6109 void Assembler::vpbroadcastw(XMMRegister dst, XMMRegister src) {
6110   assert(VM_Version::supports_avx2(), "");
6111   InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
6112   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6113   emit_int8(0x79);
6114   emit_int8((unsigned char)(0xC0 | encode));
6115 }
6116 
6117 void Assembler::vpbroadcastd(XMMRegister dst, XMMRegister src) {
6118   assert(VM_Version::supports_avx2(), "");
6119   InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6120   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6121   emit_int8(0x58);
6122   emit_int8((unsigned char)(0xC0 | encode));
6123 }
6124 
6125 
6126 // xmm/mem sourced byte/word/dword/qword replicate
6127 
6128 // duplicate 1-byte integer data from src into programmed locations in dest : requires AVX512BW and AVX512VL
6129 void Assembler::evpbroadcastb(XMMRegister dst, XMMRegister src, int vector_len) {
6130   assert(VM_Version::supports_evex(), "");
6131   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
6132   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6133   emit_int8(0x78);
6134   emit_int8((unsigned char)(0xC0 | encode));
6135 }
6136 
6137 void Assembler::evpbroadcastb(XMMRegister dst, Address src, int vector_len) {
6138   assert(VM_Version::supports_evex(), "");
6139   assert(dst != xnoreg, "sanity");
6140   InstructionMark im(this);
6141   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
6142   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_8bit);
6143   // swap src<->dst for encoding
6144   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6145   emit_int8(0x78);
6146   emit_operand(dst, src);
6147 }
6148 
6149 // duplicate 2-byte integer data from src into programmed locations in dest : requires AVX512BW and AVX512VL
6150 void Assembler::evpbroadcastw(XMMRegister dst, XMMRegister src, int vector_len) {
6151   assert(VM_Version::supports_evex(), "");
6152   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
6153   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6154   emit_int8(0x79);
6155   emit_int8((unsigned char)(0xC0 | encode));
6156 }
6157 
6158 void Assembler::evpbroadcastw(XMMRegister dst, Address src, int vector_len) {
6159   assert(VM_Version::supports_evex(), "");
6160   assert(dst != xnoreg, "sanity");
6161   InstructionMark im(this);
6162   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
6163   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_16bit);
6164   // swap src<->dst for encoding
6165   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6166   emit_int8(0x79);
6167   emit_operand(dst, src);
6168 }
6169 
6170 // duplicate 4-byte integer data from src into programmed locations in dest : requires AVX512VL
6171 void Assembler::evpbroadcastd(XMMRegister dst, XMMRegister src, int vector_len) {
6172   assert(VM_Version::supports_evex(), "");
6173   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6174   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6175   emit_int8(0x58);
6176   emit_int8((unsigned char)(0xC0 | encode));
6177 }
6178 
6179 void Assembler::evpbroadcastd(XMMRegister dst, Address src, int vector_len) {
6180   assert(VM_Version::supports_evex(), "");
6181   assert(dst != xnoreg, "sanity");
6182   InstructionMark im(this);
6183   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6184   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
6185   // swap src<->dst for encoding
6186   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6187   emit_int8(0x58);
6188   emit_operand(dst, src);
6189 }
6190 
6191 // duplicate 8-byte integer data from src into programmed locations in dest : requires AVX512VL
6192 void Assembler::evpbroadcastq(XMMRegister dst, XMMRegister src, int vector_len) {
6193   assert(VM_Version::supports_evex(), "");
6194   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6195   attributes.set_rex_vex_w_reverted();
6196   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6197   emit_int8(0x59);
6198   emit_int8((unsigned char)(0xC0 | encode));
6199 }
6200 
6201 void Assembler::evpbroadcastq(XMMRegister dst, Address src, int vector_len) {
6202   assert(VM_Version::supports_evex(), "");
6203   assert(dst != xnoreg, "sanity");
6204   InstructionMark im(this);
6205   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6206   attributes.set_rex_vex_w_reverted();
6207   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
6208   // swap src<->dst for encoding
6209   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6210   emit_int8(0x59);
6211   emit_operand(dst, src);
6212 }
6213 
6214 
6215 // scalar single/double precision replicate
6216 
6217 // duplicate single precision data from src into programmed locations in dest : requires AVX512VL
6218 void Assembler::evpbroadcastss(XMMRegister dst, XMMRegister src, int vector_len) {
6219   assert(VM_Version::supports_evex(), "");
6220   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6221   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6222   emit_int8(0x18);
6223   emit_int8((unsigned char)(0xC0 | encode));
6224 }
6225 
6226 void Assembler::evpbroadcastss(XMMRegister dst, Address src, int vector_len) {
6227   assert(VM_Version::supports_evex(), "");
6228   assert(dst != xnoreg, "sanity");
6229   InstructionMark im(this);
6230   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6231   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
6232   // swap src<->dst for encoding
6233   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6234   emit_int8(0x18);
6235   emit_operand(dst, src);
6236 }
6237 
6238 // duplicate double precision data from src into programmed locations in dest : requires AVX512VL
6239 void Assembler::evpbroadcastsd(XMMRegister dst, XMMRegister src, int vector_len) {
6240   assert(VM_Version::supports_evex(), "");
6241   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6242   attributes.set_rex_vex_w_reverted();
6243   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6244   emit_int8(0x19);
6245   emit_int8((unsigned char)(0xC0 | encode));
6246 }
6247 
6248 void Assembler::evpbroadcastsd(XMMRegister dst, Address src, int vector_len) {
6249   assert(VM_Version::supports_evex(), "");
6250   assert(dst != xnoreg, "sanity");
6251   InstructionMark im(this);
6252   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6253   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
6254   attributes.set_rex_vex_w_reverted();
6255   // swap src<->dst for encoding
6256   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6257   emit_int8(0x19);
6258   emit_operand(dst, src);
6259 }
6260 
6261 
6262 // gpr source broadcast forms
6263 
6264 // duplicate 1-byte integer data from src into programmed locations in dest : requires AVX512BW and AVX512VL
6265 void Assembler::evpbroadcastb(XMMRegister dst, Register src, int vector_len) {
6266   assert(VM_Version::supports_evex(), "");
6267   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
6268   attributes.set_is_evex_instruction();
6269   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6270   emit_int8(0x7A);
6271   emit_int8((unsigned char)(0xC0 | encode));
6272 }
6273 
6274 // duplicate 2-byte integer data from src into programmed locations in dest : requires AVX512BW and AVX512VL
6275 void Assembler::evpbroadcastw(XMMRegister dst, Register src, int vector_len) {
6276   assert(VM_Version::supports_evex(), "");
6277   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
6278   attributes.set_is_evex_instruction();
6279   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6280   emit_int8(0x7B);
6281   emit_int8((unsigned char)(0xC0 | encode));
6282 }
6283 
6284 // duplicate 4-byte integer data from src into programmed locations in dest : requires AVX512VL
6285 void Assembler::evpbroadcastd(XMMRegister dst, Register src, int vector_len) {
6286   assert(VM_Version::supports_evex(), "");
6287   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6288   attributes.set_is_evex_instruction();
6289   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6290   emit_int8(0x7C);
6291   emit_int8((unsigned char)(0xC0 | encode));
6292 }
6293 
6294 // duplicate 8-byte integer data from src into programmed locations in dest : requires AVX512VL
6295 void Assembler::evpbroadcastq(XMMRegister dst, Register src, int vector_len) {
6296   assert(VM_Version::supports_evex(), "");
6297   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6298   attributes.set_is_evex_instruction();
6299   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6300   emit_int8(0x7C);
6301   emit_int8((unsigned char)(0xC0 | encode));
6302 }
6303 
6304 
6305 // Carry-Less Multiplication Quadword
6306 void Assembler::pclmulqdq(XMMRegister dst, XMMRegister src, int mask) {
6307   assert(VM_Version::supports_clmul(), "");
6308   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
6309   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6310   emit_int8(0x44);
6311   emit_int8((unsigned char)(0xC0 | encode));
6312   emit_int8((unsigned char)mask);
6313 }
6314 
6315 // Carry-Less Multiplication Quadword
6316 void Assembler::vpclmulqdq(XMMRegister dst, XMMRegister nds, XMMRegister src, int mask) {
6317   assert(VM_Version::supports_avx() && VM_Version::supports_clmul(), "");
6318   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
6319   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6320   emit_int8(0x44);
6321   emit_int8((unsigned char)(0xC0 | encode));
6322   emit_int8((unsigned char)mask);
6323 }
6324 
6325 void Assembler::vzeroupper() {
6326   assert(VM_Version::supports_avx(), "");
6327   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
6328   (void)vex_prefix_and_encode(0, 0, 0, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
6329   emit_int8(0x77);
6330 }
6331 
6332 
6333 #ifndef _LP64
6334 // 32bit only pieces of the assembler
6335 
6336 void Assembler::cmp_literal32(Register src1, int32_t imm32, RelocationHolder const& rspec) {
6337   // NO PREFIX AS NEVER 64BIT
6338   InstructionMark im(this);
6339   emit_int8((unsigned char)0x81);
6340   emit_int8((unsigned char)(0xF8 | src1->encoding()));
6341   emit_data(imm32, rspec, 0);
6342 }
6343 
6344 void Assembler::cmp_literal32(Address src1, int32_t imm32, RelocationHolder const& rspec) {
6345   // NO PREFIX AS NEVER 64BIT (not even 32bit versions of 64bit regs
6346   InstructionMark im(this);
6347   emit_int8((unsigned char)0x81);
6348   emit_operand(rdi, src1);
6349   emit_data(imm32, rspec, 0);
6350 }
6351 
6352 // The 64-bit (32bit platform) cmpxchg compares the value at adr with the contents of rdx:rax,
6353 // and stores rcx:rbx into adr if so; otherwise, the value at adr is loaded
6354 // into rdx:rax.  The ZF is set if the compared values were equal, and cleared otherwise.
6355 void Assembler::cmpxchg8(Address adr) {
6356   InstructionMark im(this);
6357   emit_int8(0x0F);
6358   emit_int8((unsigned char)0xC7);
6359   emit_operand(rcx, adr);
6360 }
6361 
6362 void Assembler::decl(Register dst) {
6363   // Don't use it directly. Use MacroAssembler::decrementl() instead.
6364  emit_int8(0x48 | dst->encoding());
6365 }
6366 
6367 #endif // _LP64
6368 
6369 // 64bit typically doesn't use the x87 but needs to for the trig funcs
6370 
6371 void Assembler::fabs() {
6372   emit_int8((unsigned char)0xD9);
6373   emit_int8((unsigned char)0xE1);
6374 }
6375 
6376 void Assembler::fadd(int i) {
6377   emit_farith(0xD8, 0xC0, i);
6378 }
6379 
6380 void Assembler::fadd_d(Address src) {
6381   InstructionMark im(this);
6382   emit_int8((unsigned char)0xDC);
6383   emit_operand32(rax, src);
6384 }
6385 
6386 void Assembler::fadd_s(Address src) {
6387   InstructionMark im(this);
6388   emit_int8((unsigned char)0xD8);
6389   emit_operand32(rax, src);
6390 }
6391 
6392 void Assembler::fadda(int i) {
6393   emit_farith(0xDC, 0xC0, i);
6394 }
6395 
6396 void Assembler::faddp(int i) {
6397   emit_farith(0xDE, 0xC0, i);
6398 }
6399 
6400 void Assembler::fchs() {
6401   emit_int8((unsigned char)0xD9);
6402   emit_int8((unsigned char)0xE0);
6403 }
6404 
6405 void Assembler::fcom(int i) {
6406   emit_farith(0xD8, 0xD0, i);
6407 }
6408 
6409 void Assembler::fcomp(int i) {
6410   emit_farith(0xD8, 0xD8, i);
6411 }
6412 
6413 void Assembler::fcomp_d(Address src) {
6414   InstructionMark im(this);
6415   emit_int8((unsigned char)0xDC);
6416   emit_operand32(rbx, src);
6417 }
6418 
6419 void Assembler::fcomp_s(Address src) {
6420   InstructionMark im(this);
6421   emit_int8((unsigned char)0xD8);
6422   emit_operand32(rbx, src);
6423 }
6424 
6425 void Assembler::fcompp() {
6426   emit_int8((unsigned char)0xDE);
6427   emit_int8((unsigned char)0xD9);
6428 }
6429 
6430 void Assembler::fcos() {
6431   emit_int8((unsigned char)0xD9);
6432   emit_int8((unsigned char)0xFF);
6433 }
6434 
6435 void Assembler::fdecstp() {
6436   emit_int8((unsigned char)0xD9);
6437   emit_int8((unsigned char)0xF6);
6438 }
6439 
6440 void Assembler::fdiv(int i) {
6441   emit_farith(0xD8, 0xF0, i);
6442 }
6443 
6444 void Assembler::fdiv_d(Address src) {
6445   InstructionMark im(this);
6446   emit_int8((unsigned char)0xDC);
6447   emit_operand32(rsi, src);
6448 }
6449 
6450 void Assembler::fdiv_s(Address src) {
6451   InstructionMark im(this);
6452   emit_int8((unsigned char)0xD8);
6453   emit_operand32(rsi, src);
6454 }
6455 
6456 void Assembler::fdiva(int i) {
6457   emit_farith(0xDC, 0xF8, i);
6458 }
6459 
6460 // Note: The Intel manual (Pentium Processor User's Manual, Vol.3, 1994)
6461 //       is erroneous for some of the floating-point instructions below.
6462 
6463 void Assembler::fdivp(int i) {
6464   emit_farith(0xDE, 0xF8, i);                    // ST(0) <- ST(0) / ST(1) and pop (Intel manual wrong)
6465 }
6466 
6467 void Assembler::fdivr(int i) {
6468   emit_farith(0xD8, 0xF8, i);
6469 }
6470 
6471 void Assembler::fdivr_d(Address src) {
6472   InstructionMark im(this);
6473   emit_int8((unsigned char)0xDC);
6474   emit_operand32(rdi, src);
6475 }
6476 
6477 void Assembler::fdivr_s(Address src) {
6478   InstructionMark im(this);
6479   emit_int8((unsigned char)0xD8);
6480   emit_operand32(rdi, src);
6481 }
6482 
6483 void Assembler::fdivra(int i) {
6484   emit_farith(0xDC, 0xF0, i);
6485 }
6486 
6487 void Assembler::fdivrp(int i) {
6488   emit_farith(0xDE, 0xF0, i);                    // ST(0) <- ST(1) / ST(0) and pop (Intel manual wrong)
6489 }
6490 
6491 void Assembler::ffree(int i) {
6492   emit_farith(0xDD, 0xC0, i);
6493 }
6494 
6495 void Assembler::fild_d(Address adr) {
6496   InstructionMark im(this);
6497   emit_int8((unsigned char)0xDF);
6498   emit_operand32(rbp, adr);
6499 }
6500 
6501 void Assembler::fild_s(Address adr) {
6502   InstructionMark im(this);
6503   emit_int8((unsigned char)0xDB);
6504   emit_operand32(rax, adr);
6505 }
6506 
6507 void Assembler::fincstp() {
6508   emit_int8((unsigned char)0xD9);
6509   emit_int8((unsigned char)0xF7);
6510 }
6511 
6512 void Assembler::finit() {
6513   emit_int8((unsigned char)0x9B);
6514   emit_int8((unsigned char)0xDB);
6515   emit_int8((unsigned char)0xE3);
6516 }
6517 
6518 void Assembler::fist_s(Address adr) {
6519   InstructionMark im(this);
6520   emit_int8((unsigned char)0xDB);
6521   emit_operand32(rdx, adr);
6522 }
6523 
6524 void Assembler::fistp_d(Address adr) {
6525   InstructionMark im(this);
6526   emit_int8((unsigned char)0xDF);
6527   emit_operand32(rdi, adr);
6528 }
6529 
6530 void Assembler::fistp_s(Address adr) {
6531   InstructionMark im(this);
6532   emit_int8((unsigned char)0xDB);
6533   emit_operand32(rbx, adr);
6534 }
6535 
6536 void Assembler::fld1() {
6537   emit_int8((unsigned char)0xD9);
6538   emit_int8((unsigned char)0xE8);
6539 }
6540 
6541 void Assembler::fld_d(Address adr) {
6542   InstructionMark im(this);
6543   emit_int8((unsigned char)0xDD);
6544   emit_operand32(rax, adr);
6545 }
6546 
6547 void Assembler::fld_s(Address adr) {
6548   InstructionMark im(this);
6549   emit_int8((unsigned char)0xD9);
6550   emit_operand32(rax, adr);
6551 }
6552 
6553 
6554 void Assembler::fld_s(int index) {
6555   emit_farith(0xD9, 0xC0, index);
6556 }
6557 
6558 void Assembler::fld_x(Address adr) {
6559   InstructionMark im(this);
6560   emit_int8((unsigned char)0xDB);
6561   emit_operand32(rbp, adr);
6562 }
6563 
6564 void Assembler::fldcw(Address src) {
6565   InstructionMark im(this);
6566   emit_int8((unsigned char)0xD9);
6567   emit_operand32(rbp, src);
6568 }
6569 
6570 void Assembler::fldenv(Address src) {
6571   InstructionMark im(this);
6572   emit_int8((unsigned char)0xD9);
6573   emit_operand32(rsp, src);
6574 }
6575 
6576 void Assembler::fldlg2() {
6577   emit_int8((unsigned char)0xD9);
6578   emit_int8((unsigned char)0xEC);
6579 }
6580 
6581 void Assembler::fldln2() {
6582   emit_int8((unsigned char)0xD9);
6583   emit_int8((unsigned char)0xED);
6584 }
6585 
6586 void Assembler::fldz() {
6587   emit_int8((unsigned char)0xD9);
6588   emit_int8((unsigned char)0xEE);
6589 }
6590 
6591 void Assembler::flog() {
6592   fldln2();
6593   fxch();
6594   fyl2x();
6595 }
6596 
6597 void Assembler::flog10() {
6598   fldlg2();
6599   fxch();
6600   fyl2x();
6601 }
6602 
6603 void Assembler::fmul(int i) {
6604   emit_farith(0xD8, 0xC8, i);
6605 }
6606 
6607 void Assembler::fmul_d(Address src) {
6608   InstructionMark im(this);
6609   emit_int8((unsigned char)0xDC);
6610   emit_operand32(rcx, src);
6611 }
6612 
6613 void Assembler::fmul_s(Address src) {
6614   InstructionMark im(this);
6615   emit_int8((unsigned char)0xD8);
6616   emit_operand32(rcx, src);
6617 }
6618 
6619 void Assembler::fmula(int i) {
6620   emit_farith(0xDC, 0xC8, i);
6621 }
6622 
6623 void Assembler::fmulp(int i) {
6624   emit_farith(0xDE, 0xC8, i);
6625 }
6626 
6627 void Assembler::fnsave(Address dst) {
6628   InstructionMark im(this);
6629   emit_int8((unsigned char)0xDD);
6630   emit_operand32(rsi, dst);
6631 }
6632 
6633 void Assembler::fnstcw(Address src) {
6634   InstructionMark im(this);
6635   emit_int8((unsigned char)0x9B);
6636   emit_int8((unsigned char)0xD9);
6637   emit_operand32(rdi, src);
6638 }
6639 
6640 void Assembler::fnstsw_ax() {
6641   emit_int8((unsigned char)0xDF);
6642   emit_int8((unsigned char)0xE0);
6643 }
6644 
6645 void Assembler::fprem() {
6646   emit_int8((unsigned char)0xD9);
6647   emit_int8((unsigned char)0xF8);
6648 }
6649 
6650 void Assembler::fprem1() {
6651   emit_int8((unsigned char)0xD9);
6652   emit_int8((unsigned char)0xF5);
6653 }
6654 
6655 void Assembler::frstor(Address src) {
6656   InstructionMark im(this);
6657   emit_int8((unsigned char)0xDD);
6658   emit_operand32(rsp, src);
6659 }
6660 
6661 void Assembler::fsin() {
6662   emit_int8((unsigned char)0xD9);
6663   emit_int8((unsigned char)0xFE);
6664 }
6665 
6666 void Assembler::fsqrt() {
6667   emit_int8((unsigned char)0xD9);
6668   emit_int8((unsigned char)0xFA);
6669 }
6670 
6671 void Assembler::fst_d(Address adr) {
6672   InstructionMark im(this);
6673   emit_int8((unsigned char)0xDD);
6674   emit_operand32(rdx, adr);
6675 }
6676 
6677 void Assembler::fst_s(Address adr) {
6678   InstructionMark im(this);
6679   emit_int8((unsigned char)0xD9);
6680   emit_operand32(rdx, adr);
6681 }
6682 
6683 void Assembler::fstp_d(Address adr) {
6684   InstructionMark im(this);
6685   emit_int8((unsigned char)0xDD);
6686   emit_operand32(rbx, adr);
6687 }
6688 
6689 void Assembler::fstp_d(int index) {
6690   emit_farith(0xDD, 0xD8, index);
6691 }
6692 
6693 void Assembler::fstp_s(Address adr) {
6694   InstructionMark im(this);
6695   emit_int8((unsigned char)0xD9);
6696   emit_operand32(rbx, adr);
6697 }
6698 
6699 void Assembler::fstp_x(Address adr) {
6700   InstructionMark im(this);
6701   emit_int8((unsigned char)0xDB);
6702   emit_operand32(rdi, adr);
6703 }
6704 
6705 void Assembler::fsub(int i) {
6706   emit_farith(0xD8, 0xE0, i);
6707 }
6708 
6709 void Assembler::fsub_d(Address src) {
6710   InstructionMark im(this);
6711   emit_int8((unsigned char)0xDC);
6712   emit_operand32(rsp, src);
6713 }
6714 
6715 void Assembler::fsub_s(Address src) {
6716   InstructionMark im(this);
6717   emit_int8((unsigned char)0xD8);
6718   emit_operand32(rsp, src);
6719 }
6720 
6721 void Assembler::fsuba(int i) {
6722   emit_farith(0xDC, 0xE8, i);
6723 }
6724 
6725 void Assembler::fsubp(int i) {
6726   emit_farith(0xDE, 0xE8, i);                    // ST(0) <- ST(0) - ST(1) and pop (Intel manual wrong)
6727 }
6728 
6729 void Assembler::fsubr(int i) {
6730   emit_farith(0xD8, 0xE8, i);
6731 }
6732 
6733 void Assembler::fsubr_d(Address src) {
6734   InstructionMark im(this);
6735   emit_int8((unsigned char)0xDC);
6736   emit_operand32(rbp, src);
6737 }
6738 
6739 void Assembler::fsubr_s(Address src) {
6740   InstructionMark im(this);
6741   emit_int8((unsigned char)0xD8);
6742   emit_operand32(rbp, src);
6743 }
6744 
6745 void Assembler::fsubra(int i) {
6746   emit_farith(0xDC, 0xE0, i);
6747 }
6748 
6749 void Assembler::fsubrp(int i) {
6750   emit_farith(0xDE, 0xE0, i);                    // ST(0) <- ST(1) - ST(0) and pop (Intel manual wrong)
6751 }
6752 
6753 void Assembler::ftan() {
6754   emit_int8((unsigned char)0xD9);
6755   emit_int8((unsigned char)0xF2);
6756   emit_int8((unsigned char)0xDD);
6757   emit_int8((unsigned char)0xD8);
6758 }
6759 
6760 void Assembler::ftst() {
6761   emit_int8((unsigned char)0xD9);
6762   emit_int8((unsigned char)0xE4);
6763 }
6764 
6765 void Assembler::fucomi(int i) {
6766   // make sure the instruction is supported (introduced for P6, together with cmov)
6767   guarantee(VM_Version::supports_cmov(), "illegal instruction");
6768   emit_farith(0xDB, 0xE8, i);
6769 }
6770 
6771 void Assembler::fucomip(int i) {
6772   // make sure the instruction is supported (introduced for P6, together with cmov)
6773   guarantee(VM_Version::supports_cmov(), "illegal instruction");
6774   emit_farith(0xDF, 0xE8, i);
6775 }
6776 
6777 void Assembler::fwait() {
6778   emit_int8((unsigned char)0x9B);
6779 }
6780 
6781 void Assembler::fxch(int i) {
6782   emit_farith(0xD9, 0xC8, i);
6783 }
6784 
6785 void Assembler::fyl2x() {
6786   emit_int8((unsigned char)0xD9);
6787   emit_int8((unsigned char)0xF1);
6788 }
6789 
6790 void Assembler::frndint() {
6791   emit_int8((unsigned char)0xD9);
6792   emit_int8((unsigned char)0xFC);
6793 }
6794 
6795 void Assembler::f2xm1() {
6796   emit_int8((unsigned char)0xD9);
6797   emit_int8((unsigned char)0xF0);
6798 }
6799 
6800 void Assembler::fldl2e() {
6801   emit_int8((unsigned char)0xD9);
6802   emit_int8((unsigned char)0xEA);
6803 }
6804 
6805 // SSE SIMD prefix byte values corresponding to VexSimdPrefix encoding.
6806 static int simd_pre[4] = { 0, 0x66, 0xF3, 0xF2 };
6807 // SSE opcode second byte values (first is 0x0F) corresponding to VexOpcode encoding.
6808 static int simd_opc[4] = { 0,    0, 0x38, 0x3A };
6809 
6810 // Generate SSE legacy REX prefix and SIMD opcode based on VEX encoding.
6811 void Assembler::rex_prefix(Address adr, XMMRegister xreg, VexSimdPrefix pre, VexOpcode opc, bool rex_w) {
6812   if (pre > 0) {
6813     emit_int8(simd_pre[pre]);
6814   }
6815   if (rex_w) {
6816     prefixq(adr, xreg);
6817   } else {
6818     prefix(adr, xreg);
6819   }
6820   if (opc > 0) {
6821     emit_int8(0x0F);
6822     int opc2 = simd_opc[opc];
6823     if (opc2 > 0) {
6824       emit_int8(opc2);
6825     }
6826   }
6827 }
6828 
6829 int Assembler::rex_prefix_and_encode(int dst_enc, int src_enc, VexSimdPrefix pre, VexOpcode opc, bool rex_w) {
6830   if (pre > 0) {
6831     emit_int8(simd_pre[pre]);
6832   }
6833   int encode = (rex_w) ? prefixq_and_encode(dst_enc, src_enc) : prefix_and_encode(dst_enc, src_enc);
6834   if (opc > 0) {
6835     emit_int8(0x0F);
6836     int opc2 = simd_opc[opc];
6837     if (opc2 > 0) {
6838       emit_int8(opc2);
6839     }
6840   }
6841   return encode;
6842 }
6843 
6844 
6845 void Assembler::vex_prefix(bool vex_r, bool vex_b, bool vex_x, int nds_enc, VexSimdPrefix pre, VexOpcode opc) {
6846   int vector_len = _attributes->get_vector_len();
6847   bool vex_w = _attributes->is_rex_vex_w();
6848   if (vex_b || vex_x || vex_w || (opc == VEX_OPCODE_0F_38) || (opc == VEX_OPCODE_0F_3A)) {
6849     prefix(VEX_3bytes);
6850 
6851     int byte1 = (vex_r ? VEX_R : 0) | (vex_x ? VEX_X : 0) | (vex_b ? VEX_B : 0);
6852     byte1 = (~byte1) & 0xE0;
6853     byte1 |= opc;
6854     emit_int8(byte1);
6855 
6856     int byte2 = ((~nds_enc) & 0xf) << 3;
6857     byte2 |= (vex_w ? VEX_W : 0) | ((vector_len > 0) ? 4 : 0) | pre;
6858     emit_int8(byte2);
6859   } else {
6860     prefix(VEX_2bytes);
6861 
6862     int byte1 = vex_r ? VEX_R : 0;
6863     byte1 = (~byte1) & 0x80;
6864     byte1 |= ((~nds_enc) & 0xf) << 3;
6865     byte1 |= ((vector_len > 0 ) ? 4 : 0) | pre;
6866     emit_int8(byte1);
6867   }
6868 }
6869 
6870 // This is a 4 byte encoding
6871 void Assembler::evex_prefix(bool vex_r, bool vex_b, bool vex_x, bool evex_r, bool evex_v, int nds_enc, VexSimdPrefix pre, VexOpcode opc){
6872   // EVEX 0x62 prefix
6873   prefix(EVEX_4bytes);
6874   bool vex_w = _attributes->is_rex_vex_w();
6875   int evex_encoding = (vex_w ? VEX_W : 0);
6876   // EVEX.b is not currently used for broadcast of single element or data rounding modes
6877   _attributes->set_evex_encoding(evex_encoding);
6878 
6879   // P0: byte 2, initialized to RXBR`00mm
6880   // instead of not'd
6881   int byte2 = (vex_r ? VEX_R : 0) | (vex_x ? VEX_X : 0) | (vex_b ? VEX_B : 0) | (evex_r ? EVEX_Rb : 0);
6882   byte2 = (~byte2) & 0xF0;
6883   // confine opc opcode extensions in mm bits to lower two bits
6884   // of form {0F, 0F_38, 0F_3A}
6885   byte2 |= opc;
6886   emit_int8(byte2);
6887 
6888   // P1: byte 3 as Wvvvv1pp
6889   int byte3 = ((~nds_enc) & 0xf) << 3;
6890   // p[10] is always 1
6891   byte3 |= EVEX_F;
6892   byte3 |= (vex_w & 1) << 7;
6893   // confine pre opcode extensions in pp bits to lower two bits
6894   // of form {66, F3, F2}
6895   byte3 |= pre;
6896   emit_int8(byte3);
6897 
6898   // P2: byte 4 as zL'Lbv'aaa
6899   int byte4 = (_attributes->is_no_reg_mask()) ? 0 : 1; // kregs are implemented in the low 3 bits as aaa (hard code k1, it will be initialized for now)
6900   // EVEX.v` for extending EVEX.vvvv or VIDX
6901   byte4 |= (evex_v ? 0: EVEX_V);
6902   // third EXEC.b for broadcast actions
6903   byte4 |= (_attributes->is_extended_context() ? EVEX_Rb : 0);
6904   // fourth EVEX.L'L for vector length : 0 is 128, 1 is 256, 2 is 512, currently we do not support 1024
6905   byte4 |= ((_attributes->get_vector_len())& 0x3) << 5;
6906   // last is EVEX.z for zero/merge actions
6907   byte4 |= (_attributes->is_clear_context() ? EVEX_Z : 0);
6908   emit_int8(byte4);
6909 }
6910 
6911 void Assembler::vex_prefix(Address adr, int nds_enc, int xreg_enc, VexSimdPrefix pre, VexOpcode opc, InstructionAttr *attributes) {
6912   bool vex_r = ((xreg_enc & 8) == 8) ? 1 : 0;
6913   bool vex_b = adr.base_needs_rex();
6914   bool vex_x = adr.index_needs_rex();
6915   set_attributes(attributes);
6916   attributes->set_current_assembler(this);
6917 
6918   // if vector length is turned off, revert to AVX for vectors smaller than 512-bit
6919   if (UseAVX > 2 && _legacy_mode_vl && attributes->uses_vl()) {
6920     switch (attributes->get_vector_len()) {
6921     case AVX_128bit:
6922     case AVX_256bit:
6923       attributes->set_is_legacy_mode();
6924       break;
6925     }
6926   }
6927 
6928   // For pure EVEX check and see if this instruction
6929   // is allowed in legacy mode and has resources which will
6930   // fit in it.  Pure EVEX instructions will use set_is_evex_instruction in their definition,
6931   // else that field is set when we encode to EVEX
6932   if (UseAVX > 2 && !attributes->is_legacy_mode() &&
6933       !_is_managed && !attributes->is_evex_instruction()) {
6934     if (!_legacy_mode_vl && attributes->get_vector_len() != AVX_512bit) {
6935       bool check_register_bank = NOT_IA32(true) IA32_ONLY(false);
6936       if (check_register_bank) {
6937         // check nds_enc and xreg_enc for upper bank usage
6938         if (nds_enc < 16 && xreg_enc < 16) {
6939           attributes->set_is_legacy_mode();
6940         }
6941       } else {
6942         attributes->set_is_legacy_mode();
6943       }
6944     }
6945   }
6946 
6947   _is_managed = false;
6948   if (UseAVX > 2 && !attributes->is_legacy_mode())
6949   {
6950     bool evex_r = (xreg_enc >= 16);
6951     bool evex_v = (nds_enc >= 16);
6952     attributes->set_is_evex_instruction();
6953     evex_prefix(vex_r, vex_b, vex_x, evex_r, evex_v, nds_enc, pre, opc);
6954   } else {
6955     if (UseAVX > 2 && attributes->is_rex_vex_w_reverted()) {
6956       attributes->set_rex_vex_w(false);
6957     }
6958     vex_prefix(vex_r, vex_b, vex_x, nds_enc, pre, opc);
6959   }
6960 }
6961 
6962 int Assembler::vex_prefix_and_encode(int dst_enc, int nds_enc, int src_enc, VexSimdPrefix pre, VexOpcode opc, InstructionAttr *attributes) {
6963   bool vex_r = ((dst_enc & 8) == 8) ? 1 : 0;
6964   bool vex_b = ((src_enc & 8) == 8) ? 1 : 0;
6965   bool vex_x = false;
6966   set_attributes(attributes);
6967   attributes->set_current_assembler(this);
6968   bool check_register_bank = NOT_IA32(true) IA32_ONLY(false);
6969 
6970   // if vector length is turned off, revert to AVX for vectors smaller than 512-bit
6971   if (UseAVX > 2 && _legacy_mode_vl && attributes->uses_vl()) {
6972     switch (attributes->get_vector_len()) {
6973     case AVX_128bit:
6974     case AVX_256bit:
6975       if (check_register_bank) {
6976         if (dst_enc >= 16 || nds_enc >= 16 || src_enc >= 16) {
6977           // up propagate arithmetic instructions to meet RA requirements
6978           attributes->set_vector_len(AVX_512bit);
6979         } else {
6980           attributes->set_is_legacy_mode();
6981         }
6982       } else {
6983         attributes->set_is_legacy_mode();
6984       }
6985       break;
6986     }
6987   }
6988 
6989   // For pure EVEX check and see if this instruction
6990   // is allowed in legacy mode and has resources which will
6991   // fit in it.  Pure EVEX instructions will use set_is_evex_instruction in their definition,
6992   // else that field is set when we encode to EVEX
6993   if (UseAVX > 2 && !attributes->is_legacy_mode() &&
6994       !_is_managed && !attributes->is_evex_instruction()) {
6995     if (!_legacy_mode_vl && attributes->get_vector_len() != AVX_512bit) {
6996       if (check_register_bank) {
6997         // check dst_enc, nds_enc and src_enc for upper bank usage
6998         if (dst_enc < 16 && nds_enc < 16 && src_enc < 16) {
6999           attributes->set_is_legacy_mode();
7000         }
7001       } else {
7002         attributes->set_is_legacy_mode();
7003       }
7004     }
7005   }
7006 
7007   _is_managed = false;
7008   if (UseAVX > 2 && !attributes->is_legacy_mode())
7009   {
7010     bool evex_r = (dst_enc >= 16);
7011     bool evex_v = (nds_enc >= 16);
7012     // can use vex_x as bank extender on rm encoding
7013     vex_x = (src_enc >= 16);
7014     attributes->set_is_evex_instruction();
7015     evex_prefix(vex_r, vex_b, vex_x, evex_r, evex_v, nds_enc, pre, opc);
7016   } else {
7017     if (UseAVX > 2 && attributes->is_rex_vex_w_reverted()) {
7018       attributes->set_rex_vex_w(false);
7019     }
7020     vex_prefix(vex_r, vex_b, vex_x, nds_enc, pre, opc);
7021   }
7022 
7023   // return modrm byte components for operands
7024   return (((dst_enc & 7) << 3) | (src_enc & 7));
7025 }
7026 
7027 
7028 void Assembler::simd_prefix(XMMRegister xreg, XMMRegister nds, Address adr, VexSimdPrefix pre,
7029                             VexOpcode opc, InstructionAttr *attributes) {
7030   if (UseAVX > 0) {
7031     int xreg_enc = xreg->encoding();
7032     int nds_enc = nds->is_valid() ? nds->encoding() : 0;
7033     vex_prefix(adr, nds_enc, xreg_enc, pre, opc, attributes);
7034   } else {
7035     assert((nds == xreg) || (nds == xnoreg), "wrong sse encoding");
7036     rex_prefix(adr, xreg, pre, opc, attributes->is_rex_vex_w());
7037   }
7038 }
7039 
7040 int Assembler::simd_prefix_and_encode(XMMRegister dst, XMMRegister nds, XMMRegister src, VexSimdPrefix pre,
7041                                       VexOpcode opc, InstructionAttr *attributes) {
7042   int dst_enc = dst->encoding();
7043   int src_enc = src->encoding();
7044   if (UseAVX > 0) {
7045     int nds_enc = nds->is_valid() ? nds->encoding() : 0;
7046     return vex_prefix_and_encode(dst_enc, nds_enc, src_enc, pre, opc, attributes);
7047   } else {
7048     assert((nds == dst) || (nds == src) || (nds == xnoreg), "wrong sse encoding");
7049     return rex_prefix_and_encode(dst_enc, src_enc, pre, opc, attributes->is_rex_vex_w());
7050   }
7051 }
7052 
7053 void Assembler::cmppd(XMMRegister dst, XMMRegister nds, XMMRegister src, int cop, int vector_len) {
7054   assert(VM_Version::supports_avx(), "");
7055   assert(!VM_Version::supports_evex(), "");
7056   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
7057   int encode = simd_prefix_and_encode(dst, nds, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7058   emit_int8((unsigned char)0xC2);
7059   emit_int8((unsigned char)(0xC0 | encode));
7060   emit_int8((unsigned char)(0xF & cop));
7061 }
7062 
7063 void Assembler::vpblendd(XMMRegister dst, XMMRegister nds, XMMRegister src1, XMMRegister src2, int vector_len) {
7064   assert(VM_Version::supports_avx(), "");
7065   assert(!VM_Version::supports_evex(), "");
7066   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
7067   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src1->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
7068   emit_int8((unsigned char)0x4B);
7069   emit_int8((unsigned char)(0xC0 | encode));
7070   int src2_enc = src2->encoding();
7071   emit_int8((unsigned char)(0xF0 & src2_enc<<4));
7072 }
7073 
7074 void Assembler::shlxl(Register dst, Register src1, Register src2) {
7075   assert(VM_Version::supports_bmi2(), "");
7076   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
7077   int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src1->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
7078   emit_int8((unsigned char)0xF7);
7079   emit_int8((unsigned char)(0xC0 | encode));
7080 }
7081 
7082 void Assembler::shlxq(Register dst, Register src1, Register src2) {
7083   assert(VM_Version::supports_bmi2(), "");
7084   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
7085   int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src1->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
7086   emit_int8((unsigned char)0xF7);
7087   emit_int8((unsigned char)(0xC0 | encode));
7088 }
7089 
7090 #ifndef _LP64
7091 
7092 void Assembler::incl(Register dst) {
7093   // Don't use it directly. Use MacroAssembler::incrementl() instead.
7094   emit_int8(0x40 | dst->encoding());
7095 }
7096 
7097 void Assembler::lea(Register dst, Address src) {
7098   leal(dst, src);
7099 }
7100 
7101 void Assembler::mov_literal32(Address dst, int32_t imm32, RelocationHolder const& rspec) {
7102   InstructionMark im(this);
7103   emit_int8((unsigned char)0xC7);
7104   emit_operand(rax, dst);
7105   emit_data((int)imm32, rspec, 0);
7106 }
7107 
7108 void Assembler::mov_literal32(Register dst, int32_t imm32, RelocationHolder const& rspec) {
7109   InstructionMark im(this);
7110   int encode = prefix_and_encode(dst->encoding());
7111   emit_int8((unsigned char)(0xB8 | encode));
7112   emit_data((int)imm32, rspec, 0);
7113 }
7114 
7115 void Assembler::popa() { // 32bit
7116   emit_int8(0x61);
7117 }
7118 
7119 void Assembler::push_literal32(int32_t imm32, RelocationHolder const& rspec) {
7120   InstructionMark im(this);
7121   emit_int8(0x68);
7122   emit_data(imm32, rspec, 0);
7123 }
7124 
7125 void Assembler::pusha() { // 32bit
7126   emit_int8(0x60);
7127 }
7128 
7129 void Assembler::set_byte_if_not_zero(Register dst) {
7130   emit_int8(0x0F);
7131   emit_int8((unsigned char)0x95);
7132   emit_int8((unsigned char)(0xE0 | dst->encoding()));
7133 }
7134 
7135 void Assembler::shldl(Register dst, Register src) {
7136   emit_int8(0x0F);
7137   emit_int8((unsigned char)0xA5);
7138   emit_int8((unsigned char)(0xC0 | src->encoding() << 3 | dst->encoding()));
7139 }
7140 
7141 // 0F A4 / r ib
7142 void Assembler::shldl(Register dst, Register src, int8_t imm8) {
7143   emit_int8(0x0F);
7144   emit_int8((unsigned char)0xA4);
7145   emit_int8((unsigned char)(0xC0 | src->encoding() << 3 | dst->encoding()));
7146   emit_int8(imm8);
7147 }
7148 
7149 void Assembler::shrdl(Register dst, Register src) {
7150   emit_int8(0x0F);
7151   emit_int8((unsigned char)0xAD);
7152   emit_int8((unsigned char)(0xC0 | src->encoding() << 3 | dst->encoding()));
7153 }
7154 
7155 #else // LP64
7156 
7157 void Assembler::set_byte_if_not_zero(Register dst) {
7158   int enc = prefix_and_encode(dst->encoding(), true);
7159   emit_int8(0x0F);
7160   emit_int8((unsigned char)0x95);
7161   emit_int8((unsigned char)(0xE0 | enc));
7162 }
7163 
7164 // 64bit only pieces of the assembler
7165 // This should only be used by 64bit instructions that can use rip-relative
7166 // it cannot be used by instructions that want an immediate value.
7167 
7168 bool Assembler::reachable(AddressLiteral adr) {
7169   int64_t disp;
7170   // None will force a 64bit literal to the code stream. Likely a placeholder
7171   // for something that will be patched later and we need to certain it will
7172   // always be reachable.
7173   if (adr.reloc() == relocInfo::none) {
7174     return false;
7175   }
7176   if (adr.reloc() == relocInfo::internal_word_type) {
7177     // This should be rip relative and easily reachable.
7178     return true;
7179   }
7180   if (adr.reloc() == relocInfo::virtual_call_type ||
7181       adr.reloc() == relocInfo::opt_virtual_call_type ||
7182       adr.reloc() == relocInfo::static_call_type ||
7183       adr.reloc() == relocInfo::static_stub_type ) {
7184     // This should be rip relative within the code cache and easily
7185     // reachable until we get huge code caches. (At which point
7186     // ic code is going to have issues).
7187     return true;
7188   }
7189   if (adr.reloc() != relocInfo::external_word_type &&
7190       adr.reloc() != relocInfo::poll_return_type &&  // these are really external_word but need special
7191       adr.reloc() != relocInfo::poll_type &&         // relocs to identify them
7192       adr.reloc() != relocInfo::runtime_call_type ) {
7193     return false;
7194   }
7195 
7196   // Stress the correction code
7197   if (ForceUnreachable) {
7198     // Must be runtimecall reloc, see if it is in the codecache
7199     // Flipping stuff in the codecache to be unreachable causes issues
7200     // with things like inline caches where the additional instructions
7201     // are not handled.
7202     if (CodeCache::find_blob(adr._target) == NULL) {
7203       return false;
7204     }
7205   }
7206   // For external_word_type/runtime_call_type if it is reachable from where we
7207   // are now (possibly a temp buffer) and where we might end up
7208   // anywhere in the codeCache then we are always reachable.
7209   // This would have to change if we ever save/restore shared code
7210   // to be more pessimistic.
7211   disp = (int64_t)adr._target - ((int64_t)CodeCache::low_bound() + sizeof(int));
7212   if (!is_simm32(disp)) return false;
7213   disp = (int64_t)adr._target - ((int64_t)CodeCache::high_bound() + sizeof(int));
7214   if (!is_simm32(disp)) return false;
7215 
7216   disp = (int64_t)adr._target - ((int64_t)pc() + sizeof(int));
7217 
7218   // Because rip relative is a disp + address_of_next_instruction and we
7219   // don't know the value of address_of_next_instruction we apply a fudge factor
7220   // to make sure we will be ok no matter the size of the instruction we get placed into.
7221   // We don't have to fudge the checks above here because they are already worst case.
7222 
7223   // 12 == override/rex byte, opcode byte, rm byte, sib byte, a 4-byte disp , 4-byte literal
7224   // + 4 because better safe than sorry.
7225   const int fudge = 12 + 4;
7226   if (disp < 0) {
7227     disp -= fudge;
7228   } else {
7229     disp += fudge;
7230   }
7231   return is_simm32(disp);
7232 }
7233 
7234 // Check if the polling page is not reachable from the code cache using rip-relative
7235 // addressing.
7236 bool Assembler::is_polling_page_far() {
7237   intptr_t addr = (intptr_t)os::get_polling_page();
7238   return ForceUnreachable ||
7239          !is_simm32(addr - (intptr_t)CodeCache::low_bound()) ||
7240          !is_simm32(addr - (intptr_t)CodeCache::high_bound());
7241 }
7242 
7243 void Assembler::emit_data64(jlong data,
7244                             relocInfo::relocType rtype,
7245                             int format) {
7246   if (rtype == relocInfo::none) {
7247     emit_int64(data);
7248   } else {
7249     emit_data64(data, Relocation::spec_simple(rtype), format);
7250   }
7251 }
7252 
7253 void Assembler::emit_data64(jlong data,
7254                             RelocationHolder const& rspec,
7255                             int format) {
7256   assert(imm_operand == 0, "default format must be immediate in this file");
7257   assert(imm_operand == format, "must be immediate");
7258   assert(inst_mark() != NULL, "must be inside InstructionMark");
7259   // Do not use AbstractAssembler::relocate, which is not intended for
7260   // embedded words.  Instead, relocate to the enclosing instruction.
7261   code_section()->relocate(inst_mark(), rspec, format);
7262 #ifdef ASSERT
7263   check_relocation(rspec, format);
7264 #endif
7265   emit_int64(data);
7266 }
7267 
7268 int Assembler::prefix_and_encode(int reg_enc, bool byteinst) {
7269   if (reg_enc >= 8) {
7270     prefix(REX_B);
7271     reg_enc -= 8;
7272   } else if (byteinst && reg_enc >= 4) {
7273     prefix(REX);
7274   }
7275   return reg_enc;
7276 }
7277 
7278 int Assembler::prefixq_and_encode(int reg_enc) {
7279   if (reg_enc < 8) {
7280     prefix(REX_W);
7281   } else {
7282     prefix(REX_WB);
7283     reg_enc -= 8;
7284   }
7285   return reg_enc;
7286 }
7287 
7288 int Assembler::prefix_and_encode(int dst_enc, bool dst_is_byte, int src_enc, bool src_is_byte) {
7289   if (dst_enc < 8) {
7290     if (src_enc >= 8) {
7291       prefix(REX_B);
7292       src_enc -= 8;
7293     } else if ((src_is_byte && src_enc >= 4) || (dst_is_byte && dst_enc >= 4)) {
7294       prefix(REX);
7295     }
7296   } else {
7297     if (src_enc < 8) {
7298       prefix(REX_R);
7299     } else {
7300       prefix(REX_RB);
7301       src_enc -= 8;
7302     }
7303     dst_enc -= 8;
7304   }
7305   return dst_enc << 3 | src_enc;
7306 }
7307 
7308 int Assembler::prefixq_and_encode(int dst_enc, int src_enc) {
7309   if (dst_enc < 8) {
7310     if (src_enc < 8) {
7311       prefix(REX_W);
7312     } else {
7313       prefix(REX_WB);
7314       src_enc -= 8;
7315     }
7316   } else {
7317     if (src_enc < 8) {
7318       prefix(REX_WR);
7319     } else {
7320       prefix(REX_WRB);
7321       src_enc -= 8;
7322     }
7323     dst_enc -= 8;
7324   }
7325   return dst_enc << 3 | src_enc;
7326 }
7327 
7328 void Assembler::prefix(Register reg) {
7329   if (reg->encoding() >= 8) {
7330     prefix(REX_B);
7331   }
7332 }
7333 
7334 void Assembler::prefix(Register dst, Register src, Prefix p) {
7335   if (src->encoding() >= 8) {
7336     p = (Prefix)(p | REX_B);
7337   }
7338   if (dst->encoding() >= 8) {
7339     p = (Prefix)( p | REX_R);
7340   }
7341   if (p != Prefix_EMPTY) {
7342     // do not generate an empty prefix
7343     prefix(p);
7344   }
7345 }
7346 
7347 void Assembler::prefix(Register dst, Address adr, Prefix p) {
7348   if (adr.base_needs_rex()) {
7349     if (adr.index_needs_rex()) {
7350       assert(false, "prefix(Register dst, Address adr, Prefix p) does not support handling of an X");
7351     } else {
7352       prefix(REX_B);
7353     }
7354   } else {
7355     if (adr.index_needs_rex()) {
7356       assert(false, "prefix(Register dst, Address adr, Prefix p) does not support handling of an X");
7357     }
7358   }
7359   if (dst->encoding() >= 8) {
7360     p = (Prefix)(p | REX_R);
7361   }
7362   if (p != Prefix_EMPTY) {
7363     // do not generate an empty prefix
7364     prefix(p);
7365   }
7366 }
7367 
7368 void Assembler::prefix(Address adr) {
7369   if (adr.base_needs_rex()) {
7370     if (adr.index_needs_rex()) {
7371       prefix(REX_XB);
7372     } else {
7373       prefix(REX_B);
7374     }
7375   } else {
7376     if (adr.index_needs_rex()) {
7377       prefix(REX_X);
7378     }
7379   }
7380 }
7381 
7382 void Assembler::prefixq(Address adr) {
7383   if (adr.base_needs_rex()) {
7384     if (adr.index_needs_rex()) {
7385       prefix(REX_WXB);
7386     } else {
7387       prefix(REX_WB);
7388     }
7389   } else {
7390     if (adr.index_needs_rex()) {
7391       prefix(REX_WX);
7392     } else {
7393       prefix(REX_W);
7394     }
7395   }
7396 }
7397 
7398 
7399 void Assembler::prefix(Address adr, Register reg, bool byteinst) {
7400   if (reg->encoding() < 8) {
7401     if (adr.base_needs_rex()) {
7402       if (adr.index_needs_rex()) {
7403         prefix(REX_XB);
7404       } else {
7405         prefix(REX_B);
7406       }
7407     } else {
7408       if (adr.index_needs_rex()) {
7409         prefix(REX_X);
7410       } else if (byteinst && reg->encoding() >= 4 ) {
7411         prefix(REX);
7412       }
7413     }
7414   } else {
7415     if (adr.base_needs_rex()) {
7416       if (adr.index_needs_rex()) {
7417         prefix(REX_RXB);
7418       } else {
7419         prefix(REX_RB);
7420       }
7421     } else {
7422       if (adr.index_needs_rex()) {
7423         prefix(REX_RX);
7424       } else {
7425         prefix(REX_R);
7426       }
7427     }
7428   }
7429 }
7430 
7431 void Assembler::prefixq(Address adr, Register src) {
7432   if (src->encoding() < 8) {
7433     if (adr.base_needs_rex()) {
7434       if (adr.index_needs_rex()) {
7435         prefix(REX_WXB);
7436       } else {
7437         prefix(REX_WB);
7438       }
7439     } else {
7440       if (adr.index_needs_rex()) {
7441         prefix(REX_WX);
7442       } else {
7443         prefix(REX_W);
7444       }
7445     }
7446   } else {
7447     if (adr.base_needs_rex()) {
7448       if (adr.index_needs_rex()) {
7449         prefix(REX_WRXB);
7450       } else {
7451         prefix(REX_WRB);
7452       }
7453     } else {
7454       if (adr.index_needs_rex()) {
7455         prefix(REX_WRX);
7456       } else {
7457         prefix(REX_WR);
7458       }
7459     }
7460   }
7461 }
7462 
7463 void Assembler::prefix(Address adr, XMMRegister reg) {
7464   if (reg->encoding() < 8) {
7465     if (adr.base_needs_rex()) {
7466       if (adr.index_needs_rex()) {
7467         prefix(REX_XB);
7468       } else {
7469         prefix(REX_B);
7470       }
7471     } else {
7472       if (adr.index_needs_rex()) {
7473         prefix(REX_X);
7474       }
7475     }
7476   } else {
7477     if (adr.base_needs_rex()) {
7478       if (adr.index_needs_rex()) {
7479         prefix(REX_RXB);
7480       } else {
7481         prefix(REX_RB);
7482       }
7483     } else {
7484       if (adr.index_needs_rex()) {
7485         prefix(REX_RX);
7486       } else {
7487         prefix(REX_R);
7488       }
7489     }
7490   }
7491 }
7492 
7493 void Assembler::prefixq(Address adr, XMMRegister src) {
7494   if (src->encoding() < 8) {
7495     if (adr.base_needs_rex()) {
7496       if (adr.index_needs_rex()) {
7497         prefix(REX_WXB);
7498       } else {
7499         prefix(REX_WB);
7500       }
7501     } else {
7502       if (adr.index_needs_rex()) {
7503         prefix(REX_WX);
7504       } else {
7505         prefix(REX_W);
7506       }
7507     }
7508   } else {
7509     if (adr.base_needs_rex()) {
7510       if (adr.index_needs_rex()) {
7511         prefix(REX_WRXB);
7512       } else {
7513         prefix(REX_WRB);
7514       }
7515     } else {
7516       if (adr.index_needs_rex()) {
7517         prefix(REX_WRX);
7518       } else {
7519         prefix(REX_WR);
7520       }
7521     }
7522   }
7523 }
7524 
7525 void Assembler::adcq(Register dst, int32_t imm32) {
7526   (void) prefixq_and_encode(dst->encoding());
7527   emit_arith(0x81, 0xD0, dst, imm32);
7528 }
7529 
7530 void Assembler::adcq(Register dst, Address src) {
7531   InstructionMark im(this);
7532   prefixq(src, dst);
7533   emit_int8(0x13);
7534   emit_operand(dst, src);
7535 }
7536 
7537 void Assembler::adcq(Register dst, Register src) {
7538   (void) prefixq_and_encode(dst->encoding(), src->encoding());
7539   emit_arith(0x13, 0xC0, dst, src);
7540 }
7541 
7542 void Assembler::addq(Address dst, int32_t imm32) {
7543   InstructionMark im(this);
7544   prefixq(dst);
7545   emit_arith_operand(0x81, rax, dst,imm32);
7546 }
7547 
7548 void Assembler::addq(Address dst, Register src) {
7549   InstructionMark im(this);
7550   prefixq(dst, src);
7551   emit_int8(0x01);
7552   emit_operand(src, dst);
7553 }
7554 
7555 void Assembler::addq(Register dst, int32_t imm32) {
7556   (void) prefixq_and_encode(dst->encoding());
7557   emit_arith(0x81, 0xC0, dst, imm32);
7558 }
7559 
7560 void Assembler::addq(Register dst, Address src) {
7561   InstructionMark im(this);
7562   prefixq(src, dst);
7563   emit_int8(0x03);
7564   emit_operand(dst, src);
7565 }
7566 
7567 void Assembler::addq(Register dst, Register src) {
7568   (void) prefixq_and_encode(dst->encoding(), src->encoding());
7569   emit_arith(0x03, 0xC0, dst, src);
7570 }
7571 
7572 void Assembler::adcxq(Register dst, Register src) {
7573   //assert(VM_Version::supports_adx(), "adx instructions not supported");
7574   emit_int8((unsigned char)0x66);
7575   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
7576   emit_int8(0x0F);
7577   emit_int8(0x38);
7578   emit_int8((unsigned char)0xF6);
7579   emit_int8((unsigned char)(0xC0 | encode));
7580 }
7581 
7582 void Assembler::adoxq(Register dst, Register src) {
7583   //assert(VM_Version::supports_adx(), "adx instructions not supported");
7584   emit_int8((unsigned char)0xF3);
7585   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
7586   emit_int8(0x0F);
7587   emit_int8(0x38);
7588   emit_int8((unsigned char)0xF6);
7589   emit_int8((unsigned char)(0xC0 | encode));
7590 }
7591 
7592 void Assembler::andq(Address dst, int32_t imm32) {
7593   InstructionMark im(this);
7594   prefixq(dst);
7595   emit_int8((unsigned char)0x81);
7596   emit_operand(rsp, dst, 4);
7597   emit_int32(imm32);
7598 }
7599 
7600 void Assembler::andq(Register dst, int32_t imm32) {
7601   (void) prefixq_and_encode(dst->encoding());
7602   emit_arith(0x81, 0xE0, dst, imm32);
7603 }
7604 
7605 void Assembler::andq(Register dst, Address src) {
7606   InstructionMark im(this);
7607   prefixq(src, dst);
7608   emit_int8(0x23);
7609   emit_operand(dst, src);
7610 }
7611 
7612 void Assembler::andq(Register dst, Register src) {
7613   (void) prefixq_and_encode(dst->encoding(), src->encoding());
7614   emit_arith(0x23, 0xC0, dst, src);
7615 }
7616 
7617 void Assembler::andnq(Register dst, Register src1, Register src2) {
7618   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
7619   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
7620   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
7621   emit_int8((unsigned char)0xF2);
7622   emit_int8((unsigned char)(0xC0 | encode));
7623 }
7624 
7625 void Assembler::andnq(Register dst, Register src1, Address src2) {
7626   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
7627   InstructionMark im(this);
7628   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
7629   vex_prefix(src2, src1->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
7630   emit_int8((unsigned char)0xF2);
7631   emit_operand(dst, src2);
7632 }
7633 
7634 void Assembler::bsfq(Register dst, Register src) {
7635   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
7636   emit_int8(0x0F);
7637   emit_int8((unsigned char)0xBC);
7638   emit_int8((unsigned char)(0xC0 | encode));
7639 }
7640 
7641 void Assembler::bsrq(Register dst, Register src) {
7642   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
7643   emit_int8(0x0F);
7644   emit_int8((unsigned char)0xBD);
7645   emit_int8((unsigned char)(0xC0 | encode));
7646 }
7647 
7648 void Assembler::bswapq(Register reg) {
7649   int encode = prefixq_and_encode(reg->encoding());
7650   emit_int8(0x0F);
7651   emit_int8((unsigned char)(0xC8 | encode));
7652 }
7653 
7654 void Assembler::blsiq(Register dst, Register src) {
7655   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
7656   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
7657   int encode = vex_prefix_and_encode(rbx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
7658   emit_int8((unsigned char)0xF3);
7659   emit_int8((unsigned char)(0xC0 | encode));
7660 }
7661 
7662 void Assembler::blsiq(Register dst, Address src) {
7663   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
7664   InstructionMark im(this);
7665   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
7666   vex_prefix(src, dst->encoding(), rbx->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
7667   emit_int8((unsigned char)0xF3);
7668   emit_operand(rbx, src);
7669 }
7670 
7671 void Assembler::blsmskq(Register dst, Register src) {
7672   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
7673   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
7674   int encode = vex_prefix_and_encode(rdx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
7675   emit_int8((unsigned char)0xF3);
7676   emit_int8((unsigned char)(0xC0 | encode));
7677 }
7678 
7679 void Assembler::blsmskq(Register dst, Address src) {
7680   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
7681   InstructionMark im(this);
7682   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
7683   vex_prefix(src, dst->encoding(), rdx->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
7684   emit_int8((unsigned char)0xF3);
7685   emit_operand(rdx, src);
7686 }
7687 
7688 void Assembler::blsrq(Register dst, Register src) {
7689   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
7690   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
7691   int encode = vex_prefix_and_encode(rcx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
7692   emit_int8((unsigned char)0xF3);
7693   emit_int8((unsigned char)(0xC0 | encode));
7694 }
7695 
7696 void Assembler::blsrq(Register dst, Address src) {
7697   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
7698   InstructionMark im(this);
7699   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
7700   vex_prefix(src, dst->encoding(), rcx->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
7701   emit_int8((unsigned char)0xF3);
7702   emit_operand(rcx, src);
7703 }
7704 
7705 void Assembler::cdqq() {
7706   prefix(REX_W);
7707   emit_int8((unsigned char)0x99);
7708 }
7709 
7710 void Assembler::clflush(Address adr) {
7711   prefix(adr);
7712   emit_int8(0x0F);
7713   emit_int8((unsigned char)0xAE);
7714   emit_operand(rdi, adr);
7715 }
7716 
7717 void Assembler::cmovq(Condition cc, Register dst, Register src) {
7718   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
7719   emit_int8(0x0F);
7720   emit_int8(0x40 | cc);
7721   emit_int8((unsigned char)(0xC0 | encode));
7722 }
7723 
7724 void Assembler::cmovq(Condition cc, Register dst, Address src) {
7725   InstructionMark im(this);
7726   prefixq(src, dst);
7727   emit_int8(0x0F);
7728   emit_int8(0x40 | cc);
7729   emit_operand(dst, src);
7730 }
7731 
7732 void Assembler::cmpq(Address dst, int32_t imm32) {
7733   InstructionMark im(this);
7734   prefixq(dst);
7735   emit_int8((unsigned char)0x81);
7736   emit_operand(rdi, dst, 4);
7737   emit_int32(imm32);
7738 }
7739 
7740 void Assembler::cmpq(Register dst, int32_t imm32) {
7741   (void) prefixq_and_encode(dst->encoding());
7742   emit_arith(0x81, 0xF8, dst, imm32);
7743 }
7744 
7745 void Assembler::cmpq(Address dst, Register src) {
7746   InstructionMark im(this);
7747   prefixq(dst, src);
7748   emit_int8(0x3B);
7749   emit_operand(src, dst);
7750 }
7751 
7752 void Assembler::cmpq(Register dst, Register src) {
7753   (void) prefixq_and_encode(dst->encoding(), src->encoding());
7754   emit_arith(0x3B, 0xC0, dst, src);
7755 }
7756 
7757 void Assembler::cmpq(Register dst, Address  src) {
7758   InstructionMark im(this);
7759   prefixq(src, dst);
7760   emit_int8(0x3B);
7761   emit_operand(dst, src);
7762 }
7763 
7764 void Assembler::cmpxchgq(Register reg, Address adr) {
7765   InstructionMark im(this);
7766   prefixq(adr, reg);
7767   emit_int8(0x0F);
7768   emit_int8((unsigned char)0xB1);
7769   emit_operand(reg, adr);
7770 }
7771 
7772 void Assembler::cvtsi2sdq(XMMRegister dst, Register src) {
7773   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
7774   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
7775   int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
7776   emit_int8(0x2A);
7777   emit_int8((unsigned char)(0xC0 | encode));
7778 }
7779 
7780 void Assembler::cvtsi2sdq(XMMRegister dst, Address src) {
7781   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
7782   InstructionMark im(this);
7783   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
7784   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
7785   simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
7786   emit_int8(0x2A);
7787   emit_operand(dst, src);
7788 }
7789 
7790 void Assembler::cvtsi2ssq(XMMRegister dst, Address src) {
7791   NOT_LP64(assert(VM_Version::supports_sse(), ""));
7792   InstructionMark im(this);
7793   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
7794   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
7795   simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
7796   emit_int8(0x2A);
7797   emit_operand(dst, src);
7798 }
7799 
7800 void Assembler::cvttsd2siq(Register dst, XMMRegister src) {
7801   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
7802   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
7803   int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
7804   emit_int8(0x2C);
7805   emit_int8((unsigned char)(0xC0 | encode));
7806 }
7807 
7808 void Assembler::cvttss2siq(Register dst, XMMRegister src) {
7809   NOT_LP64(assert(VM_Version::supports_sse(), ""));
7810   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
7811   int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
7812   emit_int8(0x2C);
7813   emit_int8((unsigned char)(0xC0 | encode));
7814 }
7815 
7816 void Assembler::decl(Register dst) {
7817   // Don't use it directly. Use MacroAssembler::decrementl() instead.
7818   // Use two-byte form (one-byte form is a REX prefix in 64-bit mode)
7819   int encode = prefix_and_encode(dst->encoding());
7820   emit_int8((unsigned char)0xFF);
7821   emit_int8((unsigned char)(0xC8 | encode));
7822 }
7823 
7824 void Assembler::decq(Register dst) {
7825   // Don't use it directly. Use MacroAssembler::decrementq() instead.
7826   // Use two-byte form (one-byte from is a REX prefix in 64-bit mode)
7827   int encode = prefixq_and_encode(dst->encoding());
7828   emit_int8((unsigned char)0xFF);
7829   emit_int8(0xC8 | encode);
7830 }
7831 
7832 void Assembler::decq(Address dst) {
7833   // Don't use it directly. Use MacroAssembler::decrementq() instead.
7834   InstructionMark im(this);
7835   prefixq(dst);
7836   emit_int8((unsigned char)0xFF);
7837   emit_operand(rcx, dst);
7838 }
7839 
7840 void Assembler::fxrstor(Address src) {
7841   prefixq(src);
7842   emit_int8(0x0F);
7843   emit_int8((unsigned char)0xAE);
7844   emit_operand(as_Register(1), src);
7845 }
7846 
7847 void Assembler::xrstor(Address src) {
7848   prefixq(src);
7849   emit_int8(0x0F);
7850   emit_int8((unsigned char)0xAE);
7851   emit_operand(as_Register(5), src);
7852 }
7853 
7854 void Assembler::fxsave(Address dst) {
7855   prefixq(dst);
7856   emit_int8(0x0F);
7857   emit_int8((unsigned char)0xAE);
7858   emit_operand(as_Register(0), dst);
7859 }
7860 
7861 void Assembler::xsave(Address dst) {
7862   prefixq(dst);
7863   emit_int8(0x0F);
7864   emit_int8((unsigned char)0xAE);
7865   emit_operand(as_Register(4), dst);
7866 }
7867 
7868 void Assembler::idivq(Register src) {
7869   int encode = prefixq_and_encode(src->encoding());
7870   emit_int8((unsigned char)0xF7);
7871   emit_int8((unsigned char)(0xF8 | encode));
7872 }
7873 
7874 void Assembler::imulq(Register dst, Register src) {
7875   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
7876   emit_int8(0x0F);
7877   emit_int8((unsigned char)0xAF);
7878   emit_int8((unsigned char)(0xC0 | encode));
7879 }
7880 
7881 void Assembler::imulq(Register dst, Register src, int value) {
7882   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
7883   if (is8bit(value)) {
7884     emit_int8(0x6B);
7885     emit_int8((unsigned char)(0xC0 | encode));
7886     emit_int8(value & 0xFF);
7887   } else {
7888     emit_int8(0x69);
7889     emit_int8((unsigned char)(0xC0 | encode));
7890     emit_int32(value);
7891   }
7892 }
7893 
7894 void Assembler::imulq(Register dst, Address src) {
7895   InstructionMark im(this);
7896   prefixq(src, dst);
7897   emit_int8(0x0F);
7898   emit_int8((unsigned char) 0xAF);
7899   emit_operand(dst, src);
7900 }
7901 
7902 void Assembler::incl(Register dst) {
7903   // Don't use it directly. Use MacroAssembler::incrementl() instead.
7904   // Use two-byte form (one-byte from is a REX prefix in 64-bit mode)
7905   int encode = prefix_and_encode(dst->encoding());
7906   emit_int8((unsigned char)0xFF);
7907   emit_int8((unsigned char)(0xC0 | encode));
7908 }
7909 
7910 void Assembler::incq(Register dst) {
7911   // Don't use it directly. Use MacroAssembler::incrementq() instead.
7912   // Use two-byte form (one-byte from is a REX prefix in 64-bit mode)
7913   int encode = prefixq_and_encode(dst->encoding());
7914   emit_int8((unsigned char)0xFF);
7915   emit_int8((unsigned char)(0xC0 | encode));
7916 }
7917 
7918 void Assembler::incq(Address dst) {
7919   // Don't use it directly. Use MacroAssembler::incrementq() instead.
7920   InstructionMark im(this);
7921   prefixq(dst);
7922   emit_int8((unsigned char)0xFF);
7923   emit_operand(rax, dst);
7924 }
7925 
7926 void Assembler::lea(Register dst, Address src) {
7927   leaq(dst, src);
7928 }
7929 
7930 void Assembler::leaq(Register dst, Address src) {
7931   InstructionMark im(this);
7932   prefixq(src, dst);
7933   emit_int8((unsigned char)0x8D);
7934   emit_operand(dst, src);
7935 }
7936 
7937 void Assembler::mov64(Register dst, int64_t imm64) {
7938   InstructionMark im(this);
7939   int encode = prefixq_and_encode(dst->encoding());
7940   emit_int8((unsigned char)(0xB8 | encode));
7941   emit_int64(imm64);
7942 }
7943 
7944 void Assembler::mov_literal64(Register dst, intptr_t imm64, RelocationHolder const& rspec) {
7945   InstructionMark im(this);
7946   int encode = prefixq_and_encode(dst->encoding());
7947   emit_int8(0xB8 | encode);
7948   emit_data64(imm64, rspec);
7949 }
7950 
7951 void Assembler::mov_narrow_oop(Register dst, int32_t imm32, RelocationHolder const& rspec) {
7952   InstructionMark im(this);
7953   int encode = prefix_and_encode(dst->encoding());
7954   emit_int8((unsigned char)(0xB8 | encode));
7955   emit_data((int)imm32, rspec, narrow_oop_operand);
7956 }
7957 
7958 void Assembler::mov_narrow_oop(Address dst, int32_t imm32,  RelocationHolder const& rspec) {
7959   InstructionMark im(this);
7960   prefix(dst);
7961   emit_int8((unsigned char)0xC7);
7962   emit_operand(rax, dst, 4);
7963   emit_data((int)imm32, rspec, narrow_oop_operand);
7964 }
7965 
7966 void Assembler::cmp_narrow_oop(Register src1, int32_t imm32, RelocationHolder const& rspec) {
7967   InstructionMark im(this);
7968   int encode = prefix_and_encode(src1->encoding());
7969   emit_int8((unsigned char)0x81);
7970   emit_int8((unsigned char)(0xF8 | encode));
7971   emit_data((int)imm32, rspec, narrow_oop_operand);
7972 }
7973 
7974 void Assembler::cmp_narrow_oop(Address src1, int32_t imm32, RelocationHolder const& rspec) {
7975   InstructionMark im(this);
7976   prefix(src1);
7977   emit_int8((unsigned char)0x81);
7978   emit_operand(rax, src1, 4);
7979   emit_data((int)imm32, rspec, narrow_oop_operand);
7980 }
7981 
7982 void Assembler::lzcntq(Register dst, Register src) {
7983   assert(VM_Version::supports_lzcnt(), "encoding is treated as BSR");
7984   emit_int8((unsigned char)0xF3);
7985   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
7986   emit_int8(0x0F);
7987   emit_int8((unsigned char)0xBD);
7988   emit_int8((unsigned char)(0xC0 | encode));
7989 }
7990 
7991 void Assembler::movdq(XMMRegister dst, Register src) {
7992   // table D-1 says MMX/SSE2
7993   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
7994   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
7995   int encode = simd_prefix_and_encode(dst, xnoreg, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7996   emit_int8(0x6E);
7997   emit_int8((unsigned char)(0xC0 | encode));
7998 }
7999 
8000 void Assembler::movdq(Register dst, XMMRegister src) {
8001   // table D-1 says MMX/SSE2
8002   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
8003   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
8004   // swap src/dst to get correct prefix
8005   int encode = simd_prefix_and_encode(src, xnoreg, as_XMMRegister(dst->encoding()), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
8006   emit_int8(0x7E);
8007   emit_int8((unsigned char)(0xC0 | encode));
8008 }
8009 
8010 void Assembler::movq(Register dst, Register src) {
8011   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
8012   emit_int8((unsigned char)0x8B);
8013   emit_int8((unsigned char)(0xC0 | encode));
8014 }
8015 
8016 void Assembler::movq(Register dst, Address src) {
8017   InstructionMark im(this);
8018   prefixq(src, dst);
8019   emit_int8((unsigned char)0x8B);
8020   emit_operand(dst, src);
8021 }
8022 
8023 void Assembler::movq(Address dst, Register src) {
8024   InstructionMark im(this);
8025   prefixq(dst, src);
8026   emit_int8((unsigned char)0x89);
8027   emit_operand(src, dst);
8028 }
8029 
8030 void Assembler::movsbq(Register dst, Address src) {
8031   InstructionMark im(this);
8032   prefixq(src, dst);
8033   emit_int8(0x0F);
8034   emit_int8((unsigned char)0xBE);
8035   emit_operand(dst, src);
8036 }
8037 
8038 void Assembler::movsbq(Register dst, Register src) {
8039   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
8040   emit_int8(0x0F);
8041   emit_int8((unsigned char)0xBE);
8042   emit_int8((unsigned char)(0xC0 | encode));
8043 }
8044 
8045 void Assembler::movslq(Register dst, int32_t imm32) {
8046   // dbx shows movslq(rcx, 3) as movq     $0x0000000049000000,(%rbx)
8047   // and movslq(r8, 3); as movl     $0x0000000048000000,(%rbx)
8048   // as a result we shouldn't use until tested at runtime...
8049   ShouldNotReachHere();
8050   InstructionMark im(this);
8051   int encode = prefixq_and_encode(dst->encoding());
8052   emit_int8((unsigned char)(0xC7 | encode));
8053   emit_int32(imm32);
8054 }
8055 
8056 void Assembler::movslq(Address dst, int32_t imm32) {
8057   assert(is_simm32(imm32), "lost bits");
8058   InstructionMark im(this);
8059   prefixq(dst);
8060   emit_int8((unsigned char)0xC7);
8061   emit_operand(rax, dst, 4);
8062   emit_int32(imm32);
8063 }
8064 
8065 void Assembler::movslq(Register dst, Address src) {
8066   InstructionMark im(this);
8067   prefixq(src, dst);
8068   emit_int8(0x63);
8069   emit_operand(dst, src);
8070 }
8071 
8072 void Assembler::movslq(Register dst, Register src) {
8073   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
8074   emit_int8(0x63);
8075   emit_int8((unsigned char)(0xC0 | encode));
8076 }
8077 
8078 void Assembler::movswq(Register dst, Address src) {
8079   InstructionMark im(this);
8080   prefixq(src, dst);
8081   emit_int8(0x0F);
8082   emit_int8((unsigned char)0xBF);
8083   emit_operand(dst, src);
8084 }
8085 
8086 void Assembler::movswq(Register dst, Register src) {
8087   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
8088   emit_int8((unsigned char)0x0F);
8089   emit_int8((unsigned char)0xBF);
8090   emit_int8((unsigned char)(0xC0 | encode));
8091 }
8092 
8093 void Assembler::movzbq(Register dst, Address src) {
8094   InstructionMark im(this);
8095   prefixq(src, dst);
8096   emit_int8((unsigned char)0x0F);
8097   emit_int8((unsigned char)0xB6);
8098   emit_operand(dst, src);
8099 }
8100 
8101 void Assembler::movzbq(Register dst, Register src) {
8102   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
8103   emit_int8(0x0F);
8104   emit_int8((unsigned char)0xB6);
8105   emit_int8(0xC0 | encode);
8106 }
8107 
8108 void Assembler::movzwq(Register dst, Address src) {
8109   InstructionMark im(this);
8110   prefixq(src, dst);
8111   emit_int8((unsigned char)0x0F);
8112   emit_int8((unsigned char)0xB7);
8113   emit_operand(dst, src);
8114 }
8115 
8116 void Assembler::movzwq(Register dst, Register src) {
8117   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
8118   emit_int8((unsigned char)0x0F);
8119   emit_int8((unsigned char)0xB7);
8120   emit_int8((unsigned char)(0xC0 | encode));
8121 }
8122 
8123 void Assembler::mulq(Address src) {
8124   InstructionMark im(this);
8125   prefixq(src);
8126   emit_int8((unsigned char)0xF7);
8127   emit_operand(rsp, src);
8128 }
8129 
8130 void Assembler::mulq(Register src) {
8131   int encode = prefixq_and_encode(src->encoding());
8132   emit_int8((unsigned char)0xF7);
8133   emit_int8((unsigned char)(0xE0 | encode));
8134 }
8135 
8136 void Assembler::mulxq(Register dst1, Register dst2, Register src) {
8137   assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
8138   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
8139   int encode = vex_prefix_and_encode(dst1->encoding(), dst2->encoding(), src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_38, &attributes);
8140   emit_int8((unsigned char)0xF6);
8141   emit_int8((unsigned char)(0xC0 | encode));
8142 }
8143 
8144 void Assembler::negq(Register dst) {
8145   int encode = prefixq_and_encode(dst->encoding());
8146   emit_int8((unsigned char)0xF7);
8147   emit_int8((unsigned char)(0xD8 | encode));
8148 }
8149 
8150 void Assembler::notq(Register dst) {
8151   int encode = prefixq_and_encode(dst->encoding());
8152   emit_int8((unsigned char)0xF7);
8153   emit_int8((unsigned char)(0xD0 | encode));
8154 }
8155 
8156 void Assembler::orq(Address dst, int32_t imm32) {
8157   InstructionMark im(this);
8158   prefixq(dst);
8159   emit_int8((unsigned char)0x81);
8160   emit_operand(rcx, dst, 4);
8161   emit_int32(imm32);
8162 }
8163 
8164 void Assembler::orq(Register dst, int32_t imm32) {
8165   (void) prefixq_and_encode(dst->encoding());
8166   emit_arith(0x81, 0xC8, dst, imm32);
8167 }
8168 
8169 void Assembler::orq(Register dst, Address src) {
8170   InstructionMark im(this);
8171   prefixq(src, dst);
8172   emit_int8(0x0B);
8173   emit_operand(dst, src);
8174 }
8175 
8176 void Assembler::orq(Register dst, Register src) {
8177   (void) prefixq_and_encode(dst->encoding(), src->encoding());
8178   emit_arith(0x0B, 0xC0, dst, src);
8179 }
8180 
8181 void Assembler::popa() { // 64bit
8182   movq(r15, Address(rsp, 0));
8183   movq(r14, Address(rsp, wordSize));
8184   movq(r13, Address(rsp, 2 * wordSize));
8185   movq(r12, Address(rsp, 3 * wordSize));
8186   movq(r11, Address(rsp, 4 * wordSize));
8187   movq(r10, Address(rsp, 5 * wordSize));
8188   movq(r9,  Address(rsp, 6 * wordSize));
8189   movq(r8,  Address(rsp, 7 * wordSize));
8190   movq(rdi, Address(rsp, 8 * wordSize));
8191   movq(rsi, Address(rsp, 9 * wordSize));
8192   movq(rbp, Address(rsp, 10 * wordSize));
8193   // skip rsp
8194   movq(rbx, Address(rsp, 12 * wordSize));
8195   movq(rdx, Address(rsp, 13 * wordSize));
8196   movq(rcx, Address(rsp, 14 * wordSize));
8197   movq(rax, Address(rsp, 15 * wordSize));
8198 
8199   addq(rsp, 16 * wordSize);
8200 }
8201 
8202 void Assembler::popcntq(Register dst, Address src) {
8203   assert(VM_Version::supports_popcnt(), "must support");
8204   InstructionMark im(this);
8205   emit_int8((unsigned char)0xF3);
8206   prefixq(src, dst);
8207   emit_int8((unsigned char)0x0F);
8208   emit_int8((unsigned char)0xB8);
8209   emit_operand(dst, src);
8210 }
8211 
8212 void Assembler::popcntq(Register dst, Register src) {
8213   assert(VM_Version::supports_popcnt(), "must support");
8214   emit_int8((unsigned char)0xF3);
8215   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
8216   emit_int8((unsigned char)0x0F);
8217   emit_int8((unsigned char)0xB8);
8218   emit_int8((unsigned char)(0xC0 | encode));
8219 }
8220 
8221 void Assembler::popq(Address dst) {
8222   InstructionMark im(this);
8223   prefixq(dst);
8224   emit_int8((unsigned char)0x8F);
8225   emit_operand(rax, dst);
8226 }
8227 
8228 void Assembler::pusha() { // 64bit
8229   // we have to store original rsp.  ABI says that 128 bytes
8230   // below rsp are local scratch.
8231   movq(Address(rsp, -5 * wordSize), rsp);
8232 
8233   subq(rsp, 16 * wordSize);
8234 
8235   movq(Address(rsp, 15 * wordSize), rax);
8236   movq(Address(rsp, 14 * wordSize), rcx);
8237   movq(Address(rsp, 13 * wordSize), rdx);
8238   movq(Address(rsp, 12 * wordSize), rbx);
8239   // skip rsp
8240   movq(Address(rsp, 10 * wordSize), rbp);
8241   movq(Address(rsp, 9 * wordSize), rsi);
8242   movq(Address(rsp, 8 * wordSize), rdi);
8243   movq(Address(rsp, 7 * wordSize), r8);
8244   movq(Address(rsp, 6 * wordSize), r9);
8245   movq(Address(rsp, 5 * wordSize), r10);
8246   movq(Address(rsp, 4 * wordSize), r11);
8247   movq(Address(rsp, 3 * wordSize), r12);
8248   movq(Address(rsp, 2 * wordSize), r13);
8249   movq(Address(rsp, wordSize), r14);
8250   movq(Address(rsp, 0), r15);
8251 }
8252 
8253 void Assembler::pushq(Address src) {
8254   InstructionMark im(this);
8255   prefixq(src);
8256   emit_int8((unsigned char)0xFF);
8257   emit_operand(rsi, src);
8258 }
8259 
8260 void Assembler::rclq(Register dst, int imm8) {
8261   assert(isShiftCount(imm8 >> 1), "illegal shift count");
8262   int encode = prefixq_and_encode(dst->encoding());
8263   if (imm8 == 1) {
8264     emit_int8((unsigned char)0xD1);
8265     emit_int8((unsigned char)(0xD0 | encode));
8266   } else {
8267     emit_int8((unsigned char)0xC1);
8268     emit_int8((unsigned char)(0xD0 | encode));
8269     emit_int8(imm8);
8270   }
8271 }
8272 
8273 void Assembler::rcrq(Register dst, int imm8) {
8274   assert(isShiftCount(imm8 >> 1), "illegal shift count");
8275   int encode = prefixq_and_encode(dst->encoding());
8276   if (imm8 == 1) {
8277     emit_int8((unsigned char)0xD1);
8278     emit_int8((unsigned char)(0xD8 | encode));
8279   } else {
8280     emit_int8((unsigned char)0xC1);
8281     emit_int8((unsigned char)(0xD8 | encode));
8282     emit_int8(imm8);
8283   }
8284 }
8285 
8286 void Assembler::rorq(Register dst, int imm8) {
8287   assert(isShiftCount(imm8 >> 1), "illegal shift count");
8288   int encode = prefixq_and_encode(dst->encoding());
8289   if (imm8 == 1) {
8290     emit_int8((unsigned char)0xD1);
8291     emit_int8((unsigned char)(0xC8 | encode));
8292   } else {
8293     emit_int8((unsigned char)0xC1);
8294     emit_int8((unsigned char)(0xc8 | encode));
8295     emit_int8(imm8);
8296   }
8297 }
8298 
8299 void Assembler::rorxq(Register dst, Register src, int imm8) {
8300   assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
8301   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
8302   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_3A, &attributes);
8303   emit_int8((unsigned char)0xF0);
8304   emit_int8((unsigned char)(0xC0 | encode));
8305   emit_int8(imm8);
8306 }
8307 
8308 void Assembler::sarq(Register dst, int imm8) {
8309   assert(isShiftCount(imm8 >> 1), "illegal shift count");
8310   int encode = prefixq_and_encode(dst->encoding());
8311   if (imm8 == 1) {
8312     emit_int8((unsigned char)0xD1);
8313     emit_int8((unsigned char)(0xF8 | encode));
8314   } else {
8315     emit_int8((unsigned char)0xC1);
8316     emit_int8((unsigned char)(0xF8 | encode));
8317     emit_int8(imm8);
8318   }
8319 }
8320 
8321 void Assembler::sarq(Register dst) {
8322   int encode = prefixq_and_encode(dst->encoding());
8323   emit_int8((unsigned char)0xD3);
8324   emit_int8((unsigned char)(0xF8 | encode));
8325 }
8326 
8327 void Assembler::sbbq(Address dst, int32_t imm32) {
8328   InstructionMark im(this);
8329   prefixq(dst);
8330   emit_arith_operand(0x81, rbx, dst, imm32);
8331 }
8332 
8333 void Assembler::sbbq(Register dst, int32_t imm32) {
8334   (void) prefixq_and_encode(dst->encoding());
8335   emit_arith(0x81, 0xD8, dst, imm32);
8336 }
8337 
8338 void Assembler::sbbq(Register dst, Address src) {
8339   InstructionMark im(this);
8340   prefixq(src, dst);
8341   emit_int8(0x1B);
8342   emit_operand(dst, src);
8343 }
8344 
8345 void Assembler::sbbq(Register dst, Register src) {
8346   (void) prefixq_and_encode(dst->encoding(), src->encoding());
8347   emit_arith(0x1B, 0xC0, dst, src);
8348 }
8349 
8350 void Assembler::shlq(Register dst, int imm8) {
8351   assert(isShiftCount(imm8 >> 1), "illegal shift count");
8352   int encode = prefixq_and_encode(dst->encoding());
8353   if (imm8 == 1) {
8354     emit_int8((unsigned char)0xD1);
8355     emit_int8((unsigned char)(0xE0 | encode));
8356   } else {
8357     emit_int8((unsigned char)0xC1);
8358     emit_int8((unsigned char)(0xE0 | encode));
8359     emit_int8(imm8);
8360   }
8361 }
8362 
8363 void Assembler::shlq(Register dst) {
8364   int encode = prefixq_and_encode(dst->encoding());
8365   emit_int8((unsigned char)0xD3);
8366   emit_int8((unsigned char)(0xE0 | encode));
8367 }
8368 
8369 void Assembler::shrq(Register dst, int imm8) {
8370   assert(isShiftCount(imm8 >> 1), "illegal shift count");
8371   int encode = prefixq_and_encode(dst->encoding());
8372   emit_int8((unsigned char)0xC1);
8373   emit_int8((unsigned char)(0xE8 | encode));
8374   emit_int8(imm8);
8375 }
8376 
8377 void Assembler::shrq(Register dst) {
8378   int encode = prefixq_and_encode(dst->encoding());
8379   emit_int8((unsigned char)0xD3);
8380   emit_int8(0xE8 | encode);
8381 }
8382 
8383 void Assembler::subq(Address dst, int32_t imm32) {
8384   InstructionMark im(this);
8385   prefixq(dst);
8386   emit_arith_operand(0x81, rbp, dst, imm32);
8387 }
8388 
8389 void Assembler::subq(Address dst, Register src) {
8390   InstructionMark im(this);
8391   prefixq(dst, src);
8392   emit_int8(0x29);
8393   emit_operand(src, dst);
8394 }
8395 
8396 void Assembler::subq(Register dst, int32_t imm32) {
8397   (void) prefixq_and_encode(dst->encoding());
8398   emit_arith(0x81, 0xE8, dst, imm32);
8399 }
8400 
8401 // Force generation of a 4 byte immediate value even if it fits into 8bit
8402 void Assembler::subq_imm32(Register dst, int32_t imm32) {
8403   (void) prefixq_and_encode(dst->encoding());
8404   emit_arith_imm32(0x81, 0xE8, dst, imm32);
8405 }
8406 
8407 void Assembler::subq(Register dst, Address src) {
8408   InstructionMark im(this);
8409   prefixq(src, dst);
8410   emit_int8(0x2B);
8411   emit_operand(dst, src);
8412 }
8413 
8414 void Assembler::subq(Register dst, Register src) {
8415   (void) prefixq_and_encode(dst->encoding(), src->encoding());
8416   emit_arith(0x2B, 0xC0, dst, src);
8417 }
8418 
8419 void Assembler::testq(Register dst, int32_t imm32) {
8420   // not using emit_arith because test
8421   // doesn't support sign-extension of
8422   // 8bit operands
8423   int encode = dst->encoding();
8424   if (encode == 0) {
8425     prefix(REX_W);
8426     emit_int8((unsigned char)0xA9);
8427   } else {
8428     encode = prefixq_and_encode(encode);
8429     emit_int8((unsigned char)0xF7);
8430     emit_int8((unsigned char)(0xC0 | encode));
8431   }
8432   emit_int32(imm32);
8433 }
8434 
8435 void Assembler::testq(Register dst, Register src) {
8436   (void) prefixq_and_encode(dst->encoding(), src->encoding());
8437   emit_arith(0x85, 0xC0, dst, src);
8438 }
8439 
8440 void Assembler::xaddq(Address dst, Register src) {
8441   InstructionMark im(this);
8442   prefixq(dst, src);
8443   emit_int8(0x0F);
8444   emit_int8((unsigned char)0xC1);
8445   emit_operand(src, dst);
8446 }
8447 
8448 void Assembler::xchgq(Register dst, Address src) {
8449   InstructionMark im(this);
8450   prefixq(src, dst);
8451   emit_int8((unsigned char)0x87);
8452   emit_operand(dst, src);
8453 }
8454 
8455 void Assembler::xchgq(Register dst, Register src) {
8456   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
8457   emit_int8((unsigned char)0x87);
8458   emit_int8((unsigned char)(0xc0 | encode));
8459 }
8460 
8461 void Assembler::xorq(Register dst, Register src) {
8462   (void) prefixq_and_encode(dst->encoding(), src->encoding());
8463   emit_arith(0x33, 0xC0, dst, src);
8464 }
8465 
8466 void Assembler::xorq(Register dst, Address src) {
8467   InstructionMark im(this);
8468   prefixq(src, dst);
8469   emit_int8(0x33);
8470   emit_operand(dst, src);
8471 }
8472 
8473 #endif // !LP64