1 /*
   2  * Copyright (c) 1997, 2018, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #include "precompiled.hpp"
  26 #include "asm/assembler.hpp"
  27 #include "asm/assembler.inline.hpp"
  28 #include "gc/shared/cardTableModRefBS.hpp"
  29 #include "gc/shared/collectedHeap.inline.hpp"
  30 #include "interpreter/interpreter.hpp"
  31 #include "memory/resourceArea.hpp"
  32 #include "prims/methodHandles.hpp"
  33 #include "runtime/biasedLocking.hpp"
  34 #include "runtime/interfaceSupport.hpp"
  35 #include "runtime/objectMonitor.hpp"
  36 #include "runtime/os.hpp"
  37 #include "runtime/sharedRuntime.hpp"
  38 #include "runtime/stubRoutines.hpp"
  39 #include "utilities/macros.hpp"
  40 #if INCLUDE_ALL_GCS
  41 #include "gc/g1/g1CollectedHeap.inline.hpp"
  42 #include "gc/g1/g1SATBCardTableModRefBS.hpp"
  43 #include "gc/g1/heapRegion.hpp"
  44 #endif // INCLUDE_ALL_GCS
  45 
  46 #ifdef PRODUCT
  47 #define BLOCK_COMMENT(str) /* nothing */
  48 #define STOP(error) stop(error)
  49 #else
  50 #define BLOCK_COMMENT(str) block_comment(str)
  51 #define STOP(error) block_comment(error); stop(error)
  52 #endif
  53 
  54 #define BIND(label) bind(label); BLOCK_COMMENT(#label ":")
  55 // Implementation of AddressLiteral
  56 
  57 // A 2-D table for managing compressed displacement(disp8) on EVEX enabled platforms.
  58 unsigned char tuple_table[Assembler::EVEX_ETUP + 1][Assembler::AVX_512bit + 1] = {
  59   // -----------------Table 4.5 -------------------- //
  60   16, 32, 64,  // EVEX_FV(0)
  61   4,  4,  4,   // EVEX_FV(1) - with Evex.b
  62   16, 32, 64,  // EVEX_FV(2) - with Evex.w
  63   8,  8,  8,   // EVEX_FV(3) - with Evex.w and Evex.b
  64   8,  16, 32,  // EVEX_HV(0)
  65   4,  4,  4,   // EVEX_HV(1) - with Evex.b
  66   // -----------------Table 4.6 -------------------- //
  67   16, 32, 64,  // EVEX_FVM(0)
  68   1,  1,  1,   // EVEX_T1S(0)
  69   2,  2,  2,   // EVEX_T1S(1)
  70   4,  4,  4,   // EVEX_T1S(2)
  71   8,  8,  8,   // EVEX_T1S(3)
  72   4,  4,  4,   // EVEX_T1F(0)
  73   8,  8,  8,   // EVEX_T1F(1)
  74   8,  8,  8,   // EVEX_T2(0)
  75   0,  16, 16,  // EVEX_T2(1)
  76   0,  16, 16,  // EVEX_T4(0)
  77   0,  0,  32,  // EVEX_T4(1)
  78   0,  0,  32,  // EVEX_T8(0)
  79   8,  16, 32,  // EVEX_HVM(0)
  80   4,  8,  16,  // EVEX_QVM(0)
  81   2,  4,  8,   // EVEX_OVM(0)
  82   16, 16, 16,  // EVEX_M128(0)
  83   8,  32, 64,  // EVEX_DUP(0)
  84   0,  0,  0    // EVEX_NTUP
  85 };
  86 
  87 AddressLiteral::AddressLiteral(address target, relocInfo::relocType rtype) {
  88   _is_lval = false;
  89   _target = target;
  90   switch (rtype) {
  91   case relocInfo::oop_type:
  92   case relocInfo::metadata_type:
  93     // Oops are a special case. Normally they would be their own section
  94     // but in cases like icBuffer they are literals in the code stream that
  95     // we don't have a section for. We use none so that we get a literal address
  96     // which is always patchable.
  97     break;
  98   case relocInfo::external_word_type:
  99     _rspec = external_word_Relocation::spec(target);
 100     break;
 101   case relocInfo::internal_word_type:
 102     _rspec = internal_word_Relocation::spec(target);
 103     break;
 104   case relocInfo::opt_virtual_call_type:
 105     _rspec = opt_virtual_call_Relocation::spec();
 106     break;
 107   case relocInfo::static_call_type:
 108     _rspec = static_call_Relocation::spec();
 109     break;
 110   case relocInfo::runtime_call_type:
 111     _rspec = runtime_call_Relocation::spec();
 112     break;
 113   case relocInfo::poll_type:
 114   case relocInfo::poll_return_type:
 115     _rspec = Relocation::spec_simple(rtype);
 116     break;
 117   case relocInfo::none:
 118     break;
 119   default:
 120     ShouldNotReachHere();
 121     break;
 122   }
 123 }
 124 
 125 // Implementation of Address
 126 
 127 #ifdef _LP64
 128 
 129 Address Address::make_array(ArrayAddress adr) {
 130   // Not implementable on 64bit machines
 131   // Should have been handled higher up the call chain.
 132   ShouldNotReachHere();
 133   return Address();
 134 }
 135 
 136 // exceedingly dangerous constructor
 137 Address::Address(int disp, address loc, relocInfo::relocType rtype) {
 138   _base  = noreg;
 139   _index = noreg;
 140   _scale = no_scale;
 141   _disp  = disp;
 142   switch (rtype) {
 143     case relocInfo::external_word_type:
 144       _rspec = external_word_Relocation::spec(loc);
 145       break;
 146     case relocInfo::internal_word_type:
 147       _rspec = internal_word_Relocation::spec(loc);
 148       break;
 149     case relocInfo::runtime_call_type:
 150       // HMM
 151       _rspec = runtime_call_Relocation::spec();
 152       break;
 153     case relocInfo::poll_type:
 154     case relocInfo::poll_return_type:
 155       _rspec = Relocation::spec_simple(rtype);
 156       break;
 157     case relocInfo::none:
 158       break;
 159     default:
 160       ShouldNotReachHere();
 161   }
 162 }
 163 #else // LP64
 164 
 165 Address Address::make_array(ArrayAddress adr) {
 166   AddressLiteral base = adr.base();
 167   Address index = adr.index();
 168   assert(index._disp == 0, "must not have disp"); // maybe it can?
 169   Address array(index._base, index._index, index._scale, (intptr_t) base.target());
 170   array._rspec = base._rspec;
 171   return array;
 172 }
 173 
 174 // exceedingly dangerous constructor
 175 Address::Address(address loc, RelocationHolder spec) {
 176   _base  = noreg;
 177   _index = noreg;
 178   _scale = no_scale;
 179   _disp  = (intptr_t) loc;
 180   _rspec = spec;
 181 }
 182 
 183 #endif // _LP64
 184 
 185 
 186 
 187 // Convert the raw encoding form into the form expected by the constructor for
 188 // Address.  An index of 4 (rsp) corresponds to having no index, so convert
 189 // that to noreg for the Address constructor.
 190 Address Address::make_raw(int base, int index, int scale, int disp, relocInfo::relocType disp_reloc) {
 191   RelocationHolder rspec;
 192   if (disp_reloc != relocInfo::none) {
 193     rspec = Relocation::spec_simple(disp_reloc);
 194   }
 195   bool valid_index = index != rsp->encoding();
 196   if (valid_index) {
 197     Address madr(as_Register(base), as_Register(index), (Address::ScaleFactor)scale, in_ByteSize(disp));
 198     madr._rspec = rspec;
 199     return madr;
 200   } else {
 201     Address madr(as_Register(base), noreg, Address::no_scale, in_ByteSize(disp));
 202     madr._rspec = rspec;
 203     return madr;
 204   }
 205 }
 206 
 207 // Implementation of Assembler
 208 
 209 int AbstractAssembler::code_fill_byte() {
 210   return (u_char)'\xF4'; // hlt
 211 }
 212 
 213 // make this go away someday
 214 void Assembler::emit_data(jint data, relocInfo::relocType rtype, int format) {
 215   if (rtype == relocInfo::none)
 216     emit_int32(data);
 217   else
 218     emit_data(data, Relocation::spec_simple(rtype), format);
 219 }
 220 
 221 void Assembler::emit_data(jint data, RelocationHolder const& rspec, int format) {
 222   assert(imm_operand == 0, "default format must be immediate in this file");
 223   assert(inst_mark() != NULL, "must be inside InstructionMark");
 224   if (rspec.type() !=  relocInfo::none) {
 225     #ifdef ASSERT
 226       check_relocation(rspec, format);
 227     #endif
 228     // Do not use AbstractAssembler::relocate, which is not intended for
 229     // embedded words.  Instead, relocate to the enclosing instruction.
 230 
 231     // hack. call32 is too wide for mask so use disp32
 232     if (format == call32_operand)
 233       code_section()->relocate(inst_mark(), rspec, disp32_operand);
 234     else
 235       code_section()->relocate(inst_mark(), rspec, format);
 236   }
 237   emit_int32(data);
 238 }
 239 
 240 static int encode(Register r) {
 241   int enc = r->encoding();
 242   if (enc >= 8) {
 243     enc -= 8;
 244   }
 245   return enc;
 246 }
 247 
 248 void Assembler::emit_arith_b(int op1, int op2, Register dst, int imm8) {
 249   assert(dst->has_byte_register(), "must have byte register");
 250   assert(isByte(op1) && isByte(op2), "wrong opcode");
 251   assert(isByte(imm8), "not a byte");
 252   assert((op1 & 0x01) == 0, "should be 8bit operation");
 253   emit_int8(op1);
 254   emit_int8(op2 | encode(dst));
 255   emit_int8(imm8);
 256 }
 257 
 258 
 259 void Assembler::emit_arith(int op1, int op2, Register dst, int32_t imm32) {
 260   assert(isByte(op1) && isByte(op2), "wrong opcode");
 261   assert((op1 & 0x01) == 1, "should be 32bit operation");
 262   assert((op1 & 0x02) == 0, "sign-extension bit should not be set");
 263   if (is8bit(imm32)) {
 264     emit_int8(op1 | 0x02); // set sign bit
 265     emit_int8(op2 | encode(dst));
 266     emit_int8(imm32 & 0xFF);
 267   } else {
 268     emit_int8(op1);
 269     emit_int8(op2 | encode(dst));
 270     emit_int32(imm32);
 271   }
 272 }
 273 
 274 // Force generation of a 4 byte immediate value even if it fits into 8bit
 275 void Assembler::emit_arith_imm32(int op1, int op2, Register dst, int32_t imm32) {
 276   assert(isByte(op1) && isByte(op2), "wrong opcode");
 277   assert((op1 & 0x01) == 1, "should be 32bit operation");
 278   assert((op1 & 0x02) == 0, "sign-extension bit should not be set");
 279   emit_int8(op1);
 280   emit_int8(op2 | encode(dst));
 281   emit_int32(imm32);
 282 }
 283 
 284 // immediate-to-memory forms
 285 void Assembler::emit_arith_operand(int op1, Register rm, Address adr, int32_t imm32) {
 286   assert((op1 & 0x01) == 1, "should be 32bit operation");
 287   assert((op1 & 0x02) == 0, "sign-extension bit should not be set");
 288   if (is8bit(imm32)) {
 289     emit_int8(op1 | 0x02); // set sign bit
 290     emit_operand(rm, adr, 1);
 291     emit_int8(imm32 & 0xFF);
 292   } else {
 293     emit_int8(op1);
 294     emit_operand(rm, adr, 4);
 295     emit_int32(imm32);
 296   }
 297 }
 298 
 299 
 300 void Assembler::emit_arith(int op1, int op2, Register dst, Register src) {
 301   assert(isByte(op1) && isByte(op2), "wrong opcode");
 302   emit_int8(op1);
 303   emit_int8(op2 | encode(dst) << 3 | encode(src));
 304 }
 305 
 306 
 307 bool Assembler::query_compressed_disp_byte(int disp, bool is_evex_inst, int vector_len,
 308                                            int cur_tuple_type, int in_size_in_bits, int cur_encoding) {
 309   int mod_idx = 0;
 310   // We will test if the displacement fits the compressed format and if so
 311   // apply the compression to the displacment iff the result is8bit.
 312   if (VM_Version::supports_evex() && is_evex_inst) {
 313     switch (cur_tuple_type) {
 314     case EVEX_FV:
 315       if ((cur_encoding & VEX_W) == VEX_W) {
 316         mod_idx = ((cur_encoding & EVEX_Rb) == EVEX_Rb) ? 3 : 2;
 317       } else {
 318         mod_idx = ((cur_encoding & EVEX_Rb) == EVEX_Rb) ? 1 : 0;
 319       }
 320       break;
 321 
 322     case EVEX_HV:
 323       mod_idx = ((cur_encoding & EVEX_Rb) == EVEX_Rb) ? 1 : 0;
 324       break;
 325 
 326     case EVEX_FVM:
 327       break;
 328 
 329     case EVEX_T1S:
 330       switch (in_size_in_bits) {
 331       case EVEX_8bit:
 332         break;
 333 
 334       case EVEX_16bit:
 335         mod_idx = 1;
 336         break;
 337 
 338       case EVEX_32bit:
 339         mod_idx = 2;
 340         break;
 341 
 342       case EVEX_64bit:
 343         mod_idx = 3;
 344         break;
 345       }
 346       break;
 347 
 348     case EVEX_T1F:
 349     case EVEX_T2:
 350     case EVEX_T4:
 351       mod_idx = (in_size_in_bits == EVEX_64bit) ? 1 : 0;
 352       break;
 353 
 354     case EVEX_T8:
 355       break;
 356 
 357     case EVEX_HVM:
 358       break;
 359 
 360     case EVEX_QVM:
 361       break;
 362 
 363     case EVEX_OVM:
 364       break;
 365 
 366     case EVEX_M128:
 367       break;
 368 
 369     case EVEX_DUP:
 370       break;
 371 
 372     default:
 373       assert(0, "no valid evex tuple_table entry");
 374       break;
 375     }
 376 
 377     if (vector_len >= AVX_128bit && vector_len <= AVX_512bit) {
 378       int disp_factor = tuple_table[cur_tuple_type + mod_idx][vector_len];
 379       if ((disp % disp_factor) == 0) {
 380         int new_disp = disp / disp_factor;
 381         if ((-0x80 <= new_disp && new_disp < 0x80)) {
 382           disp = new_disp;
 383         }
 384       } else {
 385         return false;
 386       }
 387     }
 388   }
 389   return (-0x80 <= disp && disp < 0x80);
 390 }
 391 
 392 
 393 bool Assembler::emit_compressed_disp_byte(int &disp) {
 394   int mod_idx = 0;
 395   // We will test if the displacement fits the compressed format and if so
 396   // apply the compression to the displacment iff the result is8bit.
 397   if (VM_Version::supports_evex() && _attributes && _attributes->is_evex_instruction()) {
 398     int evex_encoding = _attributes->get_evex_encoding();
 399     int tuple_type = _attributes->get_tuple_type();
 400     switch (tuple_type) {
 401     case EVEX_FV:
 402       if ((evex_encoding & VEX_W) == VEX_W) {
 403         mod_idx = ((evex_encoding & EVEX_Rb) == EVEX_Rb) ? 3 : 2;
 404       } else {
 405         mod_idx = ((evex_encoding & EVEX_Rb) == EVEX_Rb) ? 1 : 0;
 406       }
 407       break;
 408 
 409     case EVEX_HV:
 410       mod_idx = ((evex_encoding & EVEX_Rb) == EVEX_Rb) ? 1 : 0;
 411       break;
 412 
 413     case EVEX_FVM:
 414       break;
 415 
 416     case EVEX_T1S:
 417       switch (_attributes->get_input_size()) {
 418       case EVEX_8bit:
 419         break;
 420 
 421       case EVEX_16bit:
 422         mod_idx = 1;
 423         break;
 424 
 425       case EVEX_32bit:
 426         mod_idx = 2;
 427         break;
 428 
 429       case EVEX_64bit:
 430         mod_idx = 3;
 431         break;
 432       }
 433       break;
 434 
 435     case EVEX_T1F:
 436     case EVEX_T2:
 437     case EVEX_T4:
 438       mod_idx = (_attributes->get_input_size() == EVEX_64bit) ? 1 : 0;
 439       break;
 440 
 441     case EVEX_T8:
 442       break;
 443 
 444     case EVEX_HVM:
 445       break;
 446 
 447     case EVEX_QVM:
 448       break;
 449 
 450     case EVEX_OVM:
 451       break;
 452 
 453     case EVEX_M128:
 454       break;
 455 
 456     case EVEX_DUP:
 457       break;
 458 
 459     default:
 460       assert(0, "no valid evex tuple_table entry");
 461       break;
 462     }
 463 
 464     int vector_len = _attributes->get_vector_len();
 465     if (vector_len >= AVX_128bit && vector_len <= AVX_512bit) {
 466       int disp_factor = tuple_table[tuple_type + mod_idx][vector_len];
 467       if ((disp % disp_factor) == 0) {
 468         int new_disp = disp / disp_factor;
 469         if (is8bit(new_disp)) {
 470           disp = new_disp;
 471         }
 472       } else {
 473         return false;
 474       }
 475     }
 476   }
 477   return is8bit(disp);
 478 }
 479 
 480 
 481 void Assembler::emit_operand(Register reg, Register base, Register index,
 482                              Address::ScaleFactor scale, int disp,
 483                              RelocationHolder const& rspec,
 484                              int rip_relative_correction) {
 485   relocInfo::relocType rtype = (relocInfo::relocType) rspec.type();
 486 
 487   // Encode the registers as needed in the fields they are used in
 488 
 489   int regenc = encode(reg) << 3;
 490   int indexenc = index->is_valid() ? encode(index) << 3 : 0;
 491   int baseenc = base->is_valid() ? encode(base) : 0;
 492 
 493   if (base->is_valid()) {
 494     if (index->is_valid()) {
 495       assert(scale != Address::no_scale, "inconsistent address");
 496       // [base + index*scale + disp]
 497       if (disp == 0 && rtype == relocInfo::none  &&
 498           base != rbp LP64_ONLY(&& base != r13)) {
 499         // [base + index*scale]
 500         // [00 reg 100][ss index base]
 501         assert(index != rsp, "illegal addressing mode");
 502         emit_int8(0x04 | regenc);
 503         emit_int8(scale << 6 | indexenc | baseenc);
 504       } else if (emit_compressed_disp_byte(disp) && rtype == relocInfo::none) {
 505         // [base + index*scale + imm8]
 506         // [01 reg 100][ss index base] imm8
 507         assert(index != rsp, "illegal addressing mode");
 508         emit_int8(0x44 | regenc);
 509         emit_int8(scale << 6 | indexenc | baseenc);
 510         emit_int8(disp & 0xFF);
 511       } else {
 512         // [base + index*scale + disp32]
 513         // [10 reg 100][ss index base] disp32
 514         assert(index != rsp, "illegal addressing mode");
 515         emit_int8(0x84 | regenc);
 516         emit_int8(scale << 6 | indexenc | baseenc);
 517         emit_data(disp, rspec, disp32_operand);
 518       }
 519     } else if (base == rsp LP64_ONLY(|| base == r12)) {
 520       // [rsp + disp]
 521       if (disp == 0 && rtype == relocInfo::none) {
 522         // [rsp]
 523         // [00 reg 100][00 100 100]
 524         emit_int8(0x04 | regenc);
 525         emit_int8(0x24);
 526       } else if (emit_compressed_disp_byte(disp) && rtype == relocInfo::none) {
 527         // [rsp + imm8]
 528         // [01 reg 100][00 100 100] disp8
 529         emit_int8(0x44 | regenc);
 530         emit_int8(0x24);
 531         emit_int8(disp & 0xFF);
 532       } else {
 533         // [rsp + imm32]
 534         // [10 reg 100][00 100 100] disp32
 535         emit_int8(0x84 | regenc);
 536         emit_int8(0x24);
 537         emit_data(disp, rspec, disp32_operand);
 538       }
 539     } else {
 540       // [base + disp]
 541       assert(base != rsp LP64_ONLY(&& base != r12), "illegal addressing mode");
 542       if (disp == 0 && rtype == relocInfo::none &&
 543           base != rbp LP64_ONLY(&& base != r13)) {
 544         // [base]
 545         // [00 reg base]
 546         emit_int8(0x00 | regenc | baseenc);
 547       } else if (emit_compressed_disp_byte(disp) && rtype == relocInfo::none) {
 548         // [base + disp8]
 549         // [01 reg base] disp8
 550         emit_int8(0x40 | regenc | baseenc);
 551         emit_int8(disp & 0xFF);
 552       } else {
 553         // [base + disp32]
 554         // [10 reg base] disp32
 555         emit_int8(0x80 | regenc | baseenc);
 556         emit_data(disp, rspec, disp32_operand);
 557       }
 558     }
 559   } else {
 560     if (index->is_valid()) {
 561       assert(scale != Address::no_scale, "inconsistent address");
 562       // [index*scale + disp]
 563       // [00 reg 100][ss index 101] disp32
 564       assert(index != rsp, "illegal addressing mode");
 565       emit_int8(0x04 | regenc);
 566       emit_int8(scale << 6 | indexenc | 0x05);
 567       emit_data(disp, rspec, disp32_operand);
 568     } else if (rtype != relocInfo::none ) {
 569       // [disp] (64bit) RIP-RELATIVE (32bit) abs
 570       // [00 000 101] disp32
 571 
 572       emit_int8(0x05 | regenc);
 573       // Note that the RIP-rel. correction applies to the generated
 574       // disp field, but _not_ to the target address in the rspec.
 575 
 576       // disp was created by converting the target address minus the pc
 577       // at the start of the instruction. That needs more correction here.
 578       // intptr_t disp = target - next_ip;
 579       assert(inst_mark() != NULL, "must be inside InstructionMark");
 580       address next_ip = pc() + sizeof(int32_t) + rip_relative_correction;
 581       int64_t adjusted = disp;
 582       // Do rip-rel adjustment for 64bit
 583       LP64_ONLY(adjusted -=  (next_ip - inst_mark()));
 584       assert(is_simm32(adjusted),
 585              "must be 32bit offset (RIP relative address)");
 586       emit_data((int32_t) adjusted, rspec, disp32_operand);
 587 
 588     } else {
 589       // 32bit never did this, did everything as the rip-rel/disp code above
 590       // [disp] ABSOLUTE
 591       // [00 reg 100][00 100 101] disp32
 592       emit_int8(0x04 | regenc);
 593       emit_int8(0x25);
 594       emit_data(disp, rspec, disp32_operand);
 595     }
 596   }
 597 }
 598 
 599 void Assembler::emit_operand(XMMRegister reg, Register base, Register index,
 600                              Address::ScaleFactor scale, int disp,
 601                              RelocationHolder const& rspec) {
 602   if (UseAVX > 2) {
 603     int xreg_enc = reg->encoding();
 604     if (xreg_enc > 15) {
 605       XMMRegister new_reg = as_XMMRegister(xreg_enc & 0xf);
 606       emit_operand((Register)new_reg, base, index, scale, disp, rspec);
 607       return;
 608     }
 609   }
 610   emit_operand((Register)reg, base, index, scale, disp, rspec);
 611 }
 612 
 613 // Secret local extension to Assembler::WhichOperand:
 614 #define end_pc_operand (_WhichOperand_limit)
 615 
 616 address Assembler::locate_operand(address inst, WhichOperand which) {
 617   // Decode the given instruction, and return the address of
 618   // an embedded 32-bit operand word.
 619 
 620   // If "which" is disp32_operand, selects the displacement portion
 621   // of an effective address specifier.
 622   // If "which" is imm64_operand, selects the trailing immediate constant.
 623   // If "which" is call32_operand, selects the displacement of a call or jump.
 624   // Caller is responsible for ensuring that there is such an operand,
 625   // and that it is 32/64 bits wide.
 626 
 627   // If "which" is end_pc_operand, find the end of the instruction.
 628 
 629   address ip = inst;
 630   bool is_64bit = false;
 631 
 632   debug_only(bool has_disp32 = false);
 633   int tail_size = 0; // other random bytes (#32, #16, etc.) at end of insn
 634 
 635   again_after_prefix:
 636   switch (0xFF & *ip++) {
 637 
 638   // These convenience macros generate groups of "case" labels for the switch.
 639 #define REP4(x) (x)+0: case (x)+1: case (x)+2: case (x)+3
 640 #define REP8(x) (x)+0: case (x)+1: case (x)+2: case (x)+3: \
 641              case (x)+4: case (x)+5: case (x)+6: case (x)+7
 642 #define REP16(x) REP8((x)+0): \
 643               case REP8((x)+8)
 644 
 645   case CS_segment:
 646   case SS_segment:
 647   case DS_segment:
 648   case ES_segment:
 649   case FS_segment:
 650   case GS_segment:
 651     // Seems dubious
 652     LP64_ONLY(assert(false, "shouldn't have that prefix"));
 653     assert(ip == inst+1, "only one prefix allowed");
 654     goto again_after_prefix;
 655 
 656   case 0x67:
 657   case REX:
 658   case REX_B:
 659   case REX_X:
 660   case REX_XB:
 661   case REX_R:
 662   case REX_RB:
 663   case REX_RX:
 664   case REX_RXB:
 665     NOT_LP64(assert(false, "64bit prefixes"));
 666     goto again_after_prefix;
 667 
 668   case REX_W:
 669   case REX_WB:
 670   case REX_WX:
 671   case REX_WXB:
 672   case REX_WR:
 673   case REX_WRB:
 674   case REX_WRX:
 675   case REX_WRXB:
 676     NOT_LP64(assert(false, "64bit prefixes"));
 677     is_64bit = true;
 678     goto again_after_prefix;
 679 
 680   case 0xFF: // pushq a; decl a; incl a; call a; jmp a
 681   case 0x88: // movb a, r
 682   case 0x89: // movl a, r
 683   case 0x8A: // movb r, a
 684   case 0x8B: // movl r, a
 685   case 0x8F: // popl a
 686     debug_only(has_disp32 = true);
 687     break;
 688 
 689   case 0x68: // pushq #32
 690     if (which == end_pc_operand) {
 691       return ip + 4;
 692     }
 693     assert(which == imm_operand && !is_64bit, "pushl has no disp32 or 64bit immediate");
 694     return ip;                  // not produced by emit_operand
 695 
 696   case 0x66: // movw ... (size prefix)
 697     again_after_size_prefix2:
 698     switch (0xFF & *ip++) {
 699     case REX:
 700     case REX_B:
 701     case REX_X:
 702     case REX_XB:
 703     case REX_R:
 704     case REX_RB:
 705     case REX_RX:
 706     case REX_RXB:
 707     case REX_W:
 708     case REX_WB:
 709     case REX_WX:
 710     case REX_WXB:
 711     case REX_WR:
 712     case REX_WRB:
 713     case REX_WRX:
 714     case REX_WRXB:
 715       NOT_LP64(assert(false, "64bit prefix found"));
 716       goto again_after_size_prefix2;
 717     case 0x8B: // movw r, a
 718     case 0x89: // movw a, r
 719       debug_only(has_disp32 = true);
 720       break;
 721     case 0xC7: // movw a, #16
 722       debug_only(has_disp32 = true);
 723       tail_size = 2;  // the imm16
 724       break;
 725     case 0x0F: // several SSE/SSE2 variants
 726       ip--;    // reparse the 0x0F
 727       goto again_after_prefix;
 728     default:
 729       ShouldNotReachHere();
 730     }
 731     break;
 732 
 733   case REP8(0xB8): // movl/q r, #32/#64(oop?)
 734     if (which == end_pc_operand)  return ip + (is_64bit ? 8 : 4);
 735     // these asserts are somewhat nonsensical
 736 #ifndef _LP64
 737     assert(which == imm_operand || which == disp32_operand,
 738            "which %d is_64_bit %d ip " INTPTR_FORMAT, which, is_64bit, p2i(ip));
 739 #else
 740     assert((which == call32_operand || which == imm_operand) && is_64bit ||
 741            which == narrow_oop_operand && !is_64bit,
 742            "which %d is_64_bit %d ip " INTPTR_FORMAT, which, is_64bit, p2i(ip));
 743 #endif // _LP64
 744     return ip;
 745 
 746   case 0x69: // imul r, a, #32
 747   case 0xC7: // movl a, #32(oop?)
 748     tail_size = 4;
 749     debug_only(has_disp32 = true); // has both kinds of operands!
 750     break;
 751 
 752   case 0x0F: // movx..., etc.
 753     switch (0xFF & *ip++) {
 754     case 0x3A: // pcmpestri
 755       tail_size = 1;
 756     case 0x38: // ptest, pmovzxbw
 757       ip++; // skip opcode
 758       debug_only(has_disp32 = true); // has both kinds of operands!
 759       break;
 760 
 761     case 0x70: // pshufd r, r/a, #8
 762       debug_only(has_disp32 = true); // has both kinds of operands!
 763     case 0x73: // psrldq r, #8
 764       tail_size = 1;
 765       break;
 766 
 767     case 0x12: // movlps
 768     case 0x28: // movaps
 769     case 0x2E: // ucomiss
 770     case 0x2F: // comiss
 771     case 0x54: // andps
 772     case 0x55: // andnps
 773     case 0x56: // orps
 774     case 0x57: // xorps
 775     case 0x58: // addpd
 776     case 0x59: // mulpd
 777     case 0x6E: // movd
 778     case 0x7E: // movd
 779     case 0xAE: // ldmxcsr, stmxcsr, fxrstor, fxsave, clflush
 780     case 0xFE: // paddd
 781       debug_only(has_disp32 = true);
 782       break;
 783 
 784     case 0xAD: // shrd r, a, %cl
 785     case 0xAF: // imul r, a
 786     case 0xBE: // movsbl r, a (movsxb)
 787     case 0xBF: // movswl r, a (movsxw)
 788     case 0xB6: // movzbl r, a (movzxb)
 789     case 0xB7: // movzwl r, a (movzxw)
 790     case REP16(0x40): // cmovl cc, r, a
 791     case 0xB0: // cmpxchgb
 792     case 0xB1: // cmpxchg
 793     case 0xC1: // xaddl
 794     case 0xC7: // cmpxchg8
 795     case REP16(0x90): // setcc a
 796       debug_only(has_disp32 = true);
 797       // fall out of the switch to decode the address
 798       break;
 799 
 800     case 0xC4: // pinsrw r, a, #8
 801       debug_only(has_disp32 = true);
 802     case 0xC5: // pextrw r, r, #8
 803       tail_size = 1;  // the imm8
 804       break;
 805 
 806     case 0xAC: // shrd r, a, #8
 807       debug_only(has_disp32 = true);
 808       tail_size = 1;  // the imm8
 809       break;
 810 
 811     case REP16(0x80): // jcc rdisp32
 812       if (which == end_pc_operand)  return ip + 4;
 813       assert(which == call32_operand, "jcc has no disp32 or imm");
 814       return ip;
 815     default:
 816       ShouldNotReachHere();
 817     }
 818     break;
 819 
 820   case 0x81: // addl a, #32; addl r, #32
 821     // also: orl, adcl, sbbl, andl, subl, xorl, cmpl
 822     // on 32bit in the case of cmpl, the imm might be an oop
 823     tail_size = 4;
 824     debug_only(has_disp32 = true); // has both kinds of operands!
 825     break;
 826 
 827   case 0x83: // addl a, #8; addl r, #8
 828     // also: orl, adcl, sbbl, andl, subl, xorl, cmpl
 829     debug_only(has_disp32 = true); // has both kinds of operands!
 830     tail_size = 1;
 831     break;
 832 
 833   case 0x9B:
 834     switch (0xFF & *ip++) {
 835     case 0xD9: // fnstcw a
 836       debug_only(has_disp32 = true);
 837       break;
 838     default:
 839       ShouldNotReachHere();
 840     }
 841     break;
 842 
 843   case REP4(0x00): // addb a, r; addl a, r; addb r, a; addl r, a
 844   case REP4(0x10): // adc...
 845   case REP4(0x20): // and...
 846   case REP4(0x30): // xor...
 847   case REP4(0x08): // or...
 848   case REP4(0x18): // sbb...
 849   case REP4(0x28): // sub...
 850   case 0xF7: // mull a
 851   case 0x8D: // lea r, a
 852   case 0x87: // xchg r, a
 853   case REP4(0x38): // cmp...
 854   case 0x85: // test r, a
 855     debug_only(has_disp32 = true); // has both kinds of operands!
 856     break;
 857 
 858   case 0xC1: // sal a, #8; sar a, #8; shl a, #8; shr a, #8
 859   case 0xC6: // movb a, #8
 860   case 0x80: // cmpb a, #8
 861   case 0x6B: // imul r, a, #8
 862     debug_only(has_disp32 = true); // has both kinds of operands!
 863     tail_size = 1; // the imm8
 864     break;
 865 
 866   case 0xC4: // VEX_3bytes
 867   case 0xC5: // VEX_2bytes
 868     assert((UseAVX > 0), "shouldn't have VEX prefix");
 869     assert(ip == inst+1, "no prefixes allowed");
 870     // C4 and C5 are also used as opcodes for PINSRW and PEXTRW instructions
 871     // but they have prefix 0x0F and processed when 0x0F processed above.
 872     //
 873     // In 32-bit mode the VEX first byte C4 and C5 alias onto LDS and LES
 874     // instructions (these instructions are not supported in 64-bit mode).
 875     // To distinguish them bits [7:6] are set in the VEX second byte since
 876     // ModRM byte can not be of the form 11xxxxxx in 32-bit mode. To set
 877     // those VEX bits REX and vvvv bits are inverted.
 878     //
 879     // Fortunately C2 doesn't generate these instructions so we don't need
 880     // to check for them in product version.
 881 
 882     // Check second byte
 883     NOT_LP64(assert((0xC0 & *ip) == 0xC0, "shouldn't have LDS and LES instructions"));
 884 
 885     int vex_opcode;
 886     // First byte
 887     if ((0xFF & *inst) == VEX_3bytes) {
 888       vex_opcode = VEX_OPCODE_MASK & *ip;
 889       ip++; // third byte
 890       is_64bit = ((VEX_W & *ip) == VEX_W);
 891     } else {
 892       vex_opcode = VEX_OPCODE_0F;
 893     }
 894     ip++; // opcode
 895     // To find the end of instruction (which == end_pc_operand).
 896     switch (vex_opcode) {
 897       case VEX_OPCODE_0F:
 898         switch (0xFF & *ip) {
 899         case 0x70: // pshufd r, r/a, #8
 900         case 0x71: // ps[rl|ra|ll]w r, #8
 901         case 0x72: // ps[rl|ra|ll]d r, #8
 902         case 0x73: // ps[rl|ra|ll]q r, #8
 903         case 0xC2: // cmp[ps|pd|ss|sd] r, r, r/a, #8
 904         case 0xC4: // pinsrw r, r, r/a, #8
 905         case 0xC5: // pextrw r/a, r, #8
 906         case 0xC6: // shufp[s|d] r, r, r/a, #8
 907           tail_size = 1;  // the imm8
 908           break;
 909         }
 910         break;
 911       case VEX_OPCODE_0F_3A:
 912         tail_size = 1;
 913         break;
 914     }
 915     ip++; // skip opcode
 916     debug_only(has_disp32 = true); // has both kinds of operands!
 917     break;
 918 
 919   case 0x62: // EVEX_4bytes
 920     assert(VM_Version::supports_evex(), "shouldn't have EVEX prefix");
 921     assert(ip == inst+1, "no prefixes allowed");
 922     // no EVEX collisions, all instructions that have 0x62 opcodes
 923     // have EVEX versions and are subopcodes of 0x66
 924     ip++; // skip P0 and exmaine W in P1
 925     is_64bit = ((VEX_W & *ip) == VEX_W);
 926     ip++; // move to P2
 927     ip++; // skip P2, move to opcode
 928     // To find the end of instruction (which == end_pc_operand).
 929     switch (0xFF & *ip) {
 930     case 0x22: // pinsrd r, r/a, #8
 931     case 0x61: // pcmpestri r, r/a, #8
 932     case 0x70: // pshufd r, r/a, #8
 933     case 0x73: // psrldq r, #8
 934       tail_size = 1;  // the imm8
 935       break;
 936     default:
 937       break;
 938     }
 939     ip++; // skip opcode
 940     debug_only(has_disp32 = true); // has both kinds of operands!
 941     break;
 942 
 943   case 0xD1: // sal a, 1; sar a, 1; shl a, 1; shr a, 1
 944   case 0xD3: // sal a, %cl; sar a, %cl; shl a, %cl; shr a, %cl
 945   case 0xD9: // fld_s a; fst_s a; fstp_s a; fldcw a
 946   case 0xDD: // fld_d a; fst_d a; fstp_d a
 947   case 0xDB: // fild_s a; fistp_s a; fld_x a; fstp_x a
 948   case 0xDF: // fild_d a; fistp_d a
 949   case 0xD8: // fadd_s a; fsubr_s a; fmul_s a; fdivr_s a; fcomp_s a
 950   case 0xDC: // fadd_d a; fsubr_d a; fmul_d a; fdivr_d a; fcomp_d a
 951   case 0xDE: // faddp_d a; fsubrp_d a; fmulp_d a; fdivrp_d a; fcompp_d a
 952     debug_only(has_disp32 = true);
 953     break;
 954 
 955   case 0xE8: // call rdisp32
 956   case 0xE9: // jmp  rdisp32
 957     if (which == end_pc_operand)  return ip + 4;
 958     assert(which == call32_operand, "call has no disp32 or imm");
 959     return ip;
 960 
 961   case 0xF0:                    // Lock
 962     assert(os::is_MP(), "only on MP");
 963     goto again_after_prefix;
 964 
 965   case 0xF3:                    // For SSE
 966   case 0xF2:                    // For SSE2
 967     switch (0xFF & *ip++) {
 968     case REX:
 969     case REX_B:
 970     case REX_X:
 971     case REX_XB:
 972     case REX_R:
 973     case REX_RB:
 974     case REX_RX:
 975     case REX_RXB:
 976     case REX_W:
 977     case REX_WB:
 978     case REX_WX:
 979     case REX_WXB:
 980     case REX_WR:
 981     case REX_WRB:
 982     case REX_WRX:
 983     case REX_WRXB:
 984       NOT_LP64(assert(false, "found 64bit prefix"));
 985       ip++;
 986     default:
 987       ip++;
 988     }
 989     debug_only(has_disp32 = true); // has both kinds of operands!
 990     break;
 991 
 992   default:
 993     ShouldNotReachHere();
 994 
 995 #undef REP8
 996 #undef REP16
 997   }
 998 
 999   assert(which != call32_operand, "instruction is not a call, jmp, or jcc");
1000 #ifdef _LP64
1001   assert(which != imm_operand, "instruction is not a movq reg, imm64");
1002 #else
1003   // assert(which != imm_operand || has_imm32, "instruction has no imm32 field");
1004   assert(which != imm_operand || has_disp32, "instruction has no imm32 field");
1005 #endif // LP64
1006   assert(which != disp32_operand || has_disp32, "instruction has no disp32 field");
1007 
1008   // parse the output of emit_operand
1009   int op2 = 0xFF & *ip++;
1010   int base = op2 & 0x07;
1011   int op3 = -1;
1012   const int b100 = 4;
1013   const int b101 = 5;
1014   if (base == b100 && (op2 >> 6) != 3) {
1015     op3 = 0xFF & *ip++;
1016     base = op3 & 0x07;   // refetch the base
1017   }
1018   // now ip points at the disp (if any)
1019 
1020   switch (op2 >> 6) {
1021   case 0:
1022     // [00 reg  100][ss index base]
1023     // [00 reg  100][00   100  esp]
1024     // [00 reg base]
1025     // [00 reg  100][ss index  101][disp32]
1026     // [00 reg  101]               [disp32]
1027 
1028     if (base == b101) {
1029       if (which == disp32_operand)
1030         return ip;              // caller wants the disp32
1031       ip += 4;                  // skip the disp32
1032     }
1033     break;
1034 
1035   case 1:
1036     // [01 reg  100][ss index base][disp8]
1037     // [01 reg  100][00   100  esp][disp8]
1038     // [01 reg base]               [disp8]
1039     ip += 1;                    // skip the disp8
1040     break;
1041 
1042   case 2:
1043     // [10 reg  100][ss index base][disp32]
1044     // [10 reg  100][00   100  esp][disp32]
1045     // [10 reg base]               [disp32]
1046     if (which == disp32_operand)
1047       return ip;                // caller wants the disp32
1048     ip += 4;                    // skip the disp32
1049     break;
1050 
1051   case 3:
1052     // [11 reg base]  (not a memory addressing mode)
1053     break;
1054   }
1055 
1056   if (which == end_pc_operand) {
1057     return ip + tail_size;
1058   }
1059 
1060 #ifdef _LP64
1061   assert(which == narrow_oop_operand && !is_64bit, "instruction is not a movl adr, imm32");
1062 #else
1063   assert(which == imm_operand, "instruction has only an imm field");
1064 #endif // LP64
1065   return ip;
1066 }
1067 
1068 address Assembler::locate_next_instruction(address inst) {
1069   // Secretly share code with locate_operand:
1070   return locate_operand(inst, end_pc_operand);
1071 }
1072 
1073 
1074 #ifdef ASSERT
1075 void Assembler::check_relocation(RelocationHolder const& rspec, int format) {
1076   address inst = inst_mark();
1077   assert(inst != NULL && inst < pc(), "must point to beginning of instruction");
1078   address opnd;
1079 
1080   Relocation* r = rspec.reloc();
1081   if (r->type() == relocInfo::none) {
1082     return;
1083   } else if (r->is_call() || format == call32_operand) {
1084     // assert(format == imm32_operand, "cannot specify a nonzero format");
1085     opnd = locate_operand(inst, call32_operand);
1086   } else if (r->is_data()) {
1087     assert(format == imm_operand || format == disp32_operand
1088            LP64_ONLY(|| format == narrow_oop_operand), "format ok");
1089     opnd = locate_operand(inst, (WhichOperand)format);
1090   } else {
1091     assert(format == imm_operand, "cannot specify a format");
1092     return;
1093   }
1094   assert(opnd == pc(), "must put operand where relocs can find it");
1095 }
1096 #endif // ASSERT
1097 
1098 void Assembler::emit_operand32(Register reg, Address adr) {
1099   assert(reg->encoding() < 8, "no extended registers");
1100   assert(!adr.base_needs_rex() && !adr.index_needs_rex(), "no extended registers");
1101   emit_operand(reg, adr._base, adr._index, adr._scale, adr._disp,
1102                adr._rspec);
1103 }
1104 
1105 void Assembler::emit_operand(Register reg, Address adr,
1106                              int rip_relative_correction) {
1107   emit_operand(reg, adr._base, adr._index, adr._scale, adr._disp,
1108                adr._rspec,
1109                rip_relative_correction);
1110 }
1111 
1112 void Assembler::emit_operand(XMMRegister reg, Address adr) {
1113   emit_operand(reg, adr._base, adr._index, adr._scale, adr._disp,
1114                adr._rspec);
1115 }
1116 
1117 // MMX operations
1118 void Assembler::emit_operand(MMXRegister reg, Address adr) {
1119   assert(!adr.base_needs_rex() && !adr.index_needs_rex(), "no extended registers");
1120   emit_operand((Register)reg, adr._base, adr._index, adr._scale, adr._disp, adr._rspec);
1121 }
1122 
1123 // work around gcc (3.2.1-7a) bug
1124 void Assembler::emit_operand(Address adr, MMXRegister reg) {
1125   assert(!adr.base_needs_rex() && !adr.index_needs_rex(), "no extended registers");
1126   emit_operand((Register)reg, adr._base, adr._index, adr._scale, adr._disp, adr._rspec);
1127 }
1128 
1129 
1130 void Assembler::emit_farith(int b1, int b2, int i) {
1131   assert(isByte(b1) && isByte(b2), "wrong opcode");
1132   assert(0 <= i &&  i < 8, "illegal stack offset");
1133   emit_int8(b1);
1134   emit_int8(b2 + i);
1135 }
1136 
1137 
1138 // Now the Assembler instructions (identical for 32/64 bits)
1139 
1140 void Assembler::adcl(Address dst, int32_t imm32) {
1141   InstructionMark im(this);
1142   prefix(dst);
1143   emit_arith_operand(0x81, rdx, dst, imm32);
1144 }
1145 
1146 void Assembler::adcl(Address dst, Register src) {
1147   InstructionMark im(this);
1148   prefix(dst, src);
1149   emit_int8(0x11);
1150   emit_operand(src, dst);
1151 }
1152 
1153 void Assembler::adcl(Register dst, int32_t imm32) {
1154   prefix(dst);
1155   emit_arith(0x81, 0xD0, dst, imm32);
1156 }
1157 
1158 void Assembler::adcl(Register dst, Address src) {
1159   InstructionMark im(this);
1160   prefix(src, dst);
1161   emit_int8(0x13);
1162   emit_operand(dst, src);
1163 }
1164 
1165 void Assembler::adcl(Register dst, Register src) {
1166   (void) prefix_and_encode(dst->encoding(), src->encoding());
1167   emit_arith(0x13, 0xC0, dst, src);
1168 }
1169 
1170 void Assembler::addl(Address dst, int32_t imm32) {
1171   InstructionMark im(this);
1172   prefix(dst);
1173   emit_arith_operand(0x81, rax, dst, imm32);
1174 }
1175 
1176 void Assembler::addb(Address dst, int imm8) {
1177   InstructionMark im(this);
1178   prefix(dst);
1179   emit_int8((unsigned char)0x80);
1180   emit_operand(rax, dst, 1);
1181   emit_int8(imm8);
1182 }
1183 
1184 void Assembler::addw(Address dst, int imm16) {
1185   InstructionMark im(this);
1186   emit_int8(0x66);
1187   prefix(dst);
1188   emit_int8((unsigned char)0x81);
1189   emit_operand(rax, dst, 2);
1190   emit_int16(imm16);
1191 }
1192 
1193 void Assembler::addl(Address dst, Register src) {
1194   InstructionMark im(this);
1195   prefix(dst, src);
1196   emit_int8(0x01);
1197   emit_operand(src, dst);
1198 }
1199 
1200 void Assembler::addl(Register dst, int32_t imm32) {
1201   prefix(dst);
1202   emit_arith(0x81, 0xC0, dst, imm32);
1203 }
1204 
1205 void Assembler::addl(Register dst, Address src) {
1206   InstructionMark im(this);
1207   prefix(src, dst);
1208   emit_int8(0x03);
1209   emit_operand(dst, src);
1210 }
1211 
1212 void Assembler::addl(Register dst, Register src) {
1213   (void) prefix_and_encode(dst->encoding(), src->encoding());
1214   emit_arith(0x03, 0xC0, dst, src);
1215 }
1216 
1217 void Assembler::addr_nop_4() {
1218   assert(UseAddressNop, "no CPU support");
1219   // 4 bytes: NOP DWORD PTR [EAX+0]
1220   emit_int8(0x0F);
1221   emit_int8(0x1F);
1222   emit_int8(0x40); // emit_rm(cbuf, 0x1, EAX_enc, EAX_enc);
1223   emit_int8(0);    // 8-bits offset (1 byte)
1224 }
1225 
1226 void Assembler::addr_nop_5() {
1227   assert(UseAddressNop, "no CPU support");
1228   // 5 bytes: NOP DWORD PTR [EAX+EAX*0+0] 8-bits offset
1229   emit_int8(0x0F);
1230   emit_int8(0x1F);
1231   emit_int8(0x44); // emit_rm(cbuf, 0x1, EAX_enc, 0x4);
1232   emit_int8(0x00); // emit_rm(cbuf, 0x0, EAX_enc, EAX_enc);
1233   emit_int8(0);    // 8-bits offset (1 byte)
1234 }
1235 
1236 void Assembler::addr_nop_7() {
1237   assert(UseAddressNop, "no CPU support");
1238   // 7 bytes: NOP DWORD PTR [EAX+0] 32-bits offset
1239   emit_int8(0x0F);
1240   emit_int8(0x1F);
1241   emit_int8((unsigned char)0x80);
1242                    // emit_rm(cbuf, 0x2, EAX_enc, EAX_enc);
1243   emit_int32(0);   // 32-bits offset (4 bytes)
1244 }
1245 
1246 void Assembler::addr_nop_8() {
1247   assert(UseAddressNop, "no CPU support");
1248   // 8 bytes: NOP DWORD PTR [EAX+EAX*0+0] 32-bits offset
1249   emit_int8(0x0F);
1250   emit_int8(0x1F);
1251   emit_int8((unsigned char)0x84);
1252                    // emit_rm(cbuf, 0x2, EAX_enc, 0x4);
1253   emit_int8(0x00); // emit_rm(cbuf, 0x0, EAX_enc, EAX_enc);
1254   emit_int32(0);   // 32-bits offset (4 bytes)
1255 }
1256 
1257 void Assembler::addsd(XMMRegister dst, XMMRegister src) {
1258   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1259   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1260   attributes.set_rex_vex_w_reverted();
1261   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
1262   emit_int8(0x58);
1263   emit_int8((unsigned char)(0xC0 | encode));
1264 }
1265 
1266 void Assembler::addsd(XMMRegister dst, Address src) {
1267   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1268   InstructionMark im(this);
1269   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1270   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
1271   attributes.set_rex_vex_w_reverted();
1272   simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
1273   emit_int8(0x58);
1274   emit_operand(dst, src);
1275 }
1276 
1277 void Assembler::addss(XMMRegister dst, XMMRegister src) {
1278   NOT_LP64(assert(VM_Version::supports_sse(), ""));
1279   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1280   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
1281   emit_int8(0x58);
1282   emit_int8((unsigned char)(0xC0 | encode));
1283 }
1284 
1285 void Assembler::addss(XMMRegister dst, Address src) {
1286   NOT_LP64(assert(VM_Version::supports_sse(), ""));
1287   InstructionMark im(this);
1288   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1289   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
1290   simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
1291   emit_int8(0x58);
1292   emit_operand(dst, src);
1293 }
1294 
1295 void Assembler::aesdec(XMMRegister dst, Address src) {
1296   assert(VM_Version::supports_aes(), "");
1297   InstructionMark im(this);
1298   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
1299   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
1300   emit_int8((unsigned char)0xDE);
1301   emit_operand(dst, src);
1302 }
1303 
1304 void Assembler::aesdec(XMMRegister dst, XMMRegister src) {
1305   assert(VM_Version::supports_aes(), "");
1306   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
1307   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
1308   emit_int8((unsigned char)0xDE);
1309   emit_int8(0xC0 | encode);
1310 }
1311 
1312 void Assembler::aesdeclast(XMMRegister dst, Address src) {
1313   assert(VM_Version::supports_aes(), "");
1314   InstructionMark im(this);
1315   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
1316   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
1317   emit_int8((unsigned char)0xDF);
1318   emit_operand(dst, src);
1319 }
1320 
1321 void Assembler::aesdeclast(XMMRegister dst, XMMRegister src) {
1322   assert(VM_Version::supports_aes(), "");
1323   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
1324   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
1325   emit_int8((unsigned char)0xDF);
1326   emit_int8((unsigned char)(0xC0 | encode));
1327 }
1328 
1329 void Assembler::aesenc(XMMRegister dst, Address src) {
1330   assert(VM_Version::supports_aes(), "");
1331   InstructionMark im(this);
1332   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
1333   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
1334   emit_int8((unsigned char)0xDC);
1335   emit_operand(dst, src);
1336 }
1337 
1338 void Assembler::aesenc(XMMRegister dst, XMMRegister src) {
1339   assert(VM_Version::supports_aes(), "");
1340   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
1341   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
1342   emit_int8((unsigned char)0xDC);
1343   emit_int8(0xC0 | encode);
1344 }
1345 
1346 void Assembler::aesenclast(XMMRegister dst, Address src) {
1347   assert(VM_Version::supports_aes(), "");
1348   InstructionMark im(this);
1349   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
1350   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
1351   emit_int8((unsigned char)0xDD);
1352   emit_operand(dst, src);
1353 }
1354 
1355 void Assembler::aesenclast(XMMRegister dst, XMMRegister src) {
1356   assert(VM_Version::supports_aes(), "");
1357   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
1358   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
1359   emit_int8((unsigned char)0xDD);
1360   emit_int8((unsigned char)(0xC0 | encode));
1361 }
1362 
1363 void Assembler::andl(Address dst, int32_t imm32) {
1364   InstructionMark im(this);
1365   prefix(dst);
1366   emit_int8((unsigned char)0x81);
1367   emit_operand(rsp, dst, 4);
1368   emit_int32(imm32);
1369 }
1370 
1371 void Assembler::andl(Register dst, int32_t imm32) {
1372   prefix(dst);
1373   emit_arith(0x81, 0xE0, dst, imm32);
1374 }
1375 
1376 void Assembler::andl(Register dst, Address src) {
1377   InstructionMark im(this);
1378   prefix(src, dst);
1379   emit_int8(0x23);
1380   emit_operand(dst, src);
1381 }
1382 
1383 void Assembler::andl(Register dst, Register src) {
1384   (void) prefix_and_encode(dst->encoding(), src->encoding());
1385   emit_arith(0x23, 0xC0, dst, src);
1386 }
1387 
1388 void Assembler::andnl(Register dst, Register src1, Register src2) {
1389   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1390   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
1391   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
1392   emit_int8((unsigned char)0xF2);
1393   emit_int8((unsigned char)(0xC0 | encode));
1394 }
1395 
1396 void Assembler::andnl(Register dst, Register src1, Address src2) {
1397   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1398   InstructionMark im(this);
1399   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
1400   vex_prefix(src2, src1->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
1401   emit_int8((unsigned char)0xF2);
1402   emit_operand(dst, src2);
1403 }
1404 
1405 void Assembler::bsfl(Register dst, Register src) {
1406   int encode = prefix_and_encode(dst->encoding(), src->encoding());
1407   emit_int8(0x0F);
1408   emit_int8((unsigned char)0xBC);
1409   emit_int8((unsigned char)(0xC0 | encode));
1410 }
1411 
1412 void Assembler::bsrl(Register dst, Register src) {
1413   int encode = prefix_and_encode(dst->encoding(), src->encoding());
1414   emit_int8(0x0F);
1415   emit_int8((unsigned char)0xBD);
1416   emit_int8((unsigned char)(0xC0 | encode));
1417 }
1418 
1419 void Assembler::bswapl(Register reg) { // bswap
1420   int encode = prefix_and_encode(reg->encoding());
1421   emit_int8(0x0F);
1422   emit_int8((unsigned char)(0xC8 | encode));
1423 }
1424 
1425 void Assembler::blsil(Register dst, Register src) {
1426   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1427   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
1428   int encode = vex_prefix_and_encode(rbx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
1429   emit_int8((unsigned char)0xF3);
1430   emit_int8((unsigned char)(0xC0 | encode));
1431 }
1432 
1433 void Assembler::blsil(Register dst, Address src) {
1434   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1435   InstructionMark im(this);
1436   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
1437   vex_prefix(src, dst->encoding(), rbx->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
1438   emit_int8((unsigned char)0xF3);
1439   emit_operand(rbx, src);
1440 }
1441 
1442 void Assembler::blsmskl(Register dst, Register src) {
1443   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1444   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
1445   int encode = vex_prefix_and_encode(rdx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
1446   emit_int8((unsigned char)0xF3);
1447   emit_int8((unsigned char)(0xC0 | encode));
1448 }
1449 
1450 void Assembler::blsmskl(Register dst, Address src) {
1451   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1452   InstructionMark im(this);
1453   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
1454   vex_prefix(src, dst->encoding(), rdx->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
1455   emit_int8((unsigned char)0xF3);
1456   emit_operand(rdx, src);
1457 }
1458 
1459 void Assembler::blsrl(Register dst, Register src) {
1460   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1461   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
1462   int encode = vex_prefix_and_encode(rcx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
1463   emit_int8((unsigned char)0xF3);
1464   emit_int8((unsigned char)(0xC0 | encode));
1465 }
1466 
1467 void Assembler::blsrl(Register dst, Address src) {
1468   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1469   InstructionMark im(this);
1470   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
1471   vex_prefix(src, dst->encoding(), rcx->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
1472   emit_int8((unsigned char)0xF3);
1473   emit_operand(rcx, src);
1474 }
1475 
1476 void Assembler::call(Label& L, relocInfo::relocType rtype) {
1477   // suspect disp32 is always good
1478   int operand = LP64_ONLY(disp32_operand) NOT_LP64(imm_operand);
1479 
1480   if (L.is_bound()) {
1481     const int long_size = 5;
1482     int offs = (int)( target(L) - pc() );
1483     assert(offs <= 0, "assembler error");
1484     InstructionMark im(this);
1485     // 1110 1000 #32-bit disp
1486     emit_int8((unsigned char)0xE8);
1487     emit_data(offs - long_size, rtype, operand);
1488   } else {
1489     InstructionMark im(this);
1490     // 1110 1000 #32-bit disp
1491     L.add_patch_at(code(), locator());
1492 
1493     emit_int8((unsigned char)0xE8);
1494     emit_data(int(0), rtype, operand);
1495   }
1496 }
1497 
1498 void Assembler::call(Register dst) {
1499   int encode = prefix_and_encode(dst->encoding());
1500   emit_int8((unsigned char)0xFF);
1501   emit_int8((unsigned char)(0xD0 | encode));
1502 }
1503 
1504 
1505 void Assembler::call(Address adr) {
1506   InstructionMark im(this);
1507   prefix(adr);
1508   emit_int8((unsigned char)0xFF);
1509   emit_operand(rdx, adr);
1510 }
1511 
1512 void Assembler::call_literal(address entry, RelocationHolder const& rspec) {
1513   assert(entry != NULL, "call most probably wrong");
1514   InstructionMark im(this);
1515   emit_int8((unsigned char)0xE8);
1516   intptr_t disp = entry - (pc() + sizeof(int32_t));
1517   assert(is_simm32(disp), "must be 32bit offset (call2)");
1518   // Technically, should use call32_operand, but this format is
1519   // implied by the fact that we're emitting a call instruction.
1520 
1521   int operand = LP64_ONLY(disp32_operand) NOT_LP64(call32_operand);
1522   emit_data((int) disp, rspec, operand);
1523 }
1524 
1525 void Assembler::cdql() {
1526   emit_int8((unsigned char)0x99);
1527 }
1528 
1529 void Assembler::cld() {
1530   emit_int8((unsigned char)0xFC);
1531 }
1532 
1533 void Assembler::cmovl(Condition cc, Register dst, Register src) {
1534   NOT_LP64(guarantee(VM_Version::supports_cmov(), "illegal instruction"));
1535   int encode = prefix_and_encode(dst->encoding(), src->encoding());
1536   emit_int8(0x0F);
1537   emit_int8(0x40 | cc);
1538   emit_int8((unsigned char)(0xC0 | encode));
1539 }
1540 
1541 
1542 void Assembler::cmovl(Condition cc, Register dst, Address src) {
1543   NOT_LP64(guarantee(VM_Version::supports_cmov(), "illegal instruction"));
1544   prefix(src, dst);
1545   emit_int8(0x0F);
1546   emit_int8(0x40 | cc);
1547   emit_operand(dst, src);
1548 }
1549 
1550 void Assembler::cmpb(Address dst, int imm8) {
1551   InstructionMark im(this);
1552   prefix(dst);
1553   emit_int8((unsigned char)0x80);
1554   emit_operand(rdi, dst, 1);
1555   emit_int8(imm8);
1556 }
1557 
1558 void Assembler::cmpl(Address dst, int32_t imm32) {
1559   InstructionMark im(this);
1560   prefix(dst);
1561   emit_int8((unsigned char)0x81);
1562   emit_operand(rdi, dst, 4);
1563   emit_int32(imm32);
1564 }
1565 
1566 void Assembler::cmpl(Register dst, int32_t imm32) {
1567   prefix(dst);
1568   emit_arith(0x81, 0xF8, dst, imm32);
1569 }
1570 
1571 void Assembler::cmpl(Register dst, Register src) {
1572   (void) prefix_and_encode(dst->encoding(), src->encoding());
1573   emit_arith(0x3B, 0xC0, dst, src);
1574 }
1575 
1576 void Assembler::cmpl(Register dst, Address  src) {
1577   InstructionMark im(this);
1578   prefix(src, dst);
1579   emit_int8((unsigned char)0x3B);
1580   emit_operand(dst, src);
1581 }
1582 
1583 void Assembler::cmpw(Address dst, int imm16) {
1584   InstructionMark im(this);
1585   assert(!dst.base_needs_rex() && !dst.index_needs_rex(), "no extended registers");
1586   emit_int8(0x66);
1587   emit_int8((unsigned char)0x81);
1588   emit_operand(rdi, dst, 2);
1589   emit_int16(imm16);
1590 }
1591 
1592 // The 32-bit cmpxchg compares the value at adr with the contents of rax,
1593 // and stores reg into adr if so; otherwise, the value at adr is loaded into rax,.
1594 // The ZF is set if the compared values were equal, and cleared otherwise.
1595 void Assembler::cmpxchgl(Register reg, Address adr) { // cmpxchg
1596   InstructionMark im(this);
1597   prefix(adr, reg);
1598   emit_int8(0x0F);
1599   emit_int8((unsigned char)0xB1);
1600   emit_operand(reg, adr);
1601 }
1602 
1603 // The 8-bit cmpxchg compares the value at adr with the contents of rax,
1604 // and stores reg into adr if so; otherwise, the value at adr is loaded into rax,.
1605 // The ZF is set if the compared values were equal, and cleared otherwise.
1606 void Assembler::cmpxchgb(Register reg, Address adr) { // cmpxchg
1607   InstructionMark im(this);
1608   prefix(adr, reg, true);
1609   emit_int8(0x0F);
1610   emit_int8((unsigned char)0xB0);
1611   emit_operand(reg, adr);
1612 }
1613 
1614 void Assembler::comisd(XMMRegister dst, Address src) {
1615   // NOTE: dbx seems to decode this as comiss even though the
1616   // 0x66 is there. Strangly ucomisd comes out correct
1617   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1618   InstructionMark im(this);
1619   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);;
1620   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
1621   attributes.set_rex_vex_w_reverted();
1622   simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
1623   emit_int8(0x2F);
1624   emit_operand(dst, src);
1625 }
1626 
1627 void Assembler::comisd(XMMRegister dst, XMMRegister src) {
1628   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1629   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1630   attributes.set_rex_vex_w_reverted();
1631   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
1632   emit_int8(0x2F);
1633   emit_int8((unsigned char)(0xC0 | encode));
1634 }
1635 
1636 void Assembler::comiss(XMMRegister dst, Address src) {
1637   NOT_LP64(assert(VM_Version::supports_sse(), ""));
1638   InstructionMark im(this);
1639   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1640   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
1641   simd_prefix(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
1642   emit_int8(0x2F);
1643   emit_operand(dst, src);
1644 }
1645 
1646 void Assembler::comiss(XMMRegister dst, XMMRegister src) {
1647   NOT_LP64(assert(VM_Version::supports_sse(), ""));
1648   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1649   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
1650   emit_int8(0x2F);
1651   emit_int8((unsigned char)(0xC0 | encode));
1652 }
1653 
1654 void Assembler::cpuid() {
1655   emit_int8(0x0F);
1656   emit_int8((unsigned char)0xA2);
1657 }
1658 
1659 // Opcode / Instruction                      Op /  En  64 - Bit Mode     Compat / Leg Mode Description                  Implemented
1660 // F2 0F 38 F0 / r       CRC32 r32, r / m8   RM        Valid             Valid             Accumulate CRC32 on r / m8.  v
1661 // F2 REX 0F 38 F0 / r   CRC32 r32, r / m8*  RM        Valid             N.E.              Accumulate CRC32 on r / m8.  -
1662 // F2 REX.W 0F 38 F0 / r CRC32 r64, r / m8   RM        Valid             N.E.              Accumulate CRC32 on r / m8.  -
1663 //
1664 // F2 0F 38 F1 / r       CRC32 r32, r / m16  RM        Valid             Valid             Accumulate CRC32 on r / m16. v
1665 //
1666 // F2 0F 38 F1 / r       CRC32 r32, r / m32  RM        Valid             Valid             Accumulate CRC32 on r / m32. v
1667 //
1668 // F2 REX.W 0F 38 F1 / r CRC32 r64, r / m64  RM        Valid             N.E.              Accumulate CRC32 on r / m64. v
1669 void Assembler::crc32(Register crc, Register v, int8_t sizeInBytes) {
1670   assert(VM_Version::supports_sse4_2(), "");
1671   int8_t w = 0x01;
1672   Prefix p = Prefix_EMPTY;
1673 
1674   emit_int8((int8_t)0xF2);
1675   switch (sizeInBytes) {
1676   case 1:
1677     w = 0;
1678     break;
1679   case 2:
1680   case 4:
1681     break;
1682   LP64_ONLY(case 8:)
1683     // This instruction is not valid in 32 bits
1684     // Note:
1685     // http://www.intel.com/content/dam/www/public/us/en/documents/manuals/64-ia-32-architectures-software-developer-instruction-set-reference-manual-325383.pdf
1686     //
1687     // Page B - 72   Vol. 2C says
1688     // qwreg2 to qwreg            1111 0010 : 0100 1R0B : 0000 1111 : 0011 1000 : 1111 0000 : 11 qwreg1 qwreg2
1689     // mem64 to qwreg             1111 0010 : 0100 1R0B : 0000 1111 : 0011 1000 : 1111 0000 : mod qwreg r / m
1690     //                                                                            F0!!!
1691     // while 3 - 208 Vol. 2A
1692     // F2 REX.W 0F 38 F1 / r       CRC32 r64, r / m64             RM         Valid      N.E.Accumulate CRC32 on r / m64.
1693     //
1694     // the 0 on a last bit is reserved for a different flavor of this instruction :
1695     // F2 REX.W 0F 38 F0 / r       CRC32 r64, r / m8              RM         Valid      N.E.Accumulate CRC32 on r / m8.
1696     p = REX_W;
1697     break;
1698   default:
1699     assert(0, "Unsupported value for a sizeInBytes argument");
1700     break;
1701   }
1702   LP64_ONLY(prefix(crc, v, p);)
1703   emit_int8((int8_t)0x0F);
1704   emit_int8(0x38);
1705   emit_int8((int8_t)(0xF0 | w));
1706   emit_int8(0xC0 | ((crc->encoding() & 0x7) << 3) | (v->encoding() & 7));
1707 }
1708 
1709 void Assembler::crc32(Register crc, Address adr, int8_t sizeInBytes) {
1710   assert(VM_Version::supports_sse4_2(), "");
1711   InstructionMark im(this);
1712   int8_t w = 0x01;
1713   Prefix p = Prefix_EMPTY;
1714 
1715   emit_int8((int8_t)0xF2);
1716   switch (sizeInBytes) {
1717   case 1:
1718     w = 0;
1719     break;
1720   case 2:
1721   case 4:
1722     break;
1723   LP64_ONLY(case 8:)
1724     // This instruction is not valid in 32 bits
1725     p = REX_W;
1726     break;
1727   default:
1728     assert(0, "Unsupported value for a sizeInBytes argument");
1729     break;
1730   }
1731   LP64_ONLY(prefix(crc, adr, p);)
1732   emit_int8((int8_t)0x0F);
1733   emit_int8(0x38);
1734   emit_int8((int8_t)(0xF0 | w));
1735   emit_operand(crc, adr);
1736 }
1737 
1738 void Assembler::cvtdq2pd(XMMRegister dst, XMMRegister src) {
1739   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1740   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
1741   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
1742   emit_int8((unsigned char)0xE6);
1743   emit_int8((unsigned char)(0xC0 | encode));
1744 }
1745 
1746 void Assembler::cvtdq2ps(XMMRegister dst, XMMRegister src) {
1747   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1748   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
1749   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
1750   emit_int8(0x5B);
1751   emit_int8((unsigned char)(0xC0 | encode));
1752 }
1753 
1754 void Assembler::cvtsd2ss(XMMRegister dst, XMMRegister src) {
1755   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1756   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1757   attributes.set_rex_vex_w_reverted();
1758   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
1759   emit_int8(0x5A);
1760   emit_int8((unsigned char)(0xC0 | encode));
1761 }
1762 
1763 void Assembler::cvtsd2ss(XMMRegister dst, Address src) {
1764   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1765   InstructionMark im(this);
1766   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1767   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
1768   attributes.set_rex_vex_w_reverted();
1769   simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
1770   emit_int8(0x5A);
1771   emit_operand(dst, src);
1772 }
1773 
1774 void Assembler::cvtsi2sdl(XMMRegister dst, Register src) {
1775   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1776   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1777   int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
1778   emit_int8(0x2A);
1779   emit_int8((unsigned char)(0xC0 | encode));
1780 }
1781 
1782 void Assembler::cvtsi2sdl(XMMRegister dst, Address src) {
1783   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1784   InstructionMark im(this);
1785   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1786   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
1787   simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
1788   emit_int8(0x2A);
1789   emit_operand(dst, src);
1790 }
1791 
1792 void Assembler::cvtsi2ssl(XMMRegister dst, Register src) {
1793   NOT_LP64(assert(VM_Version::supports_sse(), ""));
1794   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1795   int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
1796   emit_int8(0x2A);
1797   emit_int8((unsigned char)(0xC0 | encode));
1798 }
1799 
1800 void Assembler::cvtsi2ssl(XMMRegister dst, Address src) {
1801   NOT_LP64(assert(VM_Version::supports_sse(), ""));
1802   InstructionMark im(this);
1803   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1804   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
1805   simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
1806   emit_int8(0x2A);
1807   emit_operand(dst, src);
1808 }
1809 
1810 void Assembler::cvtsi2ssq(XMMRegister dst, Register src) {
1811   NOT_LP64(assert(VM_Version::supports_sse(), ""));
1812   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1813   int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
1814   emit_int8(0x2A);
1815   emit_int8((unsigned char)(0xC0 | encode));
1816 }
1817 
1818 void Assembler::cvtss2sd(XMMRegister dst, XMMRegister src) {
1819   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1820   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1821   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
1822   emit_int8(0x5A);
1823   emit_int8((unsigned char)(0xC0 | encode));
1824 }
1825 
1826 void Assembler::cvtss2sd(XMMRegister dst, Address src) {
1827   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1828   InstructionMark im(this);
1829   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1830   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
1831   simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
1832   emit_int8(0x5A);
1833   emit_operand(dst, src);
1834 }
1835 
1836 
1837 void Assembler::cvttsd2sil(Register dst, XMMRegister src) {
1838   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1839   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1840   int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
1841   emit_int8(0x2C);
1842   emit_int8((unsigned char)(0xC0 | encode));
1843 }
1844 
1845 void Assembler::cvttss2sil(Register dst, XMMRegister src) {
1846   NOT_LP64(assert(VM_Version::supports_sse(), ""));
1847   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1848   int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
1849   emit_int8(0x2C);
1850   emit_int8((unsigned char)(0xC0 | encode));
1851 }
1852 
1853 void Assembler::cvttpd2dq(XMMRegister dst, XMMRegister src) {
1854   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1855   int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_128bit;
1856   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
1857   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
1858   emit_int8((unsigned char)0xE6);
1859   emit_int8((unsigned char)(0xC0 | encode));
1860 }
1861 
1862 void Assembler::decl(Address dst) {
1863   // Don't use it directly. Use MacroAssembler::decrement() instead.
1864   InstructionMark im(this);
1865   prefix(dst);
1866   emit_int8((unsigned char)0xFF);
1867   emit_operand(rcx, dst);
1868 }
1869 
1870 void Assembler::divsd(XMMRegister dst, Address src) {
1871   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1872   InstructionMark im(this);
1873   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1874   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
1875   attributes.set_rex_vex_w_reverted();
1876   simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
1877   emit_int8(0x5E);
1878   emit_operand(dst, src);
1879 }
1880 
1881 void Assembler::divsd(XMMRegister dst, XMMRegister src) {
1882   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1883   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1884   attributes.set_rex_vex_w_reverted();
1885   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
1886   emit_int8(0x5E);
1887   emit_int8((unsigned char)(0xC0 | encode));
1888 }
1889 
1890 void Assembler::divss(XMMRegister dst, Address src) {
1891   NOT_LP64(assert(VM_Version::supports_sse(), ""));
1892   InstructionMark im(this);
1893   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1894   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
1895   simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
1896   emit_int8(0x5E);
1897   emit_operand(dst, src);
1898 }
1899 
1900 void Assembler::divss(XMMRegister dst, XMMRegister src) {
1901   NOT_LP64(assert(VM_Version::supports_sse(), ""));
1902   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1903   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
1904   emit_int8(0x5E);
1905   emit_int8((unsigned char)(0xC0 | encode));
1906 }
1907 
1908 void Assembler::emms() {
1909   NOT_LP64(assert(VM_Version::supports_mmx(), ""));
1910   emit_int8(0x0F);
1911   emit_int8(0x77);
1912 }
1913 
1914 void Assembler::hlt() {
1915   emit_int8((unsigned char)0xF4);
1916 }
1917 
1918 void Assembler::idivl(Register src) {
1919   int encode = prefix_and_encode(src->encoding());
1920   emit_int8((unsigned char)0xF7);
1921   emit_int8((unsigned char)(0xF8 | encode));
1922 }
1923 
1924 void Assembler::divl(Register src) { // Unsigned
1925   int encode = prefix_and_encode(src->encoding());
1926   emit_int8((unsigned char)0xF7);
1927   emit_int8((unsigned char)(0xF0 | encode));
1928 }
1929 
1930 void Assembler::imull(Register src) {
1931   int encode = prefix_and_encode(src->encoding());
1932   emit_int8((unsigned char)0xF7);
1933   emit_int8((unsigned char)(0xE8 | encode));
1934 }
1935 
1936 void Assembler::imull(Register dst, Register src) {
1937   int encode = prefix_and_encode(dst->encoding(), src->encoding());
1938   emit_int8(0x0F);
1939   emit_int8((unsigned char)0xAF);
1940   emit_int8((unsigned char)(0xC0 | encode));
1941 }
1942 
1943 
1944 void Assembler::imull(Register dst, Register src, int value) {
1945   int encode = prefix_and_encode(dst->encoding(), src->encoding());
1946   if (is8bit(value)) {
1947     emit_int8(0x6B);
1948     emit_int8((unsigned char)(0xC0 | encode));
1949     emit_int8(value & 0xFF);
1950   } else {
1951     emit_int8(0x69);
1952     emit_int8((unsigned char)(0xC0 | encode));
1953     emit_int32(value);
1954   }
1955 }
1956 
1957 void Assembler::imull(Register dst, Address src) {
1958   InstructionMark im(this);
1959   prefix(src, dst);
1960   emit_int8(0x0F);
1961   emit_int8((unsigned char) 0xAF);
1962   emit_operand(dst, src);
1963 }
1964 
1965 
1966 void Assembler::incl(Address dst) {
1967   // Don't use it directly. Use MacroAssembler::increment() instead.
1968   InstructionMark im(this);
1969   prefix(dst);
1970   emit_int8((unsigned char)0xFF);
1971   emit_operand(rax, dst);
1972 }
1973 
1974 void Assembler::jcc(Condition cc, Label& L, bool maybe_short) {
1975   InstructionMark im(this);
1976   assert((0 <= cc) && (cc < 16), "illegal cc");
1977   if (L.is_bound()) {
1978     address dst = target(L);
1979     assert(dst != NULL, "jcc most probably wrong");
1980 
1981     const int short_size = 2;
1982     const int long_size = 6;
1983     intptr_t offs = (intptr_t)dst - (intptr_t)pc();
1984     if (maybe_short && is8bit(offs - short_size)) {
1985       // 0111 tttn #8-bit disp
1986       emit_int8(0x70 | cc);
1987       emit_int8((offs - short_size) & 0xFF);
1988     } else {
1989       // 0000 1111 1000 tttn #32-bit disp
1990       assert(is_simm32(offs - long_size),
1991              "must be 32bit offset (call4)");
1992       emit_int8(0x0F);
1993       emit_int8((unsigned char)(0x80 | cc));
1994       emit_int32(offs - long_size);
1995     }
1996   } else {
1997     // Note: could eliminate cond. jumps to this jump if condition
1998     //       is the same however, seems to be rather unlikely case.
1999     // Note: use jccb() if label to be bound is very close to get
2000     //       an 8-bit displacement
2001     L.add_patch_at(code(), locator());
2002     emit_int8(0x0F);
2003     emit_int8((unsigned char)(0x80 | cc));
2004     emit_int32(0);
2005   }
2006 }
2007 
2008 void Assembler::jccb(Condition cc, Label& L) {
2009   if (L.is_bound()) {
2010     const int short_size = 2;
2011     address entry = target(L);
2012 #ifdef ASSERT
2013     intptr_t dist = (intptr_t)entry - ((intptr_t)pc() + short_size);
2014     intptr_t delta = short_branch_delta();
2015     if (delta != 0) {
2016       dist += (dist < 0 ? (-delta) :delta);
2017     }
2018     assert(is8bit(dist), "Dispacement too large for a short jmp");
2019 #endif
2020     intptr_t offs = (intptr_t)entry - (intptr_t)pc();
2021     // 0111 tttn #8-bit disp
2022     emit_int8(0x70 | cc);
2023     emit_int8((offs - short_size) & 0xFF);
2024   } else {
2025     InstructionMark im(this);
2026     L.add_patch_at(code(), locator());
2027     emit_int8(0x70 | cc);
2028     emit_int8(0);
2029   }
2030 }
2031 
2032 void Assembler::jmp(Address adr) {
2033   InstructionMark im(this);
2034   prefix(adr);
2035   emit_int8((unsigned char)0xFF);
2036   emit_operand(rsp, adr);
2037 }
2038 
2039 void Assembler::jmp(Label& L, bool maybe_short) {
2040   if (L.is_bound()) {
2041     address entry = target(L);
2042     assert(entry != NULL, "jmp most probably wrong");
2043     InstructionMark im(this);
2044     const int short_size = 2;
2045     const int long_size = 5;
2046     intptr_t offs = entry - pc();
2047     if (maybe_short && is8bit(offs - short_size)) {
2048       emit_int8((unsigned char)0xEB);
2049       emit_int8((offs - short_size) & 0xFF);
2050     } else {
2051       emit_int8((unsigned char)0xE9);
2052       emit_int32(offs - long_size);
2053     }
2054   } else {
2055     // By default, forward jumps are always 32-bit displacements, since
2056     // we can't yet know where the label will be bound.  If you're sure that
2057     // the forward jump will not run beyond 256 bytes, use jmpb to
2058     // force an 8-bit displacement.
2059     InstructionMark im(this);
2060     L.add_patch_at(code(), locator());
2061     emit_int8((unsigned char)0xE9);
2062     emit_int32(0);
2063   }
2064 }
2065 
2066 void Assembler::jmp(Register entry) {
2067   int encode = prefix_and_encode(entry->encoding());
2068   emit_int8((unsigned char)0xFF);
2069   emit_int8((unsigned char)(0xE0 | encode));
2070 }
2071 
2072 void Assembler::jmp_literal(address dest, RelocationHolder const& rspec) {
2073   InstructionMark im(this);
2074   emit_int8((unsigned char)0xE9);
2075   assert(dest != NULL, "must have a target");
2076   intptr_t disp = dest - (pc() + sizeof(int32_t));
2077   assert(is_simm32(disp), "must be 32bit offset (jmp)");
2078   emit_data(disp, rspec.reloc(), call32_operand);
2079 }
2080 
2081 void Assembler::jmpb(Label& L) {
2082   if (L.is_bound()) {
2083     const int short_size = 2;
2084     address entry = target(L);
2085     assert(entry != NULL, "jmp most probably wrong");
2086 #ifdef ASSERT
2087     intptr_t dist = (intptr_t)entry - ((intptr_t)pc() + short_size);
2088     intptr_t delta = short_branch_delta();
2089     if (delta != 0) {
2090       dist += (dist < 0 ? (-delta) :delta);
2091     }
2092     assert(is8bit(dist), "Dispacement too large for a short jmp");
2093 #endif
2094     intptr_t offs = entry - pc();
2095     emit_int8((unsigned char)0xEB);
2096     emit_int8((offs - short_size) & 0xFF);
2097   } else {
2098     InstructionMark im(this);
2099     L.add_patch_at(code(), locator());
2100     emit_int8((unsigned char)0xEB);
2101     emit_int8(0);
2102   }
2103 }
2104 
2105 void Assembler::ldmxcsr( Address src) {
2106   if (UseAVX > 0 ) {
2107     InstructionMark im(this);
2108     InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2109     vex_prefix(src, 0, 0, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2110     emit_int8((unsigned char)0xAE);
2111     emit_operand(as_Register(2), src);
2112   } else {
2113     NOT_LP64(assert(VM_Version::supports_sse(), ""));
2114     InstructionMark im(this);
2115     prefix(src);
2116     emit_int8(0x0F);
2117     emit_int8((unsigned char)0xAE);
2118     emit_operand(as_Register(2), src);
2119   }
2120 }
2121 
2122 void Assembler::leal(Register dst, Address src) {
2123   InstructionMark im(this);
2124 #ifdef _LP64
2125   emit_int8(0x67); // addr32
2126   prefix(src, dst);
2127 #endif // LP64
2128   emit_int8((unsigned char)0x8D);
2129   emit_operand(dst, src);
2130 }
2131 
2132 void Assembler::lfence() {
2133   emit_int8(0x0F);
2134   emit_int8((unsigned char)0xAE);
2135   emit_int8((unsigned char)0xE8);
2136 }
2137 
2138 void Assembler::lock() {
2139   emit_int8((unsigned char)0xF0);
2140 }
2141 
2142 void Assembler::lzcntl(Register dst, Register src) {
2143   assert(VM_Version::supports_lzcnt(), "encoding is treated as BSR");
2144   emit_int8((unsigned char)0xF3);
2145   int encode = prefix_and_encode(dst->encoding(), src->encoding());
2146   emit_int8(0x0F);
2147   emit_int8((unsigned char)0xBD);
2148   emit_int8((unsigned char)(0xC0 | encode));
2149 }
2150 
2151 // Emit mfence instruction
2152 void Assembler::mfence() {
2153   NOT_LP64(assert(VM_Version::supports_sse2(), "unsupported");)
2154   emit_int8(0x0F);
2155   emit_int8((unsigned char)0xAE);
2156   emit_int8((unsigned char)0xF0);
2157 }
2158 
2159 // Emit sfence instruction
2160 void Assembler::sfence() {
2161   NOT_LP64(assert(VM_Version::supports_sse2(), "unsupported");)
2162   emit_int8(0x0F);
2163   emit_int8((unsigned char)0xAE);
2164   emit_int8((unsigned char)0xF8);
2165 }
2166 
2167 void Assembler::mov(Register dst, Register src) {
2168   LP64_ONLY(movq(dst, src)) NOT_LP64(movl(dst, src));
2169 }
2170 
2171 void Assembler::movapd(XMMRegister dst, XMMRegister src) {
2172   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2173   int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_128bit;
2174   InstructionAttr attributes(vector_len, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
2175   attributes.set_rex_vex_w_reverted();
2176   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2177   emit_int8(0x28);
2178   emit_int8((unsigned char)(0xC0 | encode));
2179 }
2180 
2181 void Assembler::movaps(XMMRegister dst, XMMRegister src) {
2182   NOT_LP64(assert(VM_Version::supports_sse(), ""));
2183   int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_128bit;
2184   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
2185   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2186   emit_int8(0x28);
2187   emit_int8((unsigned char)(0xC0 | encode));
2188 }
2189 
2190 void Assembler::movlhps(XMMRegister dst, XMMRegister src) {
2191   NOT_LP64(assert(VM_Version::supports_sse(), ""));
2192   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2193   int encode = simd_prefix_and_encode(dst, src, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2194   emit_int8(0x16);
2195   emit_int8((unsigned char)(0xC0 | encode));
2196 }
2197 
2198 void Assembler::movb(Register dst, Address src) {
2199   NOT_LP64(assert(dst->has_byte_register(), "must have byte register"));
2200   InstructionMark im(this);
2201   prefix(src, dst, true);
2202   emit_int8((unsigned char)0x8A);
2203   emit_operand(dst, src);
2204 }
2205 
2206 void Assembler::movddup(XMMRegister dst, XMMRegister src) {
2207   NOT_LP64(assert(VM_Version::supports_sse3(), ""));
2208   int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_128bit;
2209   InstructionAttr attributes(vector_len, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2210   attributes.set_rex_vex_w_reverted();
2211   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2212   emit_int8(0x12);
2213   emit_int8(0xC0 | encode);
2214 }
2215 
2216 void Assembler::kmovbl(KRegister dst, Register src) {
2217   assert(VM_Version::supports_avx512dq(), "");
2218   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2219   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2220   emit_int8((unsigned char)0x92);
2221   emit_int8((unsigned char)(0xC0 | encode));
2222 }
2223 
2224 void Assembler::kmovbl(Register dst, KRegister src) {
2225   assert(VM_Version::supports_avx512dq(), "");
2226   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2227   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2228   emit_int8((unsigned char)0x93);
2229   emit_int8((unsigned char)(0xC0 | encode));
2230 }
2231 
2232 void Assembler::kmovwl(KRegister dst, Register src) {
2233   assert(VM_Version::supports_evex(), "");
2234   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2235   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2236   emit_int8((unsigned char)0x92);
2237   emit_int8((unsigned char)(0xC0 | encode));
2238 }
2239 
2240 void Assembler::kmovwl(Register dst, KRegister src) {
2241   assert(VM_Version::supports_evex(), "");
2242   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2243   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2244   emit_int8((unsigned char)0x93);
2245   emit_int8((unsigned char)(0xC0 | encode));
2246 }
2247 
2248 void Assembler::kmovwl(KRegister dst, Address src) {
2249   assert(VM_Version::supports_evex(), "");
2250   InstructionMark im(this);
2251   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2252   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2253   emit_int8((unsigned char)0x90);
2254   emit_operand((Register)dst, src);
2255 }
2256 
2257 void Assembler::kmovdl(KRegister dst, Register src) {
2258   assert(VM_Version::supports_avx512bw(), "");
2259   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2260   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2261   emit_int8((unsigned char)0x92);
2262   emit_int8((unsigned char)(0xC0 | encode));
2263 }
2264 
2265 void Assembler::kmovdl(Register dst, KRegister src) {
2266   assert(VM_Version::supports_avx512bw(), "");
2267   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2268   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2269   emit_int8((unsigned char)0x93);
2270   emit_int8((unsigned char)(0xC0 | encode));
2271 }
2272 
2273 void Assembler::kmovql(KRegister dst, KRegister src) {
2274   assert(VM_Version::supports_avx512bw(), "");
2275   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2276   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2277   emit_int8((unsigned char)0x90);
2278   emit_int8((unsigned char)(0xC0 | encode));
2279 }
2280 
2281 void Assembler::kmovql(KRegister dst, Address src) {
2282   assert(VM_Version::supports_avx512bw(), "");
2283   InstructionMark im(this);
2284   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2285   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2286   emit_int8((unsigned char)0x90);
2287   emit_operand((Register)dst, src);
2288 }
2289 
2290 void Assembler::kmovql(Address dst, KRegister src) {
2291   assert(VM_Version::supports_avx512bw(), "");
2292   InstructionMark im(this);
2293   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2294   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2295   emit_int8((unsigned char)0x90);
2296   emit_operand((Register)src, dst);
2297 }
2298 
2299 void Assembler::kmovql(KRegister dst, Register src) {
2300   assert(VM_Version::supports_avx512bw(), "");
2301   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2302   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2303   emit_int8((unsigned char)0x92);
2304   emit_int8((unsigned char)(0xC0 | encode));
2305 }
2306 
2307 void Assembler::kmovql(Register dst, KRegister src) {
2308   assert(VM_Version::supports_avx512bw(), "");
2309   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2310   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2311   emit_int8((unsigned char)0x93);
2312   emit_int8((unsigned char)(0xC0 | encode));
2313 }
2314 
2315 void Assembler::knotwl(KRegister dst, KRegister src) {
2316   assert(VM_Version::supports_evex(), "");
2317   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2318   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2319   emit_int8((unsigned char)0x44);
2320   emit_int8((unsigned char)(0xC0 | encode));
2321 }
2322 
2323 // This instruction produces ZF or CF flags
2324 void Assembler::kortestbl(KRegister src1, KRegister src2) {
2325   assert(VM_Version::supports_avx512dq(), "");
2326   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2327   int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2328   emit_int8((unsigned char)0x98);
2329   emit_int8((unsigned char)(0xC0 | encode));
2330 }
2331 
2332 // This instruction produces ZF or CF flags
2333 void Assembler::kortestwl(KRegister src1, KRegister src2) {
2334   assert(VM_Version::supports_evex(), "");
2335   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2336   int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2337   emit_int8((unsigned char)0x98);
2338   emit_int8((unsigned char)(0xC0 | encode));
2339 }
2340 
2341 // This instruction produces ZF or CF flags
2342 void Assembler::kortestdl(KRegister src1, KRegister src2) {
2343   assert(VM_Version::supports_avx512bw(), "");
2344   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2345   int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2346   emit_int8((unsigned char)0x98);
2347   emit_int8((unsigned char)(0xC0 | encode));
2348 }
2349 
2350 // This instruction produces ZF or CF flags
2351 void Assembler::kortestql(KRegister src1, KRegister src2) {
2352   assert(VM_Version::supports_avx512bw(), "");
2353   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2354   int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2355   emit_int8((unsigned char)0x98);
2356   emit_int8((unsigned char)(0xC0 | encode));
2357 }
2358 
2359 // This instruction produces ZF or CF flags
2360 void Assembler::ktestql(KRegister src1, KRegister src2) {
2361   assert(VM_Version::supports_avx512bw(), "");
2362   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2363   int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2364   emit_int8((unsigned char)0x99);
2365   emit_int8((unsigned char)(0xC0 | encode));
2366 }
2367 
2368 void Assembler::ktestq(KRegister src1, KRegister src2) {
2369   assert(VM_Version::supports_avx512bw(), "");
2370   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2371   int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2372   emit_int8((unsigned char)0x99);
2373   emit_int8((unsigned char)(0xC0 | encode));
2374 }
2375 
2376 void Assembler::ktestd(KRegister src1, KRegister src2) {
2377   assert(VM_Version::supports_avx512bw(), "");
2378   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2379   int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2380   emit_int8((unsigned char)0x99);
2381   emit_int8((unsigned char)(0xC0 | encode));
2382 }
2383 
2384 void Assembler::movb(Address dst, int imm8) {
2385   InstructionMark im(this);
2386    prefix(dst);
2387   emit_int8((unsigned char)0xC6);
2388   emit_operand(rax, dst, 1);
2389   emit_int8(imm8);
2390 }
2391 
2392 
2393 void Assembler::movb(Address dst, Register src) {
2394   assert(src->has_byte_register(), "must have byte register");
2395   InstructionMark im(this);
2396   prefix(dst, src, true);
2397   emit_int8((unsigned char)0x88);
2398   emit_operand(src, dst);
2399 }
2400 
2401 void Assembler::movdl(XMMRegister dst, Register src) {
2402   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2403   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2404   int encode = simd_prefix_and_encode(dst, xnoreg, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2405   emit_int8(0x6E);
2406   emit_int8((unsigned char)(0xC0 | encode));
2407 }
2408 
2409 void Assembler::movdl(Register dst, XMMRegister src) {
2410   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2411   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2412   // swap src/dst to get correct prefix
2413   int encode = simd_prefix_and_encode(src, xnoreg, as_XMMRegister(dst->encoding()), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2414   emit_int8(0x7E);
2415   emit_int8((unsigned char)(0xC0 | encode));
2416 }
2417 
2418 void Assembler::movdl(XMMRegister dst, Address src) {
2419   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2420   InstructionMark im(this);
2421   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2422   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
2423   simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2424   emit_int8(0x6E);
2425   emit_operand(dst, src);
2426 }
2427 
2428 void Assembler::movdl(Address dst, XMMRegister src) {
2429   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2430   InstructionMark im(this);
2431   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2432   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
2433   simd_prefix(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2434   emit_int8(0x7E);
2435   emit_operand(src, dst);
2436 }
2437 
2438 void Assembler::movdqa(XMMRegister dst, XMMRegister src) {
2439   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2440   int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_128bit;
2441   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2442   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2443   emit_int8(0x6F);
2444   emit_int8((unsigned char)(0xC0 | encode));
2445 }
2446 
2447 void Assembler::movdqa(XMMRegister dst, Address src) {
2448   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2449   InstructionMark im(this);
2450   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2451   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2452   simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2453   emit_int8(0x6F);
2454   emit_operand(dst, src);
2455 }
2456 
2457 void Assembler::movdqu(XMMRegister dst, Address src) {
2458   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2459   InstructionMark im(this);
2460   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2461   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2462   simd_prefix(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2463   emit_int8(0x6F);
2464   emit_operand(dst, src);
2465 }
2466 
2467 void Assembler::movdqu(XMMRegister dst, XMMRegister src) {
2468   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2469   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2470   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2471   emit_int8(0x6F);
2472   emit_int8((unsigned char)(0xC0 | encode));
2473 }
2474 
2475 void Assembler::movdqu(Address dst, XMMRegister src) {
2476   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2477   InstructionMark im(this);
2478   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2479   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2480   attributes.reset_is_clear_context();
2481   simd_prefix(src, xnoreg, dst, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2482   emit_int8(0x7F);
2483   emit_operand(src, dst);
2484 }
2485 
2486 // Move Unaligned 256bit Vector
2487 void Assembler::vmovdqu(XMMRegister dst, XMMRegister src) {
2488   assert(UseAVX > 0, "");
2489   InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2490   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2491   emit_int8(0x6F);
2492   emit_int8((unsigned char)(0xC0 | encode));
2493 }
2494 
2495 void Assembler::vmovdqu(XMMRegister dst, Address src) {
2496   assert(UseAVX > 0, "");
2497   InstructionMark im(this);
2498   InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2499   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2500   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2501   emit_int8(0x6F);
2502   emit_operand(dst, src);
2503 }
2504 
2505 void Assembler::vmovdqu(Address dst, XMMRegister src) {
2506   assert(UseAVX > 0, "");
2507   InstructionMark im(this);
2508   InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2509   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2510   attributes.reset_is_clear_context();
2511   // swap src<->dst for encoding
2512   assert(src != xnoreg, "sanity");
2513   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2514   emit_int8(0x7F);
2515   emit_operand(src, dst);
2516 }
2517 
2518 void Assembler::vmovdqa(Address dst, XMMRegister src) {
2519   assert(UseAVX > 0, "");
2520   InstructionMark im(this);
2521   InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2522   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2523   // swap src<->dst for encoding
2524   assert(src != xnoreg, "sanity");
2525   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2526   emit_int8(0x7F);
2527   emit_operand(src, dst);
2528 }
2529 
2530 void Assembler::vmovntpd(Address dst, XMMRegister src) {
2531   assert(UseAVX > 0, "");
2532   InstructionMark im(this);
2533   InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2534   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2535   // swap src<->dst for encoding
2536   assert(src != xnoreg, "sanity");
2537   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2538   emit_int8(0x2B);
2539   emit_operand(src, dst);
2540 }
2541 
2542 // Move Unaligned EVEX enabled Vector (programmable : 8,16,32,64)
2543 void Assembler::evmovdqub(XMMRegister dst, XMMRegister src, int vector_len) {
2544   assert(VM_Version::supports_evex(), "");
2545   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
2546   attributes.set_is_evex_instruction();
2547   int prefix = (_legacy_mode_bw) ? VEX_SIMD_F2 : VEX_SIMD_F3;
2548   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), (Assembler::VexSimdPrefix)prefix, VEX_OPCODE_0F, &attributes);
2549   emit_int8(0x6F);
2550   emit_int8((unsigned char)(0xC0 | encode));
2551 }
2552 
2553 void Assembler::evmovdqub(XMMRegister dst, Address src, int vector_len) {
2554   assert(VM_Version::supports_evex(), "");
2555   InstructionMark im(this);
2556   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
2557   int prefix = (_legacy_mode_bw) ? VEX_SIMD_F2 : VEX_SIMD_F3;
2558   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2559   attributes.set_is_evex_instruction();
2560   vex_prefix(src, 0, dst->encoding(), (Assembler::VexSimdPrefix)prefix, VEX_OPCODE_0F, &attributes);
2561   emit_int8(0x6F);
2562   emit_operand(dst, src);
2563 }
2564 
2565 void Assembler::evmovdqub(Address dst, XMMRegister src, int vector_len) {
2566   assert(VM_Version::supports_evex(), "");
2567   assert(src != xnoreg, "sanity");
2568   InstructionMark im(this);
2569   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
2570   int prefix = (_legacy_mode_bw) ? VEX_SIMD_F2 : VEX_SIMD_F3;
2571   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2572   attributes.set_is_evex_instruction();
2573   vex_prefix(dst, 0, src->encoding(), (Assembler::VexSimdPrefix)prefix, VEX_OPCODE_0F, &attributes);
2574   emit_int8(0x7F);
2575   emit_operand(src, dst);
2576 }
2577 
2578 void Assembler::evmovdqub(XMMRegister dst, KRegister mask, Address src, int vector_len) {
2579   assert(VM_Version::supports_avx512vlbw(), "");
2580   assert(is_vector_masking(), "");    // For stub code use only
2581   InstructionMark im(this);
2582   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
2583   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2584   attributes.set_embedded_opmask_register_specifier(mask);
2585   attributes.set_is_evex_instruction();
2586   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2587   emit_int8(0x6F);
2588   emit_operand(dst, src);
2589 }
2590 
2591 void Assembler::evmovdquw(XMMRegister dst, Address src, int vector_len) {
2592   assert(VM_Version::supports_evex(), "");
2593   InstructionMark im(this);
2594   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
2595   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2596   attributes.set_is_evex_instruction();
2597   int prefix = (_legacy_mode_bw) ? VEX_SIMD_F2 : VEX_SIMD_F3;
2598   vex_prefix(src, 0, dst->encoding(), (Assembler::VexSimdPrefix)prefix, VEX_OPCODE_0F, &attributes);
2599   emit_int8(0x6F);
2600   emit_operand(dst, src);
2601 }
2602 
2603 void Assembler::evmovdquw(XMMRegister dst, KRegister mask, Address src, int vector_len) {
2604   assert(is_vector_masking(), "");
2605   assert(VM_Version::supports_avx512vlbw(), "");
2606   InstructionMark im(this);
2607   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
2608   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2609   attributes.set_embedded_opmask_register_specifier(mask);
2610   attributes.set_is_evex_instruction();
2611   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2612   emit_int8(0x6F);
2613   emit_operand(dst, src);
2614 }
2615 
2616 void Assembler::evmovdquw(Address dst, XMMRegister src, int vector_len) {
2617   assert(VM_Version::supports_evex(), "");
2618   assert(src != xnoreg, "sanity");
2619   InstructionMark im(this);
2620   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
2621   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2622   attributes.set_is_evex_instruction();
2623   int prefix = (_legacy_mode_bw) ? VEX_SIMD_F2 : VEX_SIMD_F3;
2624   vex_prefix(dst, 0, src->encoding(), (Assembler::VexSimdPrefix)prefix, VEX_OPCODE_0F, &attributes);
2625   emit_int8(0x7F);
2626   emit_operand(src, dst);
2627 }
2628 
2629 void Assembler::evmovdquw(Address dst, KRegister mask, XMMRegister src, int vector_len) {
2630   assert(VM_Version::supports_avx512vlbw(), "");
2631   assert(src != xnoreg, "sanity");
2632   InstructionMark im(this);
2633   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2634   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2635   attributes.reset_is_clear_context();
2636   attributes.set_embedded_opmask_register_specifier(mask);
2637   attributes.set_is_evex_instruction();
2638   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2639   emit_int8(0x7F);
2640   emit_operand(src, dst);
2641 }
2642 
2643 void Assembler::evmovdqul(XMMRegister dst, XMMRegister src, int vector_len) {
2644   assert(VM_Version::supports_evex(), "");
2645   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2646   attributes.set_is_evex_instruction();
2647   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2648   emit_int8(0x6F);
2649   emit_int8((unsigned char)(0xC0 | encode));
2650 }
2651 
2652 void Assembler::evmovdqul(XMMRegister dst, Address src, int vector_len) {
2653   assert(VM_Version::supports_evex(), "");
2654   InstructionMark im(this);
2655   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false , /* uses_vl */ true);
2656   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2657   attributes.set_is_evex_instruction();
2658   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2659   emit_int8(0x6F);
2660   emit_operand(dst, src);
2661 }
2662 
2663 void Assembler::evmovdqul(Address dst, XMMRegister src, int vector_len) {
2664   assert(VM_Version::supports_evex(), "");
2665   assert(src != xnoreg, "sanity");
2666   InstructionMark im(this);
2667   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2668   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2669   attributes.reset_is_clear_context();
2670   attributes.set_is_evex_instruction();
2671   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2672   emit_int8(0x7F);
2673   emit_operand(src, dst);
2674 }
2675 
2676 void Assembler::evmovdquq(XMMRegister dst, XMMRegister src, int vector_len) {
2677   assert(VM_Version::supports_evex(), "");
2678   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2679   attributes.set_is_evex_instruction();
2680   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2681   emit_int8(0x6F);
2682   emit_int8((unsigned char)(0xC0 | encode));
2683 }
2684 
2685 void Assembler::evmovdquq(XMMRegister dst, Address src, int vector_len) {
2686   assert(VM_Version::supports_evex(), "");
2687   InstructionMark im(this);
2688   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2689   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2690   attributes.set_is_evex_instruction();
2691   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2692   emit_int8(0x6F);
2693   emit_operand(dst, src);
2694 }
2695 
2696 void Assembler::evmovdquq(Address dst, XMMRegister src, int vector_len) {
2697   assert(VM_Version::supports_evex(), "");
2698   assert(src != xnoreg, "sanity");
2699   InstructionMark im(this);
2700   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2701   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2702   attributes.reset_is_clear_context();
2703   attributes.set_is_evex_instruction();
2704   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2705   emit_int8(0x7F);
2706   emit_operand(src, dst);
2707 }
2708 
2709 // Uses zero extension on 64bit
2710 
2711 void Assembler::movl(Register dst, int32_t imm32) {
2712   int encode = prefix_and_encode(dst->encoding());
2713   emit_int8((unsigned char)(0xB8 | encode));
2714   emit_int32(imm32);
2715 }
2716 
2717 void Assembler::movl(Register dst, Register src) {
2718   int encode = prefix_and_encode(dst->encoding(), src->encoding());
2719   emit_int8((unsigned char)0x8B);
2720   emit_int8((unsigned char)(0xC0 | encode));
2721 }
2722 
2723 void Assembler::movl(Register dst, Address src) {
2724   InstructionMark im(this);
2725   prefix(src, dst);
2726   emit_int8((unsigned char)0x8B);
2727   emit_operand(dst, src);
2728 }
2729 
2730 void Assembler::movl(Address dst, int32_t imm32) {
2731   InstructionMark im(this);
2732   prefix(dst);
2733   emit_int8((unsigned char)0xC7);
2734   emit_operand(rax, dst, 4);
2735   emit_int32(imm32);
2736 }
2737 
2738 void Assembler::movl(Address dst, Register src) {
2739   InstructionMark im(this);
2740   prefix(dst, src);
2741   emit_int8((unsigned char)0x89);
2742   emit_operand(src, dst);
2743 }
2744 
2745 // New cpus require to use movsd and movss to avoid partial register stall
2746 // when loading from memory. But for old Opteron use movlpd instead of movsd.
2747 // The selection is done in MacroAssembler::movdbl() and movflt().
2748 void Assembler::movlpd(XMMRegister dst, Address src) {
2749   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2750   InstructionMark im(this);
2751   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2752   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
2753   attributes.set_rex_vex_w_reverted();
2754   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2755   emit_int8(0x12);
2756   emit_operand(dst, src);
2757 }
2758 
2759 void Assembler::movq( MMXRegister dst, Address src ) {
2760   assert( VM_Version::supports_mmx(), "" );
2761   emit_int8(0x0F);
2762   emit_int8(0x6F);
2763   emit_operand(dst, src);
2764 }
2765 
2766 void Assembler::movq( Address dst, MMXRegister src ) {
2767   assert( VM_Version::supports_mmx(), "" );
2768   emit_int8(0x0F);
2769   emit_int8(0x7F);
2770   // workaround gcc (3.2.1-7a) bug
2771   // In that version of gcc with only an emit_operand(MMX, Address)
2772   // gcc will tail jump and try and reverse the parameters completely
2773   // obliterating dst in the process. By having a version available
2774   // that doesn't need to swap the args at the tail jump the bug is
2775   // avoided.
2776   emit_operand(dst, src);
2777 }
2778 
2779 void Assembler::movq(XMMRegister dst, Address src) {
2780   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2781   InstructionMark im(this);
2782   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2783   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
2784   attributes.set_rex_vex_w_reverted();
2785   simd_prefix(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2786   emit_int8(0x7E);
2787   emit_operand(dst, src);
2788 }
2789 
2790 void Assembler::movq(Address dst, XMMRegister src) {
2791   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2792   InstructionMark im(this);
2793   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2794   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
2795   attributes.set_rex_vex_w_reverted();
2796   simd_prefix(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2797   emit_int8((unsigned char)0xD6);
2798   emit_operand(src, dst);
2799 }
2800 
2801 void Assembler::movsbl(Register dst, Address src) { // movsxb
2802   InstructionMark im(this);
2803   prefix(src, dst);
2804   emit_int8(0x0F);
2805   emit_int8((unsigned char)0xBE);
2806   emit_operand(dst, src);
2807 }
2808 
2809 void Assembler::movsbl(Register dst, Register src) { // movsxb
2810   NOT_LP64(assert(src->has_byte_register(), "must have byte register"));
2811   int encode = prefix_and_encode(dst->encoding(), false, src->encoding(), true);
2812   emit_int8(0x0F);
2813   emit_int8((unsigned char)0xBE);
2814   emit_int8((unsigned char)(0xC0 | encode));
2815 }
2816 
2817 void Assembler::movsd(XMMRegister dst, XMMRegister src) {
2818   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2819   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2820   attributes.set_rex_vex_w_reverted();
2821   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2822   emit_int8(0x10);
2823   emit_int8((unsigned char)(0xC0 | encode));
2824 }
2825 
2826 void Assembler::movsd(XMMRegister dst, Address src) {
2827   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2828   InstructionMark im(this);
2829   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2830   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
2831   attributes.set_rex_vex_w_reverted();
2832   simd_prefix(dst, xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2833   emit_int8(0x10);
2834   emit_operand(dst, src);
2835 }
2836 
2837 void Assembler::movsd(Address dst, XMMRegister src) {
2838   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2839   InstructionMark im(this);
2840   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2841   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
2842   attributes.reset_is_clear_context();
2843   attributes.set_rex_vex_w_reverted();
2844   simd_prefix(src, xnoreg, dst, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2845   emit_int8(0x11);
2846   emit_operand(src, dst);
2847 }
2848 
2849 void Assembler::movss(XMMRegister dst, XMMRegister src) {
2850   NOT_LP64(assert(VM_Version::supports_sse(), ""));
2851   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2852   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2853   emit_int8(0x10);
2854   emit_int8((unsigned char)(0xC0 | encode));
2855 }
2856 
2857 void Assembler::movss(XMMRegister dst, Address src) {
2858   NOT_LP64(assert(VM_Version::supports_sse(), ""));
2859   InstructionMark im(this);
2860   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2861   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
2862   simd_prefix(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2863   emit_int8(0x10);
2864   emit_operand(dst, src);
2865 }
2866 
2867 void Assembler::movss(Address dst, XMMRegister src) {
2868   NOT_LP64(assert(VM_Version::supports_sse(), ""));
2869   InstructionMark im(this);
2870   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2871   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
2872   attributes.reset_is_clear_context();
2873   simd_prefix(src, xnoreg, dst, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2874   emit_int8(0x11);
2875   emit_operand(src, dst);
2876 }
2877 
2878 void Assembler::movswl(Register dst, Address src) { // movsxw
2879   InstructionMark im(this);
2880   prefix(src, dst);
2881   emit_int8(0x0F);
2882   emit_int8((unsigned char)0xBF);
2883   emit_operand(dst, src);
2884 }
2885 
2886 void Assembler::movswl(Register dst, Register src) { // movsxw
2887   int encode = prefix_and_encode(dst->encoding(), src->encoding());
2888   emit_int8(0x0F);
2889   emit_int8((unsigned char)0xBF);
2890   emit_int8((unsigned char)(0xC0 | encode));
2891 }
2892 
2893 void Assembler::movw(Address dst, int imm16) {
2894   InstructionMark im(this);
2895 
2896   emit_int8(0x66); // switch to 16-bit mode
2897   prefix(dst);
2898   emit_int8((unsigned char)0xC7);
2899   emit_operand(rax, dst, 2);
2900   emit_int16(imm16);
2901 }
2902 
2903 void Assembler::movw(Register dst, Address src) {
2904   InstructionMark im(this);
2905   emit_int8(0x66);
2906   prefix(src, dst);
2907   emit_int8((unsigned char)0x8B);
2908   emit_operand(dst, src);
2909 }
2910 
2911 void Assembler::movw(Address dst, Register src) {
2912   InstructionMark im(this);
2913   emit_int8(0x66);
2914   prefix(dst, src);
2915   emit_int8((unsigned char)0x89);
2916   emit_operand(src, dst);
2917 }
2918 
2919 void Assembler::movzbl(Register dst, Address src) { // movzxb
2920   InstructionMark im(this);
2921   prefix(src, dst);
2922   emit_int8(0x0F);
2923   emit_int8((unsigned char)0xB6);
2924   emit_operand(dst, src);
2925 }
2926 
2927 void Assembler::movzbl(Register dst, Register src) { // movzxb
2928   NOT_LP64(assert(src->has_byte_register(), "must have byte register"));
2929   int encode = prefix_and_encode(dst->encoding(), false, src->encoding(), true);
2930   emit_int8(0x0F);
2931   emit_int8((unsigned char)0xB6);
2932   emit_int8(0xC0 | encode);
2933 }
2934 
2935 void Assembler::movzwl(Register dst, Address src) { // movzxw
2936   InstructionMark im(this);
2937   prefix(src, dst);
2938   emit_int8(0x0F);
2939   emit_int8((unsigned char)0xB7);
2940   emit_operand(dst, src);
2941 }
2942 
2943 void Assembler::movzwl(Register dst, Register src) { // movzxw
2944   int encode = prefix_and_encode(dst->encoding(), src->encoding());
2945   emit_int8(0x0F);
2946   emit_int8((unsigned char)0xB7);
2947   emit_int8(0xC0 | encode);
2948 }
2949 
2950 void Assembler::mull(Address src) {
2951   InstructionMark im(this);
2952   prefix(src);
2953   emit_int8((unsigned char)0xF7);
2954   emit_operand(rsp, src);
2955 }
2956 
2957 void Assembler::mull(Register src) {
2958   int encode = prefix_and_encode(src->encoding());
2959   emit_int8((unsigned char)0xF7);
2960   emit_int8((unsigned char)(0xE0 | encode));
2961 }
2962 
2963 void Assembler::mulsd(XMMRegister dst, Address src) {
2964   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2965   InstructionMark im(this);
2966   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2967   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
2968   attributes.set_rex_vex_w_reverted();
2969   simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2970   emit_int8(0x59);
2971   emit_operand(dst, src);
2972 }
2973 
2974 void Assembler::mulsd(XMMRegister dst, XMMRegister src) {
2975   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2976   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2977   attributes.set_rex_vex_w_reverted();
2978   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2979   emit_int8(0x59);
2980   emit_int8((unsigned char)(0xC0 | encode));
2981 }
2982 
2983 void Assembler::mulss(XMMRegister dst, Address src) {
2984   NOT_LP64(assert(VM_Version::supports_sse(), ""));
2985   InstructionMark im(this);
2986   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2987   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
2988   simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2989   emit_int8(0x59);
2990   emit_operand(dst, src);
2991 }
2992 
2993 void Assembler::mulss(XMMRegister dst, XMMRegister src) {
2994   NOT_LP64(assert(VM_Version::supports_sse(), ""));
2995   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2996   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2997   emit_int8(0x59);
2998   emit_int8((unsigned char)(0xC0 | encode));
2999 }
3000 
3001 void Assembler::negl(Register dst) {
3002   int encode = prefix_and_encode(dst->encoding());
3003   emit_int8((unsigned char)0xF7);
3004   emit_int8((unsigned char)(0xD8 | encode));
3005 }
3006 
3007 void Assembler::nop(int i) {
3008 #ifdef ASSERT
3009   assert(i > 0, " ");
3010   // The fancy nops aren't currently recognized by debuggers making it a
3011   // pain to disassemble code while debugging. If asserts are on clearly
3012   // speed is not an issue so simply use the single byte traditional nop
3013   // to do alignment.
3014 
3015   for (; i > 0 ; i--) emit_int8((unsigned char)0x90);
3016   return;
3017 
3018 #endif // ASSERT
3019 
3020   if (UseAddressNop && VM_Version::is_intel()) {
3021     //
3022     // Using multi-bytes nops "0x0F 0x1F [address]" for Intel
3023     //  1: 0x90
3024     //  2: 0x66 0x90
3025     //  3: 0x66 0x66 0x90 (don't use "0x0F 0x1F 0x00" - need patching safe padding)
3026     //  4: 0x0F 0x1F 0x40 0x00
3027     //  5: 0x0F 0x1F 0x44 0x00 0x00
3028     //  6: 0x66 0x0F 0x1F 0x44 0x00 0x00
3029     //  7: 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00
3030     //  8: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
3031     //  9: 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
3032     // 10: 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
3033     // 11: 0x66 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
3034 
3035     // The rest coding is Intel specific - don't use consecutive address nops
3036 
3037     // 12: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
3038     // 13: 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
3039     // 14: 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
3040     // 15: 0x66 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
3041 
3042     while(i >= 15) {
3043       // For Intel don't generate consecutive addess nops (mix with regular nops)
3044       i -= 15;
3045       emit_int8(0x66);   // size prefix
3046       emit_int8(0x66);   // size prefix
3047       emit_int8(0x66);   // size prefix
3048       addr_nop_8();
3049       emit_int8(0x66);   // size prefix
3050       emit_int8(0x66);   // size prefix
3051       emit_int8(0x66);   // size prefix
3052       emit_int8((unsigned char)0x90);
3053                          // nop
3054     }
3055     switch (i) {
3056       case 14:
3057         emit_int8(0x66); // size prefix
3058       case 13:
3059         emit_int8(0x66); // size prefix
3060       case 12:
3061         addr_nop_8();
3062         emit_int8(0x66); // size prefix
3063         emit_int8(0x66); // size prefix
3064         emit_int8(0x66); // size prefix
3065         emit_int8((unsigned char)0x90);
3066                          // nop
3067         break;
3068       case 11:
3069         emit_int8(0x66); // size prefix
3070       case 10:
3071         emit_int8(0x66); // size prefix
3072       case 9:
3073         emit_int8(0x66); // size prefix
3074       case 8:
3075         addr_nop_8();
3076         break;
3077       case 7:
3078         addr_nop_7();
3079         break;
3080       case 6:
3081         emit_int8(0x66); // size prefix
3082       case 5:
3083         addr_nop_5();
3084         break;
3085       case 4:
3086         addr_nop_4();
3087         break;
3088       case 3:
3089         // Don't use "0x0F 0x1F 0x00" - need patching safe padding
3090         emit_int8(0x66); // size prefix
3091       case 2:
3092         emit_int8(0x66); // size prefix
3093       case 1:
3094         emit_int8((unsigned char)0x90);
3095                          // nop
3096         break;
3097       default:
3098         assert(i == 0, " ");
3099     }
3100     return;
3101   }
3102   if (UseAddressNop && VM_Version::is_amd()) {
3103     //
3104     // Using multi-bytes nops "0x0F 0x1F [address]" for AMD.
3105     //  1: 0x90
3106     //  2: 0x66 0x90
3107     //  3: 0x66 0x66 0x90 (don't use "0x0F 0x1F 0x00" - need patching safe padding)
3108     //  4: 0x0F 0x1F 0x40 0x00
3109     //  5: 0x0F 0x1F 0x44 0x00 0x00
3110     //  6: 0x66 0x0F 0x1F 0x44 0x00 0x00
3111     //  7: 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00
3112     //  8: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
3113     //  9: 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
3114     // 10: 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
3115     // 11: 0x66 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
3116 
3117     // The rest coding is AMD specific - use consecutive address nops
3118 
3119     // 12: 0x66 0x0F 0x1F 0x44 0x00 0x00 0x66 0x0F 0x1F 0x44 0x00 0x00
3120     // 13: 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00 0x66 0x0F 0x1F 0x44 0x00 0x00
3121     // 14: 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00
3122     // 15: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00
3123     // 16: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
3124     //     Size prefixes (0x66) are added for larger sizes
3125 
3126     while(i >= 22) {
3127       i -= 11;
3128       emit_int8(0x66); // size prefix
3129       emit_int8(0x66); // size prefix
3130       emit_int8(0x66); // size prefix
3131       addr_nop_8();
3132     }
3133     // Generate first nop for size between 21-12
3134     switch (i) {
3135       case 21:
3136         i -= 1;
3137         emit_int8(0x66); // size prefix
3138       case 20:
3139       case 19:
3140         i -= 1;
3141         emit_int8(0x66); // size prefix
3142       case 18:
3143       case 17:
3144         i -= 1;
3145         emit_int8(0x66); // size prefix
3146       case 16:
3147       case 15:
3148         i -= 8;
3149         addr_nop_8();
3150         break;
3151       case 14:
3152       case 13:
3153         i -= 7;
3154         addr_nop_7();
3155         break;
3156       case 12:
3157         i -= 6;
3158         emit_int8(0x66); // size prefix
3159         addr_nop_5();
3160         break;
3161       default:
3162         assert(i < 12, " ");
3163     }
3164 
3165     // Generate second nop for size between 11-1
3166     switch (i) {
3167       case 11:
3168         emit_int8(0x66); // size prefix
3169       case 10:
3170         emit_int8(0x66); // size prefix
3171       case 9:
3172         emit_int8(0x66); // size prefix
3173       case 8:
3174         addr_nop_8();
3175         break;
3176       case 7:
3177         addr_nop_7();
3178         break;
3179       case 6:
3180         emit_int8(0x66); // size prefix
3181       case 5:
3182         addr_nop_5();
3183         break;
3184       case 4:
3185         addr_nop_4();
3186         break;
3187       case 3:
3188         // Don't use "0x0F 0x1F 0x00" - need patching safe padding
3189         emit_int8(0x66); // size prefix
3190       case 2:
3191         emit_int8(0x66); // size prefix
3192       case 1:
3193         emit_int8((unsigned char)0x90);
3194                          // nop
3195         break;
3196       default:
3197         assert(i == 0, " ");
3198     }
3199     return;
3200   }
3201 
3202   if (UseAddressNop && VM_Version::is_zx()) {
3203     //
3204     // Using multi-bytes nops "0x0F 0x1F [address]" for ZX
3205     //  1: 0x90
3206     //  2: 0x66 0x90
3207     //  3: 0x66 0x66 0x90 (don't use "0x0F 0x1F 0x00" - need patching safe padding)
3208     //  4: 0x0F 0x1F 0x40 0x00
3209     //  5: 0x0F 0x1F 0x44 0x00 0x00
3210     //  6: 0x66 0x0F 0x1F 0x44 0x00 0x00
3211     //  7: 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00
3212     //  8: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
3213     //  9: 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
3214     // 10: 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
3215     // 11: 0x66 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
3216 
3217     // The rest coding is ZX specific - don't use consecutive address nops
3218 
3219     // 12: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
3220     // 13: 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
3221     // 14: 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
3222     // 15: 0x66 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
3223 
3224     while (i >= 15) {
3225       // For ZX don't generate consecutive addess nops (mix with regular nops)
3226       i -= 15;
3227       emit_int8(0x66);   // size prefix
3228       emit_int8(0x66);   // size prefix
3229       emit_int8(0x66);   // size prefix
3230       addr_nop_8();
3231       emit_int8(0x66);   // size prefix
3232       emit_int8(0x66);   // size prefix
3233       emit_int8(0x66);   // size prefix
3234       emit_int8((unsigned char)0x90);
3235                          // nop
3236     }
3237     switch (i) {
3238       case 14:
3239         emit_int8(0x66); // size prefix
3240       case 13:
3241         emit_int8(0x66); // size prefix
3242       case 12:
3243         addr_nop_8();
3244         emit_int8(0x66); // size prefix
3245         emit_int8(0x66); // size prefix
3246         emit_int8(0x66); // size prefix
3247         emit_int8((unsigned char)0x90);
3248                          // nop
3249         break;
3250       case 11:
3251         emit_int8(0x66); // size prefix
3252       case 10:
3253         emit_int8(0x66); // size prefix
3254       case 9:
3255         emit_int8(0x66); // size prefix
3256       case 8:
3257         addr_nop_8();
3258         break;
3259       case 7:
3260         addr_nop_7();
3261         break;
3262       case 6:
3263         emit_int8(0x66); // size prefix
3264       case 5:
3265         addr_nop_5();
3266         break;
3267       case 4:
3268         addr_nop_4();
3269         break;
3270       case 3:
3271         // Don't use "0x0F 0x1F 0x00" - need patching safe padding
3272         emit_int8(0x66); // size prefix
3273       case 2:
3274         emit_int8(0x66); // size prefix
3275       case 1:
3276         emit_int8((unsigned char)0x90);
3277                          // nop
3278         break;
3279       default:
3280         assert(i == 0, " ");
3281     }
3282     return;
3283   }
3284 
3285   // Using nops with size prefixes "0x66 0x90".
3286   // From AMD Optimization Guide:
3287   //  1: 0x90
3288   //  2: 0x66 0x90
3289   //  3: 0x66 0x66 0x90
3290   //  4: 0x66 0x66 0x66 0x90
3291   //  5: 0x66 0x66 0x90 0x66 0x90
3292   //  6: 0x66 0x66 0x90 0x66 0x66 0x90
3293   //  7: 0x66 0x66 0x66 0x90 0x66 0x66 0x90
3294   //  8: 0x66 0x66 0x66 0x90 0x66 0x66 0x66 0x90
3295   //  9: 0x66 0x66 0x90 0x66 0x66 0x90 0x66 0x66 0x90
3296   // 10: 0x66 0x66 0x66 0x90 0x66 0x66 0x90 0x66 0x66 0x90
3297   //
3298   while(i > 12) {
3299     i -= 4;
3300     emit_int8(0x66); // size prefix
3301     emit_int8(0x66);
3302     emit_int8(0x66);
3303     emit_int8((unsigned char)0x90);
3304                      // nop
3305   }
3306   // 1 - 12 nops
3307   if(i > 8) {
3308     if(i > 9) {
3309       i -= 1;
3310       emit_int8(0x66);
3311     }
3312     i -= 3;
3313     emit_int8(0x66);
3314     emit_int8(0x66);
3315     emit_int8((unsigned char)0x90);
3316   }
3317   // 1 - 8 nops
3318   if(i > 4) {
3319     if(i > 6) {
3320       i -= 1;
3321       emit_int8(0x66);
3322     }
3323     i -= 3;
3324     emit_int8(0x66);
3325     emit_int8(0x66);
3326     emit_int8((unsigned char)0x90);
3327   }
3328   switch (i) {
3329     case 4:
3330       emit_int8(0x66);
3331     case 3:
3332       emit_int8(0x66);
3333     case 2:
3334       emit_int8(0x66);
3335     case 1:
3336       emit_int8((unsigned char)0x90);
3337       break;
3338     default:
3339       assert(i == 0, " ");
3340   }
3341 }
3342 
3343 void Assembler::notl(Register dst) {
3344   int encode = prefix_and_encode(dst->encoding());
3345   emit_int8((unsigned char)0xF7);
3346   emit_int8((unsigned char)(0xD0 | encode));
3347 }
3348 
3349 void Assembler::orl(Address dst, int32_t imm32) {
3350   InstructionMark im(this);
3351   prefix(dst);
3352   emit_arith_operand(0x81, rcx, dst, imm32);
3353 }
3354 
3355 void Assembler::orl(Register dst, int32_t imm32) {
3356   prefix(dst);
3357   emit_arith(0x81, 0xC8, dst, imm32);
3358 }
3359 
3360 void Assembler::orl(Register dst, Address src) {
3361   InstructionMark im(this);
3362   prefix(src, dst);
3363   emit_int8(0x0B);
3364   emit_operand(dst, src);
3365 }
3366 
3367 void Assembler::orl(Register dst, Register src) {
3368   (void) prefix_and_encode(dst->encoding(), src->encoding());
3369   emit_arith(0x0B, 0xC0, dst, src);
3370 }
3371 
3372 void Assembler::orl(Address dst, Register src) {
3373   InstructionMark im(this);
3374   prefix(dst, src);
3375   emit_int8(0x09);
3376   emit_operand(src, dst);
3377 }
3378 
3379 void Assembler::packuswb(XMMRegister dst, Address src) {
3380   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3381   assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
3382   InstructionMark im(this);
3383   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
3384   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
3385   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3386   emit_int8(0x67);
3387   emit_operand(dst, src);
3388 }
3389 
3390 void Assembler::packuswb(XMMRegister dst, XMMRegister src) {
3391   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3392   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
3393   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3394   emit_int8(0x67);
3395   emit_int8((unsigned char)(0xC0 | encode));
3396 }
3397 
3398 void Assembler::vpackuswb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
3399   assert(UseAVX > 0, "some form of AVX must be enabled");
3400   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
3401   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3402   emit_int8(0x67);
3403   emit_int8((unsigned char)(0xC0 | encode));
3404 }
3405 
3406 void Assembler::vpermq(XMMRegister dst, XMMRegister src, int imm8, int vector_len) {
3407   assert(VM_Version::supports_avx2(), "");
3408   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
3409   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3410   emit_int8(0x00);
3411   emit_int8(0xC0 | encode);
3412   emit_int8(imm8);
3413 }
3414 
3415 void Assembler::vperm2i128(XMMRegister dst,  XMMRegister nds, XMMRegister src, int imm8) {
3416   assert(VM_Version::supports_avx2(), "");
3417   InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3418   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3419   emit_int8(0x46);
3420   emit_int8(0xC0 | encode);
3421   emit_int8(imm8);
3422 }
3423 
3424 void Assembler::vperm2f128(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8) {
3425   assert(VM_Version::supports_avx(), "");
3426   InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3427   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3428   emit_int8(0x06);
3429   emit_int8(0xC0 | encode);
3430   emit_int8(imm8);
3431 }
3432 
3433 
3434 void Assembler::pause() {
3435   emit_int8((unsigned char)0xF3);
3436   emit_int8((unsigned char)0x90);
3437 }
3438 
3439 void Assembler::ud2() {
3440   emit_int8(0x0F);
3441   emit_int8(0x0B);
3442 }
3443 
3444 void Assembler::pcmpestri(XMMRegister dst, Address src, int imm8) {
3445   assert(VM_Version::supports_sse4_2(), "");
3446   InstructionMark im(this);
3447   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3448   simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3449   emit_int8(0x61);
3450   emit_operand(dst, src);
3451   emit_int8(imm8);
3452 }
3453 
3454 void Assembler::pcmpestri(XMMRegister dst, XMMRegister src, int imm8) {
3455   assert(VM_Version::supports_sse4_2(), "");
3456   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3457   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3458   emit_int8(0x61);
3459   emit_int8((unsigned char)(0xC0 | encode));
3460   emit_int8(imm8);
3461 }
3462 
3463 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
3464 void Assembler::pcmpeqb(XMMRegister dst, XMMRegister src) {
3465   assert(VM_Version::supports_sse2(), "");
3466   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3467   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3468   emit_int8(0x74);
3469   emit_int8((unsigned char)(0xC0 | encode));
3470 }
3471 
3472 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
3473 void Assembler::vpcmpeqb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
3474   assert(VM_Version::supports_avx(), "");
3475   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3476   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3477   emit_int8(0x74);
3478   emit_int8((unsigned char)(0xC0 | encode));
3479 }
3480 
3481 // In this context, kdst is written the mask used to process the equal components
3482 void Assembler::evpcmpeqb(KRegister kdst, XMMRegister nds, XMMRegister src, int vector_len) {
3483   assert(VM_Version::supports_avx512bw(), "");
3484   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
3485   attributes.set_is_evex_instruction();
3486   int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3487   emit_int8(0x74);
3488   emit_int8((unsigned char)(0xC0 | encode));
3489 }
3490 
3491 void Assembler::evpcmpgtb(KRegister kdst, XMMRegister nds, Address src, int vector_len) {
3492   assert(VM_Version::supports_avx512vlbw(), "");
3493   InstructionMark im(this);
3494   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3495   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
3496   attributes.set_is_evex_instruction();
3497   int dst_enc = kdst->encoding();
3498   vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3499   emit_int8(0x64);
3500   emit_operand(as_Register(dst_enc), src);
3501 }
3502 
3503 void Assembler::evpcmpgtb(KRegister kdst, KRegister mask, XMMRegister nds, Address src, int vector_len) {
3504   assert(is_vector_masking(), "");
3505   assert(VM_Version::supports_avx512vlbw(), "");
3506   InstructionMark im(this);
3507   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
3508   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
3509   attributes.reset_is_clear_context();
3510   attributes.set_embedded_opmask_register_specifier(mask);
3511   attributes.set_is_evex_instruction();
3512   int dst_enc = kdst->encoding();
3513   vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3514   emit_int8(0x64);
3515   emit_operand(as_Register(dst_enc), src);
3516 }
3517 
3518 void Assembler::evpcmpuw(KRegister kdst, XMMRegister nds, XMMRegister src, ComparisonPredicate vcc, int vector_len) {
3519   assert(VM_Version::supports_avx512vlbw(), "");
3520   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3521   attributes.set_is_evex_instruction();
3522   int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3523   emit_int8(0x3E);
3524   emit_int8((unsigned char)(0xC0 | encode));
3525   emit_int8(vcc);
3526 }
3527 
3528 void Assembler::evpcmpuw(KRegister kdst, KRegister mask, XMMRegister nds, XMMRegister src, ComparisonPredicate vcc, int vector_len) {
3529   assert(is_vector_masking(), "");
3530   assert(VM_Version::supports_avx512vlbw(), "");
3531   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
3532   attributes.reset_is_clear_context();
3533   attributes.set_embedded_opmask_register_specifier(mask);
3534   attributes.set_is_evex_instruction();
3535   int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3536   emit_int8(0x3E);
3537   emit_int8((unsigned char)(0xC0 | encode));
3538   emit_int8(vcc);
3539 }
3540 
3541 void Assembler::evpcmpuw(KRegister kdst, XMMRegister nds, Address src, ComparisonPredicate vcc, int vector_len) {
3542   assert(VM_Version::supports_avx512vlbw(), "");
3543   InstructionMark im(this);
3544   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3545   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
3546   attributes.set_is_evex_instruction();
3547   int dst_enc = kdst->encoding();
3548   vex_prefix(src, nds->encoding(), kdst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3549   emit_int8(0x3E);
3550   emit_operand(as_Register(dst_enc), src);
3551   emit_int8(vcc);
3552 }
3553 
3554 void Assembler::evpcmpeqb(KRegister kdst, XMMRegister nds, Address src, int vector_len) {
3555   assert(VM_Version::supports_avx512bw(), "");
3556   InstructionMark im(this);
3557   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3558   attributes.set_is_evex_instruction();
3559   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
3560   int dst_enc = kdst->encoding();
3561   vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3562   emit_int8(0x74);
3563   emit_operand(as_Register(dst_enc), src);
3564 }
3565 
3566 void Assembler::evpcmpeqb(KRegister kdst, KRegister mask, XMMRegister nds, Address src, int vector_len) {
3567   assert(VM_Version::supports_avx512vlbw(), "");
3568   assert(is_vector_masking(), "");    // For stub code use only
3569   InstructionMark im(this);
3570   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_reg_mask */ false, /* uses_vl */ false);
3571   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
3572   attributes.reset_is_clear_context();
3573   attributes.set_embedded_opmask_register_specifier(mask);
3574   attributes.set_is_evex_instruction();
3575   vex_prefix(src, nds->encoding(), kdst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3576   emit_int8(0x74);
3577   emit_operand(as_Register(kdst->encoding()), src);
3578 }
3579 
3580 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
3581 void Assembler::pcmpeqw(XMMRegister dst, XMMRegister src) {
3582   assert(VM_Version::supports_sse2(), "");
3583   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3584   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3585   emit_int8(0x75);
3586   emit_int8((unsigned char)(0xC0 | encode));
3587 }
3588 
3589 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
3590 void Assembler::vpcmpeqw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
3591   assert(VM_Version::supports_avx(), "");
3592   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3593   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3594   emit_int8(0x75);
3595   emit_int8((unsigned char)(0xC0 | encode));
3596 }
3597 
3598 // In this context, kdst is written the mask used to process the equal components
3599 void Assembler::evpcmpeqw(KRegister kdst, XMMRegister nds, XMMRegister src, int vector_len) {
3600   assert(VM_Version::supports_avx512bw(), "");
3601   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
3602   attributes.set_is_evex_instruction();
3603   int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3604   emit_int8(0x75);
3605   emit_int8((unsigned char)(0xC0 | encode));
3606 }
3607 
3608 void Assembler::evpcmpeqw(KRegister kdst, XMMRegister nds, Address src, int vector_len) {
3609   assert(VM_Version::supports_avx512bw(), "");
3610   InstructionMark im(this);
3611   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
3612   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
3613   attributes.set_is_evex_instruction();
3614   int dst_enc = kdst->encoding();
3615   vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3616   emit_int8(0x75);
3617   emit_operand(as_Register(dst_enc), src);
3618 }
3619 
3620 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
3621 void Assembler::pcmpeqd(XMMRegister dst, XMMRegister src) {
3622   assert(VM_Version::supports_sse2(), "");
3623   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3624   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3625   emit_int8(0x76);
3626   emit_int8((unsigned char)(0xC0 | encode));
3627 }
3628 
3629 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
3630 void Assembler::vpcmpeqd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
3631   assert(VM_Version::supports_avx(), "");
3632   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3633   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3634   emit_int8(0x76);
3635   emit_int8((unsigned char)(0xC0 | encode));
3636 }
3637 
3638 // In this context, kdst is written the mask used to process the equal components
3639 void Assembler::evpcmpeqd(KRegister kdst, XMMRegister nds, XMMRegister src, int vector_len) {
3640   assert(VM_Version::supports_evex(), "");
3641   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
3642   attributes.set_is_evex_instruction();
3643   attributes.reset_is_clear_context();
3644   int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3645   emit_int8(0x76);
3646   emit_int8((unsigned char)(0xC0 | encode));
3647 }
3648 
3649 void Assembler::evpcmpeqd(KRegister kdst, XMMRegister nds, Address src, int vector_len) {
3650   assert(VM_Version::supports_evex(), "");
3651   InstructionMark im(this);
3652   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
3653   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
3654   attributes.reset_is_clear_context();
3655   attributes.set_is_evex_instruction();
3656   int dst_enc = kdst->encoding();
3657   vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3658   emit_int8(0x76);
3659   emit_operand(as_Register(dst_enc), src);
3660 }
3661 
3662 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
3663 void Assembler::pcmpeqq(XMMRegister dst, XMMRegister src) {
3664   assert(VM_Version::supports_sse4_1(), "");
3665   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3666   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3667   emit_int8(0x29);
3668   emit_int8((unsigned char)(0xC0 | encode));
3669 }
3670 
3671 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
3672 void Assembler::vpcmpeqq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
3673   assert(VM_Version::supports_avx(), "");
3674   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3675   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3676   emit_int8(0x29);
3677   emit_int8((unsigned char)(0xC0 | encode));
3678 }
3679 
3680 // In this context, kdst is written the mask used to process the equal components
3681 void Assembler::evpcmpeqq(KRegister kdst, XMMRegister nds, XMMRegister src, int vector_len) {
3682   assert(VM_Version::supports_evex(), "");
3683   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
3684   attributes.reset_is_clear_context();
3685   attributes.set_is_evex_instruction();
3686   int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3687   emit_int8(0x29);
3688   emit_int8((unsigned char)(0xC0 | encode));
3689 }
3690 
3691 // In this context, kdst is written the mask used to process the equal components
3692 void Assembler::evpcmpeqq(KRegister kdst, XMMRegister nds, Address src, int vector_len) {
3693   assert(VM_Version::supports_evex(), "");
3694   InstructionMark im(this);
3695   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
3696   attributes.reset_is_clear_context();
3697   attributes.set_is_evex_instruction();
3698   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
3699   int dst_enc = kdst->encoding();
3700   vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3701   emit_int8(0x29);
3702   emit_operand(as_Register(dst_enc), src);
3703 }
3704 
3705 void Assembler::pmovmskb(Register dst, XMMRegister src) {
3706   assert(VM_Version::supports_sse2(), "");
3707   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3708   int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3709   emit_int8((unsigned char)0xD7);
3710   emit_int8((unsigned char)(0xC0 | encode));
3711 }
3712 
3713 void Assembler::vpmovmskb(Register dst, XMMRegister src) {
3714   assert(VM_Version::supports_avx2(), "");
3715   InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3716   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3717   emit_int8((unsigned char)0xD7);
3718   emit_int8((unsigned char)(0xC0 | encode));
3719 }
3720 
3721 void Assembler::pextrd(Register dst, XMMRegister src, int imm8) {
3722   assert(VM_Version::supports_sse4_1(), "");
3723   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
3724   int encode = simd_prefix_and_encode(src, xnoreg, as_XMMRegister(dst->encoding()), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3725   emit_int8(0x16);
3726   emit_int8((unsigned char)(0xC0 | encode));
3727   emit_int8(imm8);
3728 }
3729 
3730 void Assembler::pextrd(Address dst, XMMRegister src, int imm8) {
3731   assert(VM_Version::supports_sse4_1(), "");
3732   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
3733   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
3734   simd_prefix(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3735   emit_int8(0x16);
3736   emit_operand(src, dst);
3737   emit_int8(imm8);
3738 }
3739 
3740 void Assembler::pextrq(Register dst, XMMRegister src, int imm8) {
3741   assert(VM_Version::supports_sse4_1(), "");
3742   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
3743   int encode = simd_prefix_and_encode(src, xnoreg, as_XMMRegister(dst->encoding()), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3744   emit_int8(0x16);
3745   emit_int8((unsigned char)(0xC0 | encode));
3746   emit_int8(imm8);
3747 }
3748 
3749 void Assembler::pextrq(Address dst, XMMRegister src, int imm8) {
3750   assert(VM_Version::supports_sse4_1(), "");
3751   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
3752   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
3753   simd_prefix(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3754   emit_int8(0x16);
3755   emit_operand(src, dst);
3756   emit_int8(imm8);
3757 }
3758 
3759 void Assembler::pextrw(Register dst, XMMRegister src, int imm8) {
3760   assert(VM_Version::supports_sse2(), "");
3761   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
3762   int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3763   emit_int8((unsigned char)0xC5);
3764   emit_int8((unsigned char)(0xC0 | encode));
3765   emit_int8(imm8);
3766 }
3767 
3768 void Assembler::pextrw(Address dst, XMMRegister src, int imm8) {
3769   assert(VM_Version::supports_sse4_1(), "");
3770   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
3771   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_16bit);
3772   simd_prefix(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3773   emit_int8((unsigned char)0x15);
3774   emit_operand(src, dst);
3775   emit_int8(imm8);
3776 }
3777 
3778 void Assembler::pextrb(Address dst, XMMRegister src, int imm8) {
3779   assert(VM_Version::supports_sse4_1(), "");
3780   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
3781   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_8bit);
3782   simd_prefix(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3783   emit_int8(0x14);
3784   emit_operand(src, dst);
3785   emit_int8(imm8);
3786 }
3787 
3788 void Assembler::pinsrd(XMMRegister dst, Register src, int imm8) {
3789   assert(VM_Version::supports_sse4_1(), "");
3790   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
3791   int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3792   emit_int8(0x22);
3793   emit_int8((unsigned char)(0xC0 | encode));
3794   emit_int8(imm8);
3795 }
3796 
3797 void Assembler::pinsrd(XMMRegister dst, Address src, int imm8) {
3798   assert(VM_Version::supports_sse4_1(), "");
3799   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
3800   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
3801   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3802   emit_int8(0x22);
3803   emit_operand(dst,src);
3804   emit_int8(imm8);
3805 }
3806 
3807 void Assembler::pinsrq(XMMRegister dst, Register src, int imm8) {
3808   assert(VM_Version::supports_sse4_1(), "");
3809   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
3810   int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3811   emit_int8(0x22);
3812   emit_int8((unsigned char)(0xC0 | encode));
3813   emit_int8(imm8);
3814 }
3815 
3816 void Assembler::pinsrq(XMMRegister dst, Address src, int imm8) {
3817   assert(VM_Version::supports_sse4_1(), "");
3818   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
3819   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
3820   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3821   emit_int8(0x22);
3822   emit_operand(dst, src);
3823   emit_int8(imm8);
3824 }
3825 
3826 void Assembler::pinsrw(XMMRegister dst, Register src, int imm8) {
3827   assert(VM_Version::supports_sse2(), "");
3828   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
3829   int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3830   emit_int8((unsigned char)0xC4);
3831   emit_int8((unsigned char)(0xC0 | encode));
3832   emit_int8(imm8);
3833 }
3834 
3835 void Assembler::pinsrw(XMMRegister dst, Address src, int imm8) {
3836   assert(VM_Version::supports_sse2(), "");
3837   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
3838   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_16bit);
3839   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3840   emit_int8((unsigned char)0xC4);
3841   emit_operand(dst, src);
3842   emit_int8(imm8);
3843 }
3844 
3845 void Assembler::pinsrb(XMMRegister dst, Address src, int imm8) {
3846   assert(VM_Version::supports_sse4_1(), "");
3847   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
3848   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_8bit);
3849   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3850   emit_int8(0x20);
3851   emit_operand(dst, src);
3852   emit_int8(imm8);
3853 }
3854 
3855 void Assembler::pmovzxbw(XMMRegister dst, Address src) {
3856   assert(VM_Version::supports_sse4_1(), "");
3857   InstructionMark im(this);
3858   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
3859   attributes.set_address_attributes(/* tuple_type */ EVEX_HVM, /* input_size_in_bits */ EVEX_NObit);
3860   simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3861   emit_int8(0x30);
3862   emit_operand(dst, src);
3863 }
3864 
3865 void Assembler::pmovzxbw(XMMRegister dst, XMMRegister src) {
3866   assert(VM_Version::supports_sse4_1(), "");
3867   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
3868   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3869   emit_int8(0x30);
3870   emit_int8((unsigned char)(0xC0 | encode));
3871 }
3872 
3873 void Assembler::vpmovzxbw(XMMRegister dst, Address src, int vector_len) {
3874   assert(VM_Version::supports_avx(), "");
3875   InstructionMark im(this);
3876   assert(dst != xnoreg, "sanity");
3877   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
3878   attributes.set_address_attributes(/* tuple_type */ EVEX_HVM, /* input_size_in_bits */ EVEX_NObit);
3879   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3880   emit_int8(0x30);
3881   emit_operand(dst, src);
3882 }
3883 
3884 void Assembler::evpmovzxbw(XMMRegister dst, KRegister mask, Address src, int vector_len) {
3885   assert(is_vector_masking(), "");
3886   assert(VM_Version::supports_avx512vlbw(), "");
3887   assert(dst != xnoreg, "sanity");
3888   InstructionMark im(this);
3889   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
3890   attributes.set_address_attributes(/* tuple_type */ EVEX_HVM, /* input_size_in_bits */ EVEX_NObit);
3891   attributes.set_embedded_opmask_register_specifier(mask);
3892   attributes.set_is_evex_instruction();
3893   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3894   emit_int8(0x30);
3895   emit_operand(dst, src);
3896 }
3897 
3898 void Assembler::evpmovwb(Address dst, XMMRegister src, int vector_len) {
3899   assert(VM_Version::supports_avx512vlbw(), "");
3900   assert(src != xnoreg, "sanity");
3901   InstructionMark im(this);
3902   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3903   attributes.set_address_attributes(/* tuple_type */ EVEX_HVM, /* input_size_in_bits */ EVEX_NObit);
3904   attributes.set_is_evex_instruction();
3905   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
3906   emit_int8(0x30);
3907   emit_operand(src, dst);
3908 }
3909 
3910 void Assembler::evpmovwb(Address dst, KRegister mask, XMMRegister src, int vector_len) {
3911   assert(is_vector_masking(), "");
3912   assert(VM_Version::supports_avx512vlbw(), "");
3913   assert(src != xnoreg, "sanity");
3914   InstructionMark im(this);
3915   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
3916   attributes.set_address_attributes(/* tuple_type */ EVEX_HVM, /* input_size_in_bits */ EVEX_NObit);
3917   attributes.reset_is_clear_context();
3918   attributes.set_embedded_opmask_register_specifier(mask);
3919   attributes.set_is_evex_instruction();
3920   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
3921   emit_int8(0x30);
3922   emit_operand(src, dst);
3923 }
3924 
3925 // generic
3926 void Assembler::pop(Register dst) {
3927   int encode = prefix_and_encode(dst->encoding());
3928   emit_int8(0x58 | encode);
3929 }
3930 
3931 void Assembler::popcntl(Register dst, Address src) {
3932   assert(VM_Version::supports_popcnt(), "must support");
3933   InstructionMark im(this);
3934   emit_int8((unsigned char)0xF3);
3935   prefix(src, dst);
3936   emit_int8(0x0F);
3937   emit_int8((unsigned char)0xB8);
3938   emit_operand(dst, src);
3939 }
3940 
3941 void Assembler::popcntl(Register dst, Register src) {
3942   assert(VM_Version::supports_popcnt(), "must support");
3943   emit_int8((unsigned char)0xF3);
3944   int encode = prefix_and_encode(dst->encoding(), src->encoding());
3945   emit_int8(0x0F);
3946   emit_int8((unsigned char)0xB8);
3947   emit_int8((unsigned char)(0xC0 | encode));
3948 }
3949 
3950 void Assembler::popf() {
3951   emit_int8((unsigned char)0x9D);
3952 }
3953 
3954 #ifndef _LP64 // no 32bit push/pop on amd64
3955 void Assembler::popl(Address dst) {
3956   // NOTE: this will adjust stack by 8byte on 64bits
3957   InstructionMark im(this);
3958   prefix(dst);
3959   emit_int8((unsigned char)0x8F);
3960   emit_operand(rax, dst);
3961 }
3962 #endif
3963 
3964 void Assembler::prefetch_prefix(Address src) {
3965   prefix(src);
3966   emit_int8(0x0F);
3967 }
3968 
3969 void Assembler::prefetchnta(Address src) {
3970   NOT_LP64(assert(VM_Version::supports_sse(), "must support"));
3971   InstructionMark im(this);
3972   prefetch_prefix(src);
3973   emit_int8(0x18);
3974   emit_operand(rax, src); // 0, src
3975 }
3976 
3977 void Assembler::prefetchr(Address src) {
3978   assert(VM_Version::supports_3dnow_prefetch(), "must support");
3979   InstructionMark im(this);
3980   prefetch_prefix(src);
3981   emit_int8(0x0D);
3982   emit_operand(rax, src); // 0, src
3983 }
3984 
3985 void Assembler::prefetcht0(Address src) {
3986   NOT_LP64(assert(VM_Version::supports_sse(), "must support"));
3987   InstructionMark im(this);
3988   prefetch_prefix(src);
3989   emit_int8(0x18);
3990   emit_operand(rcx, src); // 1, src
3991 }
3992 
3993 void Assembler::prefetcht1(Address src) {
3994   NOT_LP64(assert(VM_Version::supports_sse(), "must support"));
3995   InstructionMark im(this);
3996   prefetch_prefix(src);
3997   emit_int8(0x18);
3998   emit_operand(rdx, src); // 2, src
3999 }
4000 
4001 void Assembler::prefetcht2(Address src) {
4002   NOT_LP64(assert(VM_Version::supports_sse(), "must support"));
4003   InstructionMark im(this);
4004   prefetch_prefix(src);
4005   emit_int8(0x18);
4006   emit_operand(rbx, src); // 3, src
4007 }
4008 
4009 void Assembler::prefetchw(Address src) {
4010   assert(VM_Version::supports_3dnow_prefetch(), "must support");
4011   InstructionMark im(this);
4012   prefetch_prefix(src);
4013   emit_int8(0x0D);
4014   emit_operand(rcx, src); // 1, src
4015 }
4016 
4017 void Assembler::prefix(Prefix p) {
4018   emit_int8(p);
4019 }
4020 
4021 void Assembler::pshufb(XMMRegister dst, XMMRegister src) {
4022   assert(VM_Version::supports_ssse3(), "");
4023   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
4024   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4025   emit_int8(0x00);
4026   emit_int8((unsigned char)(0xC0 | encode));
4027 }
4028 
4029 void Assembler::vpshufb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4030   assert(vector_len == AVX_128bit? VM_Version::supports_avx() :
4031          vector_len == AVX_256bit? VM_Version::supports_avx2() :
4032          0, "");
4033   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
4034   int encode = simd_prefix_and_encode(dst, nds, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4035   emit_int8(0x00);
4036   emit_int8((unsigned char)(0xC0 | encode));
4037 }
4038 
4039 void Assembler::pshufb(XMMRegister dst, Address src) {
4040   assert(VM_Version::supports_ssse3(), "");
4041   InstructionMark im(this);
4042   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
4043   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
4044   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4045   emit_int8(0x00);
4046   emit_operand(dst, src);
4047 }
4048 
4049 void Assembler::pshufd(XMMRegister dst, XMMRegister src, int mode) {
4050   assert(isByte(mode), "invalid value");
4051   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4052   int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_128bit;
4053   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4054   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4055   emit_int8(0x70);
4056   emit_int8((unsigned char)(0xC0 | encode));
4057   emit_int8(mode & 0xFF);
4058 }
4059 
4060 void Assembler::vpshufd(XMMRegister dst, XMMRegister src, int mode, int vector_len) {
4061   assert(vector_len == AVX_128bit? VM_Version::supports_avx() :
4062          vector_len == AVX_256bit? VM_Version::supports_avx2() :
4063          0, "");
4064   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4065   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4066   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4067   emit_int8(0x70);
4068   emit_int8((unsigned char)(0xC0 | encode));
4069   emit_int8(mode & 0xFF);
4070 }
4071 
4072 void Assembler::pshufd(XMMRegister dst, Address src, int mode) {
4073   assert(isByte(mode), "invalid value");
4074   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4075   assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
4076   InstructionMark im(this);
4077   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4078   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
4079   simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4080   emit_int8(0x70);
4081   emit_operand(dst, src);
4082   emit_int8(mode & 0xFF);
4083 }
4084 
4085 void Assembler::pshuflw(XMMRegister dst, XMMRegister src, int mode) {
4086   assert(isByte(mode), "invalid value");
4087   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4088   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
4089   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
4090   emit_int8(0x70);
4091   emit_int8((unsigned char)(0xC0 | encode));
4092   emit_int8(mode & 0xFF);
4093 }
4094 
4095 void Assembler::pshuflw(XMMRegister dst, Address src, int mode) {
4096   assert(isByte(mode), "invalid value");
4097   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4098   assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
4099   InstructionMark im(this);
4100   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
4101   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
4102   simd_prefix(dst, xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
4103   emit_int8(0x70);
4104   emit_operand(dst, src);
4105   emit_int8(mode & 0xFF);
4106 }
4107 
4108 void Assembler::psrldq(XMMRegister dst, int shift) {
4109   // Shift left 128 bit value in dst XMMRegister by shift number of bytes.
4110   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4111   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
4112   int encode = simd_prefix_and_encode(xmm3, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4113   emit_int8(0x73);
4114   emit_int8((unsigned char)(0xC0 | encode));
4115   emit_int8(shift);
4116 }
4117 
4118 void Assembler::pslldq(XMMRegister dst, int shift) {
4119   // Shift left 128 bit value in dst XMMRegister by shift number of bytes.
4120   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4121   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
4122   // XMM7 is for /7 encoding: 66 0F 73 /7 ib
4123   int encode = simd_prefix_and_encode(xmm7, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4124   emit_int8(0x73);
4125   emit_int8((unsigned char)(0xC0 | encode));
4126   emit_int8(shift);
4127 }
4128 
4129 void Assembler::ptest(XMMRegister dst, Address src) {
4130   assert(VM_Version::supports_sse4_1(), "");
4131   assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
4132   InstructionMark im(this);
4133   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
4134   simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4135   emit_int8(0x17);
4136   emit_operand(dst, src);
4137 }
4138 
4139 void Assembler::ptest(XMMRegister dst, XMMRegister src) {
4140   assert(VM_Version::supports_sse4_1(), "");
4141   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
4142   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4143   emit_int8(0x17);
4144   emit_int8((unsigned char)(0xC0 | encode));
4145 }
4146 
4147 void Assembler::vptest(XMMRegister dst, Address src) {
4148   assert(VM_Version::supports_avx(), "");
4149   InstructionMark im(this);
4150   InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
4151   assert(dst != xnoreg, "sanity");
4152   // swap src<->dst for encoding
4153   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4154   emit_int8(0x17);
4155   emit_operand(dst, src);
4156 }
4157 
4158 void Assembler::vptest(XMMRegister dst, XMMRegister src) {
4159   assert(VM_Version::supports_avx(), "");
4160   InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
4161   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4162   emit_int8(0x17);
4163   emit_int8((unsigned char)(0xC0 | encode));
4164 }
4165 
4166 void Assembler::punpcklbw(XMMRegister dst, Address src) {
4167   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4168   assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
4169   InstructionMark im(this);
4170   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_vlbw, /* no_mask_reg */ false, /* uses_vl */ true);
4171   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
4172   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4173   emit_int8(0x60);
4174   emit_operand(dst, src);
4175 }
4176 
4177 void Assembler::punpcklbw(XMMRegister dst, XMMRegister src) {
4178   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4179   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_vlbw, /* no_mask_reg */ false, /* uses_vl */ true);
4180   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4181   emit_int8(0x60);
4182   emit_int8((unsigned char)(0xC0 | encode));
4183 }
4184 
4185 void Assembler::punpckldq(XMMRegister dst, Address src) {
4186   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4187   assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
4188   InstructionMark im(this);
4189   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4190   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
4191   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4192   emit_int8(0x62);
4193   emit_operand(dst, src);
4194 }
4195 
4196 void Assembler::punpckldq(XMMRegister dst, XMMRegister src) {
4197   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4198   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4199   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4200   emit_int8(0x62);
4201   emit_int8((unsigned char)(0xC0 | encode));
4202 }
4203 
4204 void Assembler::punpcklqdq(XMMRegister dst, XMMRegister src) {
4205   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4206   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4207   attributes.set_rex_vex_w_reverted();
4208   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4209   emit_int8(0x6C);
4210   emit_int8((unsigned char)(0xC0 | encode));
4211 }
4212 
4213 void Assembler::push(int32_t imm32) {
4214   // in 64bits we push 64bits onto the stack but only
4215   // take a 32bit immediate
4216   emit_int8(0x68);
4217   emit_int32(imm32);
4218 }
4219 
4220 void Assembler::push(Register src) {
4221   int encode = prefix_and_encode(src->encoding());
4222 
4223   emit_int8(0x50 | encode);
4224 }
4225 
4226 void Assembler::pushf() {
4227   emit_int8((unsigned char)0x9C);
4228 }
4229 
4230 #ifndef _LP64 // no 32bit push/pop on amd64
4231 void Assembler::pushl(Address src) {
4232   // Note this will push 64bit on 64bit
4233   InstructionMark im(this);
4234   prefix(src);
4235   emit_int8((unsigned char)0xFF);
4236   emit_operand(rsi, src);
4237 }
4238 #endif
4239 
4240 void Assembler::rcll(Register dst, int imm8) {
4241   assert(isShiftCount(imm8), "illegal shift count");
4242   int encode = prefix_and_encode(dst->encoding());
4243   if (imm8 == 1) {
4244     emit_int8((unsigned char)0xD1);
4245     emit_int8((unsigned char)(0xD0 | encode));
4246   } else {
4247     emit_int8((unsigned char)0xC1);
4248     emit_int8((unsigned char)0xD0 | encode);
4249     emit_int8(imm8);
4250   }
4251 }
4252 
4253 void Assembler::rcpps(XMMRegister dst, XMMRegister src) {
4254   NOT_LP64(assert(VM_Version::supports_sse(), ""));
4255   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
4256   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
4257   emit_int8(0x53);
4258   emit_int8((unsigned char)(0xC0 | encode));
4259 }
4260 
4261 void Assembler::rcpss(XMMRegister dst, XMMRegister src) {
4262   NOT_LP64(assert(VM_Version::supports_sse(), ""));
4263   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
4264   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
4265   emit_int8(0x53);
4266   emit_int8((unsigned char)(0xC0 | encode));
4267 }
4268 
4269 void Assembler::rdtsc() {
4270   emit_int8((unsigned char)0x0F);
4271   emit_int8((unsigned char)0x31);
4272 }
4273 
4274 // copies data from [esi] to [edi] using rcx pointer sized words
4275 // generic
4276 void Assembler::rep_mov() {
4277   emit_int8((unsigned char)0xF3);
4278   // MOVSQ
4279   LP64_ONLY(prefix(REX_W));
4280   emit_int8((unsigned char)0xA5);
4281 }
4282 
4283 // sets rcx bytes with rax, value at [edi]
4284 void Assembler::rep_stosb() {
4285   emit_int8((unsigned char)0xF3); // REP
4286   LP64_ONLY(prefix(REX_W));
4287   emit_int8((unsigned char)0xAA); // STOSB
4288 }
4289 
4290 // sets rcx pointer sized words with rax, value at [edi]
4291 // generic
4292 void Assembler::rep_stos() {
4293   emit_int8((unsigned char)0xF3); // REP
4294   LP64_ONLY(prefix(REX_W));       // LP64:STOSQ, LP32:STOSD
4295   emit_int8((unsigned char)0xAB);
4296 }
4297 
4298 // scans rcx pointer sized words at [edi] for occurance of rax,
4299 // generic
4300 void Assembler::repne_scan() { // repne_scan
4301   emit_int8((unsigned char)0xF2);
4302   // SCASQ
4303   LP64_ONLY(prefix(REX_W));
4304   emit_int8((unsigned char)0xAF);
4305 }
4306 
4307 #ifdef _LP64
4308 // scans rcx 4 byte words at [edi] for occurance of rax,
4309 // generic
4310 void Assembler::repne_scanl() { // repne_scan
4311   emit_int8((unsigned char)0xF2);
4312   // SCASL
4313   emit_int8((unsigned char)0xAF);
4314 }
4315 #endif
4316 
4317 void Assembler::ret(int imm16) {
4318   if (imm16 == 0) {
4319     emit_int8((unsigned char)0xC3);
4320   } else {
4321     emit_int8((unsigned char)0xC2);
4322     emit_int16(imm16);
4323   }
4324 }
4325 
4326 void Assembler::sahf() {
4327 #ifdef _LP64
4328   // Not supported in 64bit mode
4329   ShouldNotReachHere();
4330 #endif
4331   emit_int8((unsigned char)0x9E);
4332 }
4333 
4334 void Assembler::sarl(Register dst, int imm8) {
4335   int encode = prefix_and_encode(dst->encoding());
4336   assert(isShiftCount(imm8), "illegal shift count");
4337   if (imm8 == 1) {
4338     emit_int8((unsigned char)0xD1);
4339     emit_int8((unsigned char)(0xF8 | encode));
4340   } else {
4341     emit_int8((unsigned char)0xC1);
4342     emit_int8((unsigned char)(0xF8 | encode));
4343     emit_int8(imm8);
4344   }
4345 }
4346 
4347 void Assembler::sarl(Register dst) {
4348   int encode = prefix_and_encode(dst->encoding());
4349   emit_int8((unsigned char)0xD3);
4350   emit_int8((unsigned char)(0xF8 | encode));
4351 }
4352 
4353 void Assembler::sbbl(Address dst, int32_t imm32) {
4354   InstructionMark im(this);
4355   prefix(dst);
4356   emit_arith_operand(0x81, rbx, dst, imm32);
4357 }
4358 
4359 void Assembler::sbbl(Register dst, int32_t imm32) {
4360   prefix(dst);
4361   emit_arith(0x81, 0xD8, dst, imm32);
4362 }
4363 
4364 
4365 void Assembler::sbbl(Register dst, Address src) {
4366   InstructionMark im(this);
4367   prefix(src, dst);
4368   emit_int8(0x1B);
4369   emit_operand(dst, src);
4370 }
4371 
4372 void Assembler::sbbl(Register dst, Register src) {
4373   (void) prefix_and_encode(dst->encoding(), src->encoding());
4374   emit_arith(0x1B, 0xC0, dst, src);
4375 }
4376 
4377 void Assembler::setb(Condition cc, Register dst) {
4378   assert(0 <= cc && cc < 16, "illegal cc");
4379   int encode = prefix_and_encode(dst->encoding(), true);
4380   emit_int8(0x0F);
4381   emit_int8((unsigned char)0x90 | cc);
4382   emit_int8((unsigned char)(0xC0 | encode));
4383 }
4384 
4385 void Assembler::palignr(XMMRegister dst, XMMRegister src, int imm8) {
4386   assert(VM_Version::supports_ssse3(), "");
4387   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ false);
4388   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
4389   emit_int8((unsigned char)0x0F);
4390   emit_int8((unsigned char)(0xC0 | encode));
4391   emit_int8(imm8);
4392 }
4393 
4394 void Assembler::vpalignr(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8, int vector_len) {
4395   assert(vector_len == AVX_128bit? VM_Version::supports_avx() :
4396          vector_len == AVX_256bit? VM_Version::supports_avx2() :
4397          0, "");
4398   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
4399   int encode = simd_prefix_and_encode(dst, nds, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
4400   emit_int8((unsigned char)0x0F);
4401   emit_int8((unsigned char)(0xC0 | encode));
4402   emit_int8(imm8);
4403 }
4404 
4405 void Assembler::pblendw(XMMRegister dst, XMMRegister src, int imm8) {
4406   assert(VM_Version::supports_sse4_1(), "");
4407   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
4408   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
4409   emit_int8((unsigned char)0x0E);
4410   emit_int8((unsigned char)(0xC0 | encode));
4411   emit_int8(imm8);
4412 }
4413 
4414 void Assembler::sha1rnds4(XMMRegister dst, XMMRegister src, int imm8) {
4415   assert(VM_Version::supports_sha(), "");
4416   int encode = rex_prefix_and_encode(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3A, /* rex_w */ false);
4417   emit_int8((unsigned char)0xCC);
4418   emit_int8((unsigned char)(0xC0 | encode));
4419   emit_int8((unsigned char)imm8);
4420 }
4421 
4422 void Assembler::sha1nexte(XMMRegister dst, XMMRegister src) {
4423   assert(VM_Version::supports_sha(), "");
4424   int encode = rex_prefix_and_encode(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, /* rex_w */ false);
4425   emit_int8((unsigned char)0xC8);
4426   emit_int8((unsigned char)(0xC0 | encode));
4427 }
4428 
4429 void Assembler::sha1msg1(XMMRegister dst, XMMRegister src) {
4430   assert(VM_Version::supports_sha(), "");
4431   int encode = rex_prefix_and_encode(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, /* rex_w */ false);
4432   emit_int8((unsigned char)0xC9);
4433   emit_int8((unsigned char)(0xC0 | encode));
4434 }
4435 
4436 void Assembler::sha1msg2(XMMRegister dst, XMMRegister src) {
4437   assert(VM_Version::supports_sha(), "");
4438   int encode = rex_prefix_and_encode(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, /* rex_w */ false);
4439   emit_int8((unsigned char)0xCA);
4440   emit_int8((unsigned char)(0xC0 | encode));
4441 }
4442 
4443 // xmm0 is implicit additional source to this instruction.
4444 void Assembler::sha256rnds2(XMMRegister dst, XMMRegister src) {
4445   assert(VM_Version::supports_sha(), "");
4446   int encode = rex_prefix_and_encode(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, /* rex_w */ false);
4447   emit_int8((unsigned char)0xCB);
4448   emit_int8((unsigned char)(0xC0 | encode));
4449 }
4450 
4451 void Assembler::sha256msg1(XMMRegister dst, XMMRegister src) {
4452   assert(VM_Version::supports_sha(), "");
4453   int encode = rex_prefix_and_encode(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, /* rex_w */ false);
4454   emit_int8((unsigned char)0xCC);
4455   emit_int8((unsigned char)(0xC0 | encode));
4456 }
4457 
4458 void Assembler::sha256msg2(XMMRegister dst, XMMRegister src) {
4459   assert(VM_Version::supports_sha(), "");
4460   int encode = rex_prefix_and_encode(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, /* rex_w */ false);
4461   emit_int8((unsigned char)0xCD);
4462   emit_int8((unsigned char)(0xC0 | encode));
4463 }
4464 
4465 
4466 void Assembler::shll(Register dst, int imm8) {
4467   assert(isShiftCount(imm8), "illegal shift count");
4468   int encode = prefix_and_encode(dst->encoding());
4469   if (imm8 == 1 ) {
4470     emit_int8((unsigned char)0xD1);
4471     emit_int8((unsigned char)(0xE0 | encode));
4472   } else {
4473     emit_int8((unsigned char)0xC1);
4474     emit_int8((unsigned char)(0xE0 | encode));
4475     emit_int8(imm8);
4476   }
4477 }
4478 
4479 void Assembler::shll(Register dst) {
4480   int encode = prefix_and_encode(dst->encoding());
4481   emit_int8((unsigned char)0xD3);
4482   emit_int8((unsigned char)(0xE0 | encode));
4483 }
4484 
4485 void Assembler::shrl(Register dst, int imm8) {
4486   assert(isShiftCount(imm8), "illegal shift count");
4487   int encode = prefix_and_encode(dst->encoding());
4488   emit_int8((unsigned char)0xC1);
4489   emit_int8((unsigned char)(0xE8 | encode));
4490   emit_int8(imm8);
4491 }
4492 
4493 void Assembler::shrl(Register dst) {
4494   int encode = prefix_and_encode(dst->encoding());
4495   emit_int8((unsigned char)0xD3);
4496   emit_int8((unsigned char)(0xE8 | encode));
4497 }
4498 
4499 // copies a single word from [esi] to [edi]
4500 void Assembler::smovl() {
4501   emit_int8((unsigned char)0xA5);
4502 }
4503 
4504 void Assembler::sqrtsd(XMMRegister dst, XMMRegister src) {
4505   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4506   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4507   attributes.set_rex_vex_w_reverted();
4508   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
4509   emit_int8(0x51);
4510   emit_int8((unsigned char)(0xC0 | encode));
4511 }
4512 
4513 void Assembler::sqrtsd(XMMRegister dst, Address src) {
4514   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4515   InstructionMark im(this);
4516   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4517   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
4518   attributes.set_rex_vex_w_reverted();
4519   simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
4520   emit_int8(0x51);
4521   emit_operand(dst, src);
4522 }
4523 
4524 void Assembler::sqrtss(XMMRegister dst, XMMRegister src) {
4525   NOT_LP64(assert(VM_Version::supports_sse(), ""));
4526   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4527   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
4528   emit_int8(0x51);
4529   emit_int8((unsigned char)(0xC0 | encode));
4530 }
4531 
4532 void Assembler::std() {
4533   emit_int8((unsigned char)0xFD);
4534 }
4535 
4536 void Assembler::sqrtss(XMMRegister dst, Address src) {
4537   NOT_LP64(assert(VM_Version::supports_sse(), ""));
4538   InstructionMark im(this);
4539   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4540   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
4541   simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
4542   emit_int8(0x51);
4543   emit_operand(dst, src);
4544 }
4545 
4546 void Assembler::stmxcsr( Address dst) {
4547   if (UseAVX > 0 ) {
4548     assert(VM_Version::supports_avx(), "");
4549     InstructionMark im(this);
4550     InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
4551     vex_prefix(dst, 0, 0, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
4552     emit_int8((unsigned char)0xAE);
4553     emit_operand(as_Register(3), dst);
4554   } else {
4555     NOT_LP64(assert(VM_Version::supports_sse(), ""));
4556     InstructionMark im(this);
4557     prefix(dst);
4558     emit_int8(0x0F);
4559     emit_int8((unsigned char)0xAE);
4560     emit_operand(as_Register(3), dst);
4561   }
4562 }
4563 
4564 void Assembler::subl(Address dst, int32_t imm32) {
4565   InstructionMark im(this);
4566   prefix(dst);
4567   emit_arith_operand(0x81, rbp, dst, imm32);
4568 }
4569 
4570 void Assembler::subl(Address dst, Register src) {
4571   InstructionMark im(this);
4572   prefix(dst, src);
4573   emit_int8(0x29);
4574   emit_operand(src, dst);
4575 }
4576 
4577 void Assembler::subl(Register dst, int32_t imm32) {
4578   prefix(dst);
4579   emit_arith(0x81, 0xE8, dst, imm32);
4580 }
4581 
4582 // Force generation of a 4 byte immediate value even if it fits into 8bit
4583 void Assembler::subl_imm32(Register dst, int32_t imm32) {
4584   prefix(dst);
4585   emit_arith_imm32(0x81, 0xE8, dst, imm32);
4586 }
4587 
4588 void Assembler::subl(Register dst, Address src) {
4589   InstructionMark im(this);
4590   prefix(src, dst);
4591   emit_int8(0x2B);
4592   emit_operand(dst, src);
4593 }
4594 
4595 void Assembler::subl(Register dst, Register src) {
4596   (void) prefix_and_encode(dst->encoding(), src->encoding());
4597   emit_arith(0x2B, 0xC0, dst, src);
4598 }
4599 
4600 void Assembler::subsd(XMMRegister dst, XMMRegister src) {
4601   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4602   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4603   attributes.set_rex_vex_w_reverted();
4604   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
4605   emit_int8(0x5C);
4606   emit_int8((unsigned char)(0xC0 | encode));
4607 }
4608 
4609 void Assembler::subsd(XMMRegister dst, Address src) {
4610   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4611   InstructionMark im(this);
4612   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4613   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
4614   attributes.set_rex_vex_w_reverted();
4615   simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
4616   emit_int8(0x5C);
4617   emit_operand(dst, src);
4618 }
4619 
4620 void Assembler::subss(XMMRegister dst, XMMRegister src) {
4621   NOT_LP64(assert(VM_Version::supports_sse(), ""));
4622   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true , /* uses_vl */ false);
4623   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
4624   emit_int8(0x5C);
4625   emit_int8((unsigned char)(0xC0 | encode));
4626 }
4627 
4628 void Assembler::subss(XMMRegister dst, Address src) {
4629   NOT_LP64(assert(VM_Version::supports_sse(), ""));
4630   InstructionMark im(this);
4631   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4632   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
4633   simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
4634   emit_int8(0x5C);
4635   emit_operand(dst, src);
4636 }
4637 
4638 void Assembler::testb(Register dst, int imm8) {
4639   NOT_LP64(assert(dst->has_byte_register(), "must have byte register"));
4640   (void) prefix_and_encode(dst->encoding(), true);
4641   emit_arith_b(0xF6, 0xC0, dst, imm8);
4642 }
4643 
4644 void Assembler::testb(Address dst, int imm8) {
4645   InstructionMark im(this);
4646   prefix(dst);
4647   emit_int8((unsigned char)0xF6);
4648   emit_operand(rax, dst, 1);
4649   emit_int8(imm8);
4650 }
4651 
4652 void Assembler::testl(Register dst, int32_t imm32) {
4653   // not using emit_arith because test
4654   // doesn't support sign-extension of
4655   // 8bit operands
4656   int encode = dst->encoding();
4657   if (encode == 0) {
4658     emit_int8((unsigned char)0xA9);
4659   } else {
4660     encode = prefix_and_encode(encode);
4661     emit_int8((unsigned char)0xF7);
4662     emit_int8((unsigned char)(0xC0 | encode));
4663   }
4664   emit_int32(imm32);
4665 }
4666 
4667 void Assembler::testl(Register dst, Register src) {
4668   (void) prefix_and_encode(dst->encoding(), src->encoding());
4669   emit_arith(0x85, 0xC0, dst, src);
4670 }
4671 
4672 void Assembler::testl(Register dst, Address src) {
4673   InstructionMark im(this);
4674   prefix(src, dst);
4675   emit_int8((unsigned char)0x85);
4676   emit_operand(dst, src);
4677 }
4678 
4679 void Assembler::tzcntl(Register dst, Register src) {
4680   assert(VM_Version::supports_bmi1(), "tzcnt instruction not supported");
4681   emit_int8((unsigned char)0xF3);
4682   int encode = prefix_and_encode(dst->encoding(), src->encoding());
4683   emit_int8(0x0F);
4684   emit_int8((unsigned char)0xBC);
4685   emit_int8((unsigned char)0xC0 | encode);
4686 }
4687 
4688 void Assembler::tzcntq(Register dst, Register src) {
4689   assert(VM_Version::supports_bmi1(), "tzcnt instruction not supported");
4690   emit_int8((unsigned char)0xF3);
4691   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
4692   emit_int8(0x0F);
4693   emit_int8((unsigned char)0xBC);
4694   emit_int8((unsigned char)(0xC0 | encode));
4695 }
4696 
4697 void Assembler::ucomisd(XMMRegister dst, Address src) {
4698   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4699   InstructionMark im(this);
4700   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4701   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
4702   attributes.set_rex_vex_w_reverted();
4703   simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4704   emit_int8(0x2E);
4705   emit_operand(dst, src);
4706 }
4707 
4708 void Assembler::ucomisd(XMMRegister dst, XMMRegister src) {
4709   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4710   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4711   attributes.set_rex_vex_w_reverted();
4712   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4713   emit_int8(0x2E);
4714   emit_int8((unsigned char)(0xC0 | encode));
4715 }
4716 
4717 void Assembler::ucomiss(XMMRegister dst, Address src) {
4718   NOT_LP64(assert(VM_Version::supports_sse(), ""));
4719   InstructionMark im(this);
4720   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4721   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
4722   simd_prefix(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
4723   emit_int8(0x2E);
4724   emit_operand(dst, src);
4725 }
4726 
4727 void Assembler::ucomiss(XMMRegister dst, XMMRegister src) {
4728   NOT_LP64(assert(VM_Version::supports_sse(), ""));
4729   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4730   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
4731   emit_int8(0x2E);
4732   emit_int8((unsigned char)(0xC0 | encode));
4733 }
4734 
4735 void Assembler::xabort(int8_t imm8) {
4736   emit_int8((unsigned char)0xC6);
4737   emit_int8((unsigned char)0xF8);
4738   emit_int8((unsigned char)(imm8 & 0xFF));
4739 }
4740 
4741 void Assembler::xaddb(Address dst, Register src) {
4742   InstructionMark im(this);
4743   prefix(dst, src, true);
4744   emit_int8(0x0F);
4745   emit_int8((unsigned char)0xC0);
4746   emit_operand(src, dst);
4747 }
4748 
4749 void Assembler::xaddw(Address dst, Register src) {
4750   InstructionMark im(this);
4751   emit_int8(0x66);
4752   prefix(dst, src);
4753   emit_int8(0x0F);
4754   emit_int8((unsigned char)0xC1);
4755   emit_operand(src, dst);
4756 }
4757 
4758 void Assembler::xaddl(Address dst, Register src) {
4759   InstructionMark im(this);
4760   prefix(dst, src);
4761   emit_int8(0x0F);
4762   emit_int8((unsigned char)0xC1);
4763   emit_operand(src, dst);
4764 }
4765 
4766 void Assembler::xbegin(Label& abort, relocInfo::relocType rtype) {
4767   InstructionMark im(this);
4768   relocate(rtype);
4769   if (abort.is_bound()) {
4770     address entry = target(abort);
4771     assert(entry != NULL, "abort entry NULL");
4772     intptr_t offset = entry - pc();
4773     emit_int8((unsigned char)0xC7);
4774     emit_int8((unsigned char)0xF8);
4775     emit_int32(offset - 6); // 2 opcode + 4 address
4776   } else {
4777     abort.add_patch_at(code(), locator());
4778     emit_int8((unsigned char)0xC7);
4779     emit_int8((unsigned char)0xF8);
4780     emit_int32(0);
4781   }
4782 }
4783 
4784 void Assembler::xchgb(Register dst, Address src) { // xchg
4785   InstructionMark im(this);
4786   prefix(src, dst, true);
4787   emit_int8((unsigned char)0x86);
4788   emit_operand(dst, src);
4789 }
4790 
4791 void Assembler::xchgw(Register dst, Address src) { // xchg
4792   InstructionMark im(this);
4793   emit_int8(0x66);
4794   prefix(src, dst);
4795   emit_int8((unsigned char)0x87);
4796   emit_operand(dst, src);
4797 }
4798 
4799 void Assembler::xchgl(Register dst, Address src) { // xchg
4800   InstructionMark im(this);
4801   prefix(src, dst);
4802   emit_int8((unsigned char)0x87);
4803   emit_operand(dst, src);
4804 }
4805 
4806 void Assembler::xchgl(Register dst, Register src) {
4807   int encode = prefix_and_encode(dst->encoding(), src->encoding());
4808   emit_int8((unsigned char)0x87);
4809   emit_int8((unsigned char)(0xC0 | encode));
4810 }
4811 
4812 void Assembler::xend() {
4813   emit_int8((unsigned char)0x0F);
4814   emit_int8((unsigned char)0x01);
4815   emit_int8((unsigned char)0xD5);
4816 }
4817 
4818 void Assembler::xgetbv() {
4819   emit_int8(0x0F);
4820   emit_int8(0x01);
4821   emit_int8((unsigned char)0xD0);
4822 }
4823 
4824 void Assembler::xorl(Register dst, int32_t imm32) {
4825   prefix(dst);
4826   emit_arith(0x81, 0xF0, dst, imm32);
4827 }
4828 
4829 void Assembler::xorl(Register dst, Address src) {
4830   InstructionMark im(this);
4831   prefix(src, dst);
4832   emit_int8(0x33);
4833   emit_operand(dst, src);
4834 }
4835 
4836 void Assembler::xorl(Register dst, Register src) {
4837   (void) prefix_and_encode(dst->encoding(), src->encoding());
4838   emit_arith(0x33, 0xC0, dst, src);
4839 }
4840 
4841 void Assembler::xorb(Register dst, Address src) {
4842   InstructionMark im(this);
4843   prefix(src, dst);
4844   emit_int8(0x32);
4845   emit_operand(dst, src);
4846 }
4847 
4848 // AVX 3-operands scalar float-point arithmetic instructions
4849 
4850 void Assembler::vaddsd(XMMRegister dst, XMMRegister nds, Address src) {
4851   assert(VM_Version::supports_avx(), "");
4852   InstructionMark im(this);
4853   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4854   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
4855   attributes.set_rex_vex_w_reverted();
4856   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
4857   emit_int8(0x58);
4858   emit_operand(dst, src);
4859 }
4860 
4861 void Assembler::vaddsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
4862   assert(VM_Version::supports_avx(), "");
4863   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4864   attributes.set_rex_vex_w_reverted();
4865   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
4866   emit_int8(0x58);
4867   emit_int8((unsigned char)(0xC0 | encode));
4868 }
4869 
4870 void Assembler::vaddss(XMMRegister dst, XMMRegister nds, Address src) {
4871   assert(VM_Version::supports_avx(), "");
4872   InstructionMark im(this);
4873   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4874   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
4875   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
4876   emit_int8(0x58);
4877   emit_operand(dst, src);
4878 }
4879 
4880 void Assembler::vaddss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
4881   assert(VM_Version::supports_avx(), "");
4882   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4883   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
4884   emit_int8(0x58);
4885   emit_int8((unsigned char)(0xC0 | encode));
4886 }
4887 
4888 void Assembler::vdivsd(XMMRegister dst, XMMRegister nds, Address src) {
4889   assert(VM_Version::supports_avx(), "");
4890   InstructionMark im(this);
4891   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4892   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
4893   attributes.set_rex_vex_w_reverted();
4894   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
4895   emit_int8(0x5E);
4896   emit_operand(dst, src);
4897 }
4898 
4899 void Assembler::vdivsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
4900   assert(VM_Version::supports_avx(), "");
4901   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4902   attributes.set_rex_vex_w_reverted();
4903   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
4904   emit_int8(0x5E);
4905   emit_int8((unsigned char)(0xC0 | encode));
4906 }
4907 
4908 void Assembler::vdivss(XMMRegister dst, XMMRegister nds, Address src) {
4909   assert(VM_Version::supports_avx(), "");
4910   InstructionMark im(this);
4911   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4912   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
4913   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
4914   emit_int8(0x5E);
4915   emit_operand(dst, src);
4916 }
4917 
4918 void Assembler::vdivss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
4919   assert(VM_Version::supports_avx(), "");
4920   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4921   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
4922   emit_int8(0x5E);
4923   emit_int8((unsigned char)(0xC0 | encode));
4924 }
4925 
4926 void Assembler::vfmadd231sd(XMMRegister dst, XMMRegister src1, XMMRegister src2) {
4927   assert(VM_Version::supports_fma(), "");
4928   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4929   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4930   emit_int8((unsigned char)0xB9);
4931   emit_int8((unsigned char)(0xC0 | encode));
4932 }
4933 
4934 void Assembler::vfmadd231ss(XMMRegister dst, XMMRegister src1, XMMRegister src2) {
4935   assert(VM_Version::supports_fma(), "");
4936   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4937   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4938   emit_int8((unsigned char)0xB9);
4939   emit_int8((unsigned char)(0xC0 | encode));
4940 }
4941 
4942 void Assembler::vmulsd(XMMRegister dst, XMMRegister nds, Address src) {
4943   assert(VM_Version::supports_avx(), "");
4944   InstructionMark im(this);
4945   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4946   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
4947   attributes.set_rex_vex_w_reverted();
4948   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
4949   emit_int8(0x59);
4950   emit_operand(dst, src);
4951 }
4952 
4953 void Assembler::vmulsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
4954   assert(VM_Version::supports_avx(), "");
4955   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4956   attributes.set_rex_vex_w_reverted();
4957   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
4958   emit_int8(0x59);
4959   emit_int8((unsigned char)(0xC0 | encode));
4960 }
4961 
4962 void Assembler::vmulss(XMMRegister dst, XMMRegister nds, Address src) {
4963   assert(VM_Version::supports_avx(), "");
4964   InstructionMark im(this);
4965   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4966   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
4967   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
4968   emit_int8(0x59);
4969   emit_operand(dst, src);
4970 }
4971 
4972 void Assembler::vmulss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
4973   assert(VM_Version::supports_avx(), "");
4974   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4975   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
4976   emit_int8(0x59);
4977   emit_int8((unsigned char)(0xC0 | encode));
4978 }
4979 
4980 void Assembler::vsubsd(XMMRegister dst, XMMRegister nds, Address src) {
4981   assert(VM_Version::supports_avx(), "");
4982   InstructionMark im(this);
4983   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4984   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
4985   attributes.set_rex_vex_w_reverted();
4986   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
4987   emit_int8(0x5C);
4988   emit_operand(dst, src);
4989 }
4990 
4991 void Assembler::vsubsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
4992   assert(VM_Version::supports_avx(), "");
4993   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4994   attributes.set_rex_vex_w_reverted();
4995   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
4996   emit_int8(0x5C);
4997   emit_int8((unsigned char)(0xC0 | encode));
4998 }
4999 
5000 void Assembler::vsubss(XMMRegister dst, XMMRegister nds, Address src) {
5001   assert(VM_Version::supports_avx(), "");
5002   InstructionMark im(this);
5003   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
5004   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
5005   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
5006   emit_int8(0x5C);
5007   emit_operand(dst, src);
5008 }
5009 
5010 void Assembler::vsubss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
5011   assert(VM_Version::supports_avx(), "");
5012   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
5013   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
5014   emit_int8(0x5C);
5015   emit_int8((unsigned char)(0xC0 | encode));
5016 }
5017 
5018 //====================VECTOR ARITHMETIC=====================================
5019 
5020 // Float-point vector arithmetic
5021 
5022 void Assembler::addpd(XMMRegister dst, XMMRegister src) {
5023   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5024   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5025   attributes.set_rex_vex_w_reverted();
5026   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5027   emit_int8(0x58);
5028   emit_int8((unsigned char)(0xC0 | encode));
5029 }
5030 
5031 void Assembler::addpd(XMMRegister dst, Address src) {
5032   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5033   InstructionMark im(this);
5034   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5035   attributes.set_rex_vex_w_reverted();
5036   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
5037   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5038   emit_int8(0x58);
5039   emit_operand(dst, src);
5040 }
5041 
5042 
5043 void Assembler::addps(XMMRegister dst, XMMRegister src) {
5044   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5045   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5046   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
5047   emit_int8(0x58);
5048   emit_int8((unsigned char)(0xC0 | encode));
5049 }
5050 
5051 void Assembler::vaddpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5052   assert(VM_Version::supports_avx(), "");
5053   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5054   attributes.set_rex_vex_w_reverted();
5055   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5056   emit_int8(0x58);
5057   emit_int8((unsigned char)(0xC0 | encode));
5058 }
5059 
5060 void Assembler::vaddps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5061   assert(VM_Version::supports_avx(), "");
5062   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5063   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
5064   emit_int8(0x58);
5065   emit_int8((unsigned char)(0xC0 | encode));
5066 }
5067 
5068 void Assembler::vaddpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5069   assert(VM_Version::supports_avx(), "");
5070   InstructionMark im(this);
5071   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5072   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
5073   attributes.set_rex_vex_w_reverted();
5074   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5075   emit_int8(0x58);
5076   emit_operand(dst, src);
5077 }
5078 
5079 void Assembler::vaddps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5080   assert(VM_Version::supports_avx(), "");
5081   InstructionMark im(this);
5082   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5083   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
5084   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
5085   emit_int8(0x58);
5086   emit_operand(dst, src);
5087 }
5088 
5089 void Assembler::subpd(XMMRegister dst, XMMRegister src) {
5090   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5091   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5092   attributes.set_rex_vex_w_reverted();
5093   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5094   emit_int8(0x5C);
5095   emit_int8((unsigned char)(0xC0 | encode));
5096 }
5097 
5098 void Assembler::subps(XMMRegister dst, XMMRegister src) {
5099   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5100   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5101   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
5102   emit_int8(0x5C);
5103   emit_int8((unsigned char)(0xC0 | encode));
5104 }
5105 
5106 void Assembler::vsubpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5107   assert(VM_Version::supports_avx(), "");
5108   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5109   attributes.set_rex_vex_w_reverted();
5110   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5111   emit_int8(0x5C);
5112   emit_int8((unsigned char)(0xC0 | encode));
5113 }
5114 
5115 void Assembler::vsubps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5116   assert(VM_Version::supports_avx(), "");
5117   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5118   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
5119   emit_int8(0x5C);
5120   emit_int8((unsigned char)(0xC0 | encode));
5121 }
5122 
5123 void Assembler::vsubpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5124   assert(VM_Version::supports_avx(), "");
5125   InstructionMark im(this);
5126   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5127   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
5128   attributes.set_rex_vex_w_reverted();
5129   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5130   emit_int8(0x5C);
5131   emit_operand(dst, src);
5132 }
5133 
5134 void Assembler::vsubps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5135   assert(VM_Version::supports_avx(), "");
5136   InstructionMark im(this);
5137   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5138   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
5139   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
5140   emit_int8(0x5C);
5141   emit_operand(dst, src);
5142 }
5143 
5144 void Assembler::mulpd(XMMRegister dst, XMMRegister src) {
5145   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5146   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5147   attributes.set_rex_vex_w_reverted();
5148   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5149   emit_int8(0x59);
5150   emit_int8((unsigned char)(0xC0 | encode));
5151 }
5152 
5153 void Assembler::mulpd(XMMRegister dst, Address src) {
5154   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5155   InstructionMark im(this);
5156   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5157   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
5158   attributes.set_rex_vex_w_reverted();
5159   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5160   emit_int8(0x59);
5161   emit_operand(dst, src);
5162 }
5163 
5164 void Assembler::mulps(XMMRegister dst, XMMRegister src) {
5165   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5166   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5167   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
5168   emit_int8(0x59);
5169   emit_int8((unsigned char)(0xC0 | encode));
5170 }
5171 
5172 void Assembler::vmulpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5173   assert(VM_Version::supports_avx(), "");
5174   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5175   attributes.set_rex_vex_w_reverted();
5176   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5177   emit_int8(0x59);
5178   emit_int8((unsigned char)(0xC0 | encode));
5179 }
5180 
5181 void Assembler::vmulps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5182   assert(VM_Version::supports_avx(), "");
5183   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5184   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
5185   emit_int8(0x59);
5186   emit_int8((unsigned char)(0xC0 | encode));
5187 }
5188 
5189 void Assembler::vmulpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5190   assert(VM_Version::supports_avx(), "");
5191   InstructionMark im(this);
5192   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5193   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
5194   attributes.set_rex_vex_w_reverted();
5195   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5196   emit_int8(0x59);
5197   emit_operand(dst, src);
5198 }
5199 
5200 void Assembler::vmulps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5201   assert(VM_Version::supports_avx(), "");
5202   InstructionMark im(this);
5203   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5204   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
5205   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
5206   emit_int8(0x59);
5207   emit_operand(dst, src);
5208 }
5209 
5210 void Assembler::vfmadd231pd(XMMRegister dst, XMMRegister src1, XMMRegister src2, int vector_len) {
5211   assert(VM_Version::supports_fma(), "");
5212   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5213   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5214   emit_int8((unsigned char)0xB8);
5215   emit_int8((unsigned char)(0xC0 | encode));
5216 }
5217 
5218 void Assembler::vfmadd231ps(XMMRegister dst, XMMRegister src1, XMMRegister src2, int vector_len) {
5219   assert(VM_Version::supports_fma(), "");
5220   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5221   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5222   emit_int8((unsigned char)0xB8);
5223   emit_int8((unsigned char)(0xC0 | encode));
5224 }
5225 
5226 void Assembler::vfmadd231pd(XMMRegister dst, XMMRegister src1, Address src2, int vector_len) {
5227   assert(VM_Version::supports_fma(), "");
5228   InstructionMark im(this);
5229   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5230   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
5231   vex_prefix(src2, src1->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5232   emit_int8((unsigned char)0xB8);
5233   emit_operand(dst, src2);
5234 }
5235 
5236 void Assembler::vfmadd231ps(XMMRegister dst, XMMRegister src1, Address src2, int vector_len) {
5237   assert(VM_Version::supports_fma(), "");
5238   InstructionMark im(this);
5239   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5240   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
5241   vex_prefix(src2, src1->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5242   emit_int8((unsigned char)0xB8);
5243   emit_operand(dst, src2);
5244 }
5245 
5246 void Assembler::divpd(XMMRegister dst, XMMRegister src) {
5247   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5248   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5249   attributes.set_rex_vex_w_reverted();
5250   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5251   emit_int8(0x5E);
5252   emit_int8((unsigned char)(0xC0 | encode));
5253 }
5254 
5255 void Assembler::divps(XMMRegister dst, XMMRegister src) {
5256   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5257   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5258   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
5259   emit_int8(0x5E);
5260   emit_int8((unsigned char)(0xC0 | encode));
5261 }
5262 
5263 void Assembler::vdivpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5264   assert(VM_Version::supports_avx(), "");
5265   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5266   attributes.set_rex_vex_w_reverted();
5267   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5268   emit_int8(0x5E);
5269   emit_int8((unsigned char)(0xC0 | encode));
5270 }
5271 
5272 void Assembler::vdivps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5273   assert(VM_Version::supports_avx(), "");
5274   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5275   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
5276   emit_int8(0x5E);
5277   emit_int8((unsigned char)(0xC0 | encode));
5278 }
5279 
5280 void Assembler::vdivpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5281   assert(VM_Version::supports_avx(), "");
5282   InstructionMark im(this);
5283   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5284   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
5285   attributes.set_rex_vex_w_reverted();
5286   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5287   emit_int8(0x5E);
5288   emit_operand(dst, src);
5289 }
5290 
5291 void Assembler::vdivps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5292   assert(VM_Version::supports_avx(), "");
5293   InstructionMark im(this);
5294   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5295   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
5296   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
5297   emit_int8(0x5E);
5298   emit_operand(dst, src);
5299 }
5300 
5301 void Assembler::vsqrtpd(XMMRegister dst, XMMRegister src, int vector_len) {
5302   assert(VM_Version::supports_avx(), "");
5303   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5304   attributes.set_rex_vex_w_reverted();
5305   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5306   emit_int8(0x51);
5307   emit_int8((unsigned char)(0xC0 | encode));
5308 }
5309 
5310 void Assembler::vsqrtpd(XMMRegister dst, Address src, int vector_len) {
5311   assert(VM_Version::supports_avx(), "");
5312   InstructionMark im(this);
5313   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5314   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
5315   attributes.set_rex_vex_w_reverted();
5316   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5317   emit_int8(0x51);
5318   emit_operand(dst, src);
5319 }
5320 
5321 void Assembler::vsqrtps(XMMRegister dst, XMMRegister src, int vector_len) {
5322   assert(VM_Version::supports_avx(), "");
5323   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5324   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
5325   emit_int8(0x51);
5326   emit_int8((unsigned char)(0xC0 | encode));
5327 }
5328 
5329 void Assembler::vsqrtps(XMMRegister dst, Address src, int vector_len) {
5330   assert(VM_Version::supports_avx(), "");
5331   InstructionMark im(this);
5332   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5333   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
5334   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
5335   emit_int8(0x51);
5336   emit_operand(dst, src);
5337 }
5338 
5339 void Assembler::andpd(XMMRegister dst, XMMRegister src) {
5340   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5341   InstructionAttr attributes(AVX_128bit, /* rex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
5342   attributes.set_rex_vex_w_reverted();
5343   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5344   emit_int8(0x54);
5345   emit_int8((unsigned char)(0xC0 | encode));
5346 }
5347 
5348 void Assembler::andps(XMMRegister dst, XMMRegister src) {
5349   NOT_LP64(assert(VM_Version::supports_sse(), ""));
5350   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
5351   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
5352   emit_int8(0x54);
5353   emit_int8((unsigned char)(0xC0 | encode));
5354 }
5355 
5356 void Assembler::andps(XMMRegister dst, Address src) {
5357   NOT_LP64(assert(VM_Version::supports_sse(), ""));
5358   InstructionMark im(this);
5359   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
5360   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
5361   simd_prefix(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
5362   emit_int8(0x54);
5363   emit_operand(dst, src);
5364 }
5365 
5366 void Assembler::andpd(XMMRegister dst, Address src) {
5367   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5368   InstructionMark im(this);
5369   InstructionAttr attributes(AVX_128bit, /* rex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
5370   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
5371   attributes.set_rex_vex_w_reverted();
5372   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5373   emit_int8(0x54);
5374   emit_operand(dst, src);
5375 }
5376 
5377 void Assembler::vandpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5378   assert(VM_Version::supports_avx(), "");
5379   InstructionAttr attributes(vector_len, /* vex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
5380   attributes.set_rex_vex_w_reverted();
5381   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5382   emit_int8(0x54);
5383   emit_int8((unsigned char)(0xC0 | encode));
5384 }
5385 
5386 void Assembler::vandps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5387   assert(VM_Version::supports_avx(), "");
5388   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
5389   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
5390   emit_int8(0x54);
5391   emit_int8((unsigned char)(0xC0 | encode));
5392 }
5393 
5394 void Assembler::vandpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5395   assert(VM_Version::supports_avx(), "");
5396   InstructionMark im(this);
5397   InstructionAttr attributes(vector_len, /* vex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
5398   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
5399   attributes.set_rex_vex_w_reverted();
5400   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5401   emit_int8(0x54);
5402   emit_operand(dst, src);
5403 }
5404 
5405 void Assembler::vandps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5406   assert(VM_Version::supports_avx(), "");
5407   InstructionMark im(this);
5408   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
5409   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
5410   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
5411   emit_int8(0x54);
5412   emit_operand(dst, src);
5413 }
5414 
5415 void Assembler::unpckhpd(XMMRegister dst, XMMRegister src) {
5416   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5417   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5418   attributes.set_rex_vex_w_reverted();
5419   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5420   emit_int8(0x15);
5421   emit_int8((unsigned char)(0xC0 | encode));
5422 }
5423 
5424 void Assembler::unpcklpd(XMMRegister dst, XMMRegister src) {
5425   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5426   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5427   attributes.set_rex_vex_w_reverted();
5428   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5429   emit_int8(0x14);
5430   emit_int8((unsigned char)(0xC0 | encode));
5431 }
5432 
5433 void Assembler::xorpd(XMMRegister dst, XMMRegister src) {
5434   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5435   InstructionAttr attributes(AVX_128bit, /* rex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
5436   attributes.set_rex_vex_w_reverted();
5437   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5438   emit_int8(0x57);
5439   emit_int8((unsigned char)(0xC0 | encode));
5440 }
5441 
5442 void Assembler::xorps(XMMRegister dst, XMMRegister src) {
5443   NOT_LP64(assert(VM_Version::supports_sse(), ""));
5444   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
5445   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
5446   emit_int8(0x57);
5447   emit_int8((unsigned char)(0xC0 | encode));
5448 }
5449 
5450 void Assembler::xorpd(XMMRegister dst, Address src) {
5451   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5452   InstructionMark im(this);
5453   InstructionAttr attributes(AVX_128bit, /* rex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
5454   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
5455   attributes.set_rex_vex_w_reverted();
5456   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5457   emit_int8(0x57);
5458   emit_operand(dst, src);
5459 }
5460 
5461 void Assembler::xorps(XMMRegister dst, Address src) {
5462   NOT_LP64(assert(VM_Version::supports_sse(), ""));
5463   InstructionMark im(this);
5464   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
5465   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
5466   simd_prefix(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
5467   emit_int8(0x57);
5468   emit_operand(dst, src);
5469 }
5470 
5471 void Assembler::vxorpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5472   assert(VM_Version::supports_avx(), "");
5473   InstructionAttr attributes(vector_len, /* vex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
5474   attributes.set_rex_vex_w_reverted();
5475   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5476   emit_int8(0x57);
5477   emit_int8((unsigned char)(0xC0 | encode));
5478 }
5479 
5480 void Assembler::vxorps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5481   assert(VM_Version::supports_avx(), "");
5482   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
5483   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
5484   emit_int8(0x57);
5485   emit_int8((unsigned char)(0xC0 | encode));
5486 }
5487 
5488 void Assembler::vxorpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5489   assert(VM_Version::supports_avx(), "");
5490   InstructionMark im(this);
5491   InstructionAttr attributes(vector_len, /* vex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
5492   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
5493   attributes.set_rex_vex_w_reverted();
5494   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5495   emit_int8(0x57);
5496   emit_operand(dst, src);
5497 }
5498 
5499 void Assembler::vxorps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5500   assert(VM_Version::supports_avx(), "");
5501   InstructionMark im(this);
5502   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
5503   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
5504   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
5505   emit_int8(0x57);
5506   emit_operand(dst, src);
5507 }
5508 
5509 // Integer vector arithmetic
5510 void Assembler::vphaddw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5511   assert(VM_Version::supports_avx() && (vector_len == 0) ||
5512          VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
5513   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
5514   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5515   emit_int8(0x01);
5516   emit_int8((unsigned char)(0xC0 | encode));
5517 }
5518 
5519 void Assembler::vphaddd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5520   assert(VM_Version::supports_avx() && (vector_len == 0) ||
5521          VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
5522   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
5523   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5524   emit_int8(0x02);
5525   emit_int8((unsigned char)(0xC0 | encode));
5526 }
5527 
5528 void Assembler::paddb(XMMRegister dst, XMMRegister src) {
5529   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5530   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5531   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5532   emit_int8((unsigned char)0xFC);
5533   emit_int8((unsigned char)(0xC0 | encode));
5534 }
5535 
5536 void Assembler::paddw(XMMRegister dst, XMMRegister src) {
5537   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5538   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5539   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5540   emit_int8((unsigned char)0xFD);
5541   emit_int8((unsigned char)(0xC0 | encode));
5542 }
5543 
5544 void Assembler::paddd(XMMRegister dst, XMMRegister src) {
5545   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5546   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5547   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5548   emit_int8((unsigned char)0xFE);
5549   emit_int8((unsigned char)(0xC0 | encode));
5550 }
5551 
5552 void Assembler::paddd(XMMRegister dst, Address src) {
5553   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5554   InstructionMark im(this);
5555   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5556   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5557   emit_int8((unsigned char)0xFE);
5558   emit_operand(dst, src);
5559 }
5560 
5561 void Assembler::paddq(XMMRegister dst, XMMRegister src) {
5562   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5563   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5564   attributes.set_rex_vex_w_reverted();
5565   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5566   emit_int8((unsigned char)0xD4);
5567   emit_int8((unsigned char)(0xC0 | encode));
5568 }
5569 
5570 void Assembler::phaddw(XMMRegister dst, XMMRegister src) {
5571   assert(VM_Version::supports_sse3(), "");
5572   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
5573   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5574   emit_int8(0x01);
5575   emit_int8((unsigned char)(0xC0 | encode));
5576 }
5577 
5578 void Assembler::phaddd(XMMRegister dst, XMMRegister src) {
5579   assert(VM_Version::supports_sse3(), "");
5580   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
5581   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5582   emit_int8(0x02);
5583   emit_int8((unsigned char)(0xC0 | encode));
5584 }
5585 
5586 void Assembler::vpaddb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5587   assert(UseAVX > 0, "requires some form of AVX");
5588   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5589   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5590   emit_int8((unsigned char)0xFC);
5591   emit_int8((unsigned char)(0xC0 | encode));
5592 }
5593 
5594 void Assembler::vpaddw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5595   assert(UseAVX > 0, "requires some form of AVX");
5596   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5597   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5598   emit_int8((unsigned char)0xFD);
5599   emit_int8((unsigned char)(0xC0 | encode));
5600 }
5601 
5602 void Assembler::vpaddd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5603   assert(UseAVX > 0, "requires some form of AVX");
5604   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5605   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5606   emit_int8((unsigned char)0xFE);
5607   emit_int8((unsigned char)(0xC0 | encode));
5608 }
5609 
5610 void Assembler::vpaddq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5611   assert(UseAVX > 0, "requires some form of AVX");
5612   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5613   attributes.set_rex_vex_w_reverted();
5614   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5615   emit_int8((unsigned char)0xD4);
5616   emit_int8((unsigned char)(0xC0 | encode));
5617 }
5618 
5619 void Assembler::vpaddb(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5620   assert(UseAVX > 0, "requires some form of AVX");
5621   InstructionMark im(this);
5622   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5623   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
5624   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5625   emit_int8((unsigned char)0xFC);
5626   emit_operand(dst, src);
5627 }
5628 
5629 void Assembler::vpaddw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5630   assert(UseAVX > 0, "requires some form of AVX");
5631   InstructionMark im(this);
5632   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5633   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
5634   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5635   emit_int8((unsigned char)0xFD);
5636   emit_operand(dst, src);
5637 }
5638 
5639 void Assembler::vpaddd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5640   assert(UseAVX > 0, "requires some form of AVX");
5641   InstructionMark im(this);
5642   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5643   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
5644   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5645   emit_int8((unsigned char)0xFE);
5646   emit_operand(dst, src);
5647 }
5648 
5649 void Assembler::vpaddq(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5650   assert(UseAVX > 0, "requires some form of AVX");
5651   InstructionMark im(this);
5652   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5653   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
5654   attributes.set_rex_vex_w_reverted();
5655   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5656   emit_int8((unsigned char)0xD4);
5657   emit_operand(dst, src);
5658 }
5659 
5660 void Assembler::psubb(XMMRegister dst, XMMRegister src) {
5661   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5662   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5663   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5664   emit_int8((unsigned char)0xF8);
5665   emit_int8((unsigned char)(0xC0 | encode));
5666 }
5667 
5668 void Assembler::psubw(XMMRegister dst, XMMRegister src) {
5669   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5670   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5671   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5672   emit_int8((unsigned char)0xF9);
5673   emit_int8((unsigned char)(0xC0 | encode));
5674 }
5675 
5676 void Assembler::psubd(XMMRegister dst, XMMRegister src) {
5677   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5678   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5679   emit_int8((unsigned char)0xFA);
5680   emit_int8((unsigned char)(0xC0 | encode));
5681 }
5682 
5683 void Assembler::psubq(XMMRegister dst, XMMRegister src) {
5684   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5685   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5686   attributes.set_rex_vex_w_reverted();
5687   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5688   emit_int8((unsigned char)0xFB);
5689   emit_int8((unsigned char)(0xC0 | encode));
5690 }
5691 
5692 void Assembler::vpsubb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5693   assert(UseAVX > 0, "requires some form of AVX");
5694   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5695   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5696   emit_int8((unsigned char)0xF8);
5697   emit_int8((unsigned char)(0xC0 | encode));
5698 }
5699 
5700 void Assembler::vpsubw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5701   assert(UseAVX > 0, "requires some form of AVX");
5702   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5703   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5704   emit_int8((unsigned char)0xF9);
5705   emit_int8((unsigned char)(0xC0 | encode));
5706 }
5707 
5708 void Assembler::vpsubd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5709   assert(UseAVX > 0, "requires some form of AVX");
5710   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5711   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5712   emit_int8((unsigned char)0xFA);
5713   emit_int8((unsigned char)(0xC0 | encode));
5714 }
5715 
5716 void Assembler::vpsubq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5717   assert(UseAVX > 0, "requires some form of AVX");
5718   InstructionAttr attributes(vector_len, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5719   attributes.set_rex_vex_w_reverted();
5720   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5721   emit_int8((unsigned char)0xFB);
5722   emit_int8((unsigned char)(0xC0 | encode));
5723 }
5724 
5725 void Assembler::vpsubb(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5726   assert(UseAVX > 0, "requires some form of AVX");
5727   InstructionMark im(this);
5728   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5729   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
5730   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5731   emit_int8((unsigned char)0xF8);
5732   emit_operand(dst, src);
5733 }
5734 
5735 void Assembler::vpsubw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5736   assert(UseAVX > 0, "requires some form of AVX");
5737   InstructionMark im(this);
5738   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5739   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
5740   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5741   emit_int8((unsigned char)0xF9);
5742   emit_operand(dst, src);
5743 }
5744 
5745 void Assembler::vpsubd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5746   assert(UseAVX > 0, "requires some form of AVX");
5747   InstructionMark im(this);
5748   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5749   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
5750   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5751   emit_int8((unsigned char)0xFA);
5752   emit_operand(dst, src);
5753 }
5754 
5755 void Assembler::vpsubq(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5756   assert(UseAVX > 0, "requires some form of AVX");
5757   InstructionMark im(this);
5758   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5759   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
5760   attributes.set_rex_vex_w_reverted();
5761   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5762   emit_int8((unsigned char)0xFB);
5763   emit_operand(dst, src);
5764 }
5765 
5766 void Assembler::pmullw(XMMRegister dst, XMMRegister src) {
5767   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5768   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5769   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5770   emit_int8((unsigned char)0xD5);
5771   emit_int8((unsigned char)(0xC0 | encode));
5772 }
5773 
5774 void Assembler::pmulld(XMMRegister dst, XMMRegister src) {
5775   assert(VM_Version::supports_sse4_1(), "");
5776   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5777   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5778   emit_int8(0x40);
5779   emit_int8((unsigned char)(0xC0 | encode));
5780 }
5781 
5782 void Assembler::vpmullw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5783   assert(UseAVX > 0, "requires some form of AVX");
5784   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5785   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5786   emit_int8((unsigned char)0xD5);
5787   emit_int8((unsigned char)(0xC0 | encode));
5788 }
5789 
5790 void Assembler::vpmulld(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5791   assert(UseAVX > 0, "requires some form of AVX");
5792   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5793   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5794   emit_int8(0x40);
5795   emit_int8((unsigned char)(0xC0 | encode));
5796 }
5797 
5798 void Assembler::vpmullq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5799   assert(UseAVX > 2, "requires some form of EVEX");
5800   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
5801   attributes.set_is_evex_instruction();
5802   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5803   emit_int8(0x40);
5804   emit_int8((unsigned char)(0xC0 | encode));
5805 }
5806 
5807 void Assembler::vpmullw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5808   assert(UseAVX > 0, "requires some form of AVX");
5809   InstructionMark im(this);
5810   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5811   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
5812   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5813   emit_int8((unsigned char)0xD5);
5814   emit_operand(dst, src);
5815 }
5816 
5817 void Assembler::vpmulld(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5818   assert(UseAVX > 0, "requires some form of AVX");
5819   InstructionMark im(this);
5820   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5821   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
5822   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5823   emit_int8(0x40);
5824   emit_operand(dst, src);
5825 }
5826 
5827 void Assembler::vpmullq(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5828   assert(UseAVX > 2, "requires some form of EVEX");
5829   InstructionMark im(this);
5830   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
5831   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
5832   attributes.set_is_evex_instruction();
5833   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5834   emit_int8(0x40);
5835   emit_operand(dst, src);
5836 }
5837 
5838 // Shift packed integers left by specified number of bits.
5839 void Assembler::psllw(XMMRegister dst, int shift) {
5840   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5841   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5842   // XMM6 is for /6 encoding: 66 0F 71 /6 ib
5843   int encode = simd_prefix_and_encode(xmm6, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5844   emit_int8(0x71);
5845   emit_int8((unsigned char)(0xC0 | encode));
5846   emit_int8(shift & 0xFF);
5847 }
5848 
5849 void Assembler::pslld(XMMRegister dst, int shift) {
5850   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5851   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5852   // XMM6 is for /6 encoding: 66 0F 72 /6 ib
5853   int encode = simd_prefix_and_encode(xmm6, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5854   emit_int8(0x72);
5855   emit_int8((unsigned char)(0xC0 | encode));
5856   emit_int8(shift & 0xFF);
5857 }
5858 
5859 void Assembler::psllq(XMMRegister dst, int shift) {
5860   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5861   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5862   // XMM6 is for /6 encoding: 66 0F 73 /6 ib
5863   int encode = simd_prefix_and_encode(xmm6, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5864   emit_int8(0x73);
5865   emit_int8((unsigned char)(0xC0 | encode));
5866   emit_int8(shift & 0xFF);
5867 }
5868 
5869 void Assembler::psllw(XMMRegister dst, XMMRegister shift) {
5870   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5871   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5872   int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5873   emit_int8((unsigned char)0xF1);
5874   emit_int8((unsigned char)(0xC0 | encode));
5875 }
5876 
5877 void Assembler::pslld(XMMRegister dst, XMMRegister shift) {
5878   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5879   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5880   int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5881   emit_int8((unsigned char)0xF2);
5882   emit_int8((unsigned char)(0xC0 | encode));
5883 }
5884 
5885 void Assembler::psllq(XMMRegister dst, XMMRegister shift) {
5886   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5887   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5888   attributes.set_rex_vex_w_reverted();
5889   int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5890   emit_int8((unsigned char)0xF3);
5891   emit_int8((unsigned char)(0xC0 | encode));
5892 }
5893 
5894 void Assembler::vpsllw(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
5895   assert(UseAVX > 0, "requires some form of AVX");
5896   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5897   // XMM6 is for /6 encoding: 66 0F 71 /6 ib
5898   int encode = vex_prefix_and_encode(xmm6->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5899   emit_int8(0x71);
5900   emit_int8((unsigned char)(0xC0 | encode));
5901   emit_int8(shift & 0xFF);
5902 }
5903 
5904 void Assembler::vpslld(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
5905   assert(UseAVX > 0, "requires some form of AVX");
5906   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5907   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5908   // XMM6 is for /6 encoding: 66 0F 72 /6 ib
5909   int encode = vex_prefix_and_encode(xmm6->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5910   emit_int8(0x72);
5911   emit_int8((unsigned char)(0xC0 | encode));
5912   emit_int8(shift & 0xFF);
5913 }
5914 
5915 void Assembler::vpsllq(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
5916   assert(UseAVX > 0, "requires some form of AVX");
5917   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5918   attributes.set_rex_vex_w_reverted();
5919   // XMM6 is for /6 encoding: 66 0F 73 /6 ib
5920   int encode = vex_prefix_and_encode(xmm6->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5921   emit_int8(0x73);
5922   emit_int8((unsigned char)(0xC0 | encode));
5923   emit_int8(shift & 0xFF);
5924 }
5925 
5926 void Assembler::vpsllw(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
5927   assert(UseAVX > 0, "requires some form of AVX");
5928   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5929   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5930   emit_int8((unsigned char)0xF1);
5931   emit_int8((unsigned char)(0xC0 | encode));
5932 }
5933 
5934 void Assembler::vpslld(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
5935   assert(UseAVX > 0, "requires some form of AVX");
5936   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5937   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5938   emit_int8((unsigned char)0xF2);
5939   emit_int8((unsigned char)(0xC0 | encode));
5940 }
5941 
5942 void Assembler::vpsllq(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
5943   assert(UseAVX > 0, "requires some form of AVX");
5944   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5945   attributes.set_rex_vex_w_reverted();
5946   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5947   emit_int8((unsigned char)0xF3);
5948   emit_int8((unsigned char)(0xC0 | encode));
5949 }
5950 
5951 // Shift packed integers logically right by specified number of bits.
5952 void Assembler::psrlw(XMMRegister dst, int shift) {
5953   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5954   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5955   // XMM2 is for /2 encoding: 66 0F 71 /2 ib
5956   int encode = simd_prefix_and_encode(xmm2, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5957   emit_int8(0x71);
5958   emit_int8((unsigned char)(0xC0 | encode));
5959   emit_int8(shift & 0xFF);
5960 }
5961 
5962 void Assembler::psrld(XMMRegister dst, int shift) {
5963   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5964   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5965   // XMM2 is for /2 encoding: 66 0F 72 /2 ib
5966   int encode = simd_prefix_and_encode(xmm2, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5967   emit_int8(0x72);
5968   emit_int8((unsigned char)(0xC0 | encode));
5969   emit_int8(shift & 0xFF);
5970 }
5971 
5972 void Assembler::psrlq(XMMRegister dst, int shift) {
5973   // Do not confuse it with psrldq SSE2 instruction which
5974   // shifts 128 bit value in xmm register by number of bytes.
5975   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5976   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5977   attributes.set_rex_vex_w_reverted();
5978   // XMM2 is for /2 encoding: 66 0F 73 /2 ib
5979   int encode = simd_prefix_and_encode(xmm2, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5980   emit_int8(0x73);
5981   emit_int8((unsigned char)(0xC0 | encode));
5982   emit_int8(shift & 0xFF);
5983 }
5984 
5985 void Assembler::psrlw(XMMRegister dst, XMMRegister shift) {
5986   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5987   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5988   int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5989   emit_int8((unsigned char)0xD1);
5990   emit_int8((unsigned char)(0xC0 | encode));
5991 }
5992 
5993 void Assembler::psrld(XMMRegister dst, XMMRegister shift) {
5994   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5995   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5996   int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5997   emit_int8((unsigned char)0xD2);
5998   emit_int8((unsigned char)(0xC0 | encode));
5999 }
6000 
6001 void Assembler::psrlq(XMMRegister dst, XMMRegister shift) {
6002   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
6003   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6004   attributes.set_rex_vex_w_reverted();
6005   int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6006   emit_int8((unsigned char)0xD3);
6007   emit_int8((unsigned char)(0xC0 | encode));
6008 }
6009 
6010 void Assembler::vpsrlw(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
6011   assert(UseAVX > 0, "requires some form of AVX");
6012   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
6013   // XMM2 is for /2 encoding: 66 0F 71 /2 ib
6014   int encode = vex_prefix_and_encode(xmm2->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6015   emit_int8(0x71);
6016   emit_int8((unsigned char)(0xC0 | encode));
6017   emit_int8(shift & 0xFF);
6018 }
6019 
6020 void Assembler::vpsrld(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
6021   assert(UseAVX > 0, "requires some form of AVX");
6022   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6023   // XMM2 is for /2 encoding: 66 0F 72 /2 ib
6024   int encode = vex_prefix_and_encode(xmm2->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6025   emit_int8(0x72);
6026   emit_int8((unsigned char)(0xC0 | encode));
6027   emit_int8(shift & 0xFF);
6028 }
6029 
6030 void Assembler::vpsrlq(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
6031   assert(UseAVX > 0, "requires some form of AVX");
6032   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6033   attributes.set_rex_vex_w_reverted();
6034   // XMM2 is for /2 encoding: 66 0F 73 /2 ib
6035   int encode = vex_prefix_and_encode(xmm2->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6036   emit_int8(0x73);
6037   emit_int8((unsigned char)(0xC0 | encode));
6038   emit_int8(shift & 0xFF);
6039 }
6040 
6041 void Assembler::vpsrlw(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
6042   assert(UseAVX > 0, "requires some form of AVX");
6043   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
6044   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6045   emit_int8((unsigned char)0xD1);
6046   emit_int8((unsigned char)(0xC0 | encode));
6047 }
6048 
6049 void Assembler::vpsrld(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
6050   assert(UseAVX > 0, "requires some form of AVX");
6051   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6052   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6053   emit_int8((unsigned char)0xD2);
6054   emit_int8((unsigned char)(0xC0 | encode));
6055 }
6056 
6057 void Assembler::vpsrlq(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
6058   assert(UseAVX > 0, "requires some form of AVX");
6059   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6060   attributes.set_rex_vex_w_reverted();
6061   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6062   emit_int8((unsigned char)0xD3);
6063   emit_int8((unsigned char)(0xC0 | encode));
6064 }
6065 
6066 // Shift packed integers arithmetically right by specified number of bits.
6067 void Assembler::psraw(XMMRegister dst, int shift) {
6068   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
6069   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
6070   // XMM4 is for /4 encoding: 66 0F 71 /4 ib
6071   int encode = simd_prefix_and_encode(xmm4, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6072   emit_int8(0x71);
6073   emit_int8((unsigned char)(0xC0 | encode));
6074   emit_int8(shift & 0xFF);
6075 }
6076 
6077 void Assembler::psrad(XMMRegister dst, int shift) {
6078   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
6079   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6080   // XMM4 is for /4 encoding: 66 0F 72 /4 ib
6081   int encode = simd_prefix_and_encode(xmm4, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6082   emit_int8(0x72);
6083   emit_int8((unsigned char)(0xC0 | encode));
6084   emit_int8(shift & 0xFF);
6085 }
6086 
6087 void Assembler::psraw(XMMRegister dst, XMMRegister shift) {
6088   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
6089   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
6090   int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6091   emit_int8((unsigned char)0xE1);
6092   emit_int8((unsigned char)(0xC0 | encode));
6093 }
6094 
6095 void Assembler::psrad(XMMRegister dst, XMMRegister shift) {
6096   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
6097   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6098   int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6099   emit_int8((unsigned char)0xE2);
6100   emit_int8((unsigned char)(0xC0 | encode));
6101 }
6102 
6103 void Assembler::vpsraw(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
6104   assert(UseAVX > 0, "requires some form of AVX");
6105   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
6106   // XMM4 is for /4 encoding: 66 0F 71 /4 ib
6107   int encode = vex_prefix_and_encode(xmm4->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6108   emit_int8(0x71);
6109   emit_int8((unsigned char)(0xC0 | encode));
6110   emit_int8(shift & 0xFF);
6111 }
6112 
6113 void Assembler::vpsrad(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
6114   assert(UseAVX > 0, "requires some form of AVX");
6115   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6116   // XMM4 is for /4 encoding: 66 0F 71 /4 ib
6117   int encode = vex_prefix_and_encode(xmm4->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6118   emit_int8(0x72);
6119   emit_int8((unsigned char)(0xC0 | encode));
6120   emit_int8(shift & 0xFF);
6121 }
6122 
6123 void Assembler::vpsraw(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
6124   assert(UseAVX > 0, "requires some form of AVX");
6125   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
6126   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6127   emit_int8((unsigned char)0xE1);
6128   emit_int8((unsigned char)(0xC0 | encode));
6129 }
6130 
6131 void Assembler::vpsrad(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
6132   assert(UseAVX > 0, "requires some form of AVX");
6133   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6134   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6135   emit_int8((unsigned char)0xE2);
6136   emit_int8((unsigned char)(0xC0 | encode));
6137 }
6138 
6139 
6140 // logical operations packed integers
6141 void Assembler::pand(XMMRegister dst, XMMRegister src) {
6142   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
6143   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6144   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6145   emit_int8((unsigned char)0xDB);
6146   emit_int8((unsigned char)(0xC0 | encode));
6147 }
6148 
6149 void Assembler::vpand(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
6150   assert(UseAVX > 0, "requires some form of AVX");
6151   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6152   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6153   emit_int8((unsigned char)0xDB);
6154   emit_int8((unsigned char)(0xC0 | encode));
6155 }
6156 
6157 void Assembler::vpand(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
6158   assert(UseAVX > 0, "requires some form of AVX");
6159   InstructionMark im(this);
6160   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6161   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
6162   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6163   emit_int8((unsigned char)0xDB);
6164   emit_operand(dst, src);
6165 }
6166 
6167 void Assembler::pandn(XMMRegister dst, XMMRegister src) {
6168   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
6169   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6170   attributes.set_rex_vex_w_reverted();
6171   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6172   emit_int8((unsigned char)0xDF);
6173   emit_int8((unsigned char)(0xC0 | encode));
6174 }
6175 
6176 void Assembler::por(XMMRegister dst, XMMRegister src) {
6177   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
6178   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6179   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6180   emit_int8((unsigned char)0xEB);
6181   emit_int8((unsigned char)(0xC0 | encode));
6182 }
6183 
6184 void Assembler::vpor(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
6185   assert(UseAVX > 0, "requires some form of AVX");
6186   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6187   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6188   emit_int8((unsigned char)0xEB);
6189   emit_int8((unsigned char)(0xC0 | encode));
6190 }
6191 
6192 void Assembler::vpor(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
6193   assert(UseAVX > 0, "requires some form of AVX");
6194   InstructionMark im(this);
6195   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6196   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
6197   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6198   emit_int8((unsigned char)0xEB);
6199   emit_operand(dst, src);
6200 }
6201 
6202 void Assembler::pxor(XMMRegister dst, XMMRegister src) {
6203   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
6204   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6205   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6206   emit_int8((unsigned char)0xEF);
6207   emit_int8((unsigned char)(0xC0 | encode));
6208 }
6209 
6210 void Assembler::vpxor(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
6211   assert(UseAVX > 0, "requires some form of AVX");
6212   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6213   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6214   emit_int8((unsigned char)0xEF);
6215   emit_int8((unsigned char)(0xC0 | encode));
6216 }
6217 
6218 void Assembler::vpxor(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
6219   assert(UseAVX > 0, "requires some form of AVX");
6220   InstructionMark im(this);
6221   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6222   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
6223   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6224   emit_int8((unsigned char)0xEF);
6225   emit_operand(dst, src);
6226 }
6227 
6228 
6229 // vinserti forms
6230 
6231 void Assembler::vinserti128(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) {
6232   assert(VM_Version::supports_avx2(), "");
6233   assert(imm8 <= 0x01, "imm8: %u", imm8);
6234   int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_256bit;
6235   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6236   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6237   emit_int8(0x38);
6238   emit_int8((unsigned char)(0xC0 | encode));
6239   // 0x00 - insert into lower 128 bits
6240   // 0x01 - insert into upper 128 bits
6241   emit_int8(imm8 & 0x01);
6242 }
6243 
6244 void Assembler::vinserti128(XMMRegister dst, XMMRegister nds, Address src, uint8_t imm8) {
6245   assert(VM_Version::supports_avx2(), "");
6246   assert(dst != xnoreg, "sanity");
6247   assert(imm8 <= 0x01, "imm8: %u", imm8);
6248   int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_256bit;
6249   InstructionMark im(this);
6250   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6251   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
6252   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6253   emit_int8(0x38);
6254   emit_operand(dst, src);
6255   // 0x00 - insert into lower 128 bits
6256   // 0x01 - insert into upper 128 bits
6257   emit_int8(imm8 & 0x01);
6258 }
6259 
6260 void Assembler::vinserti32x4(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) {
6261   assert(VM_Version::supports_evex(), "");
6262   assert(imm8 <= 0x03, "imm8: %u", imm8);
6263   InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6264   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6265   emit_int8(0x38);
6266   emit_int8((unsigned char)(0xC0 | encode));
6267   // 0x00 - insert into q0 128 bits (0..127)
6268   // 0x01 - insert into q1 128 bits (128..255)
6269   // 0x02 - insert into q2 128 bits (256..383)
6270   // 0x03 - insert into q3 128 bits (384..511)
6271   emit_int8(imm8 & 0x03);
6272 }
6273 
6274 void Assembler::vinserti32x4(XMMRegister dst, XMMRegister nds, Address src, uint8_t imm8) {
6275   assert(VM_Version::supports_avx(), "");
6276   assert(dst != xnoreg, "sanity");
6277   assert(imm8 <= 0x03, "imm8: %u", imm8);
6278   int vector_len = VM_Version::supports_evex() ? AVX_512bit : AVX_256bit;
6279   InstructionMark im(this);
6280   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6281   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
6282   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6283   emit_int8(0x18);
6284   emit_operand(dst, src);
6285   // 0x00 - insert into q0 128 bits (0..127)
6286   // 0x01 - insert into q1 128 bits (128..255)
6287   // 0x02 - insert into q2 128 bits (256..383)
6288   // 0x03 - insert into q3 128 bits (384..511)
6289   emit_int8(imm8 & 0x03);
6290 }
6291 
6292 void Assembler::vinserti64x4(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) {
6293   assert(VM_Version::supports_evex(), "");
6294   assert(imm8 <= 0x01, "imm8: %u", imm8);
6295   InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6296   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6297   emit_int8(0x38);
6298   emit_int8((unsigned char)(0xC0 | encode));
6299   // 0x00 - insert into lower 256 bits
6300   // 0x01 - insert into upper 256 bits
6301   emit_int8(imm8 & 0x01);
6302 }
6303 
6304 
6305 // vinsertf forms
6306 
6307 void Assembler::vinsertf128(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) {
6308   assert(VM_Version::supports_avx(), "");
6309   assert(imm8 <= 0x01, "imm8: %u", imm8);
6310   int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_256bit;
6311   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6312   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6313   emit_int8(0x18);
6314   emit_int8((unsigned char)(0xC0 | encode));
6315   // 0x00 - insert into lower 128 bits
6316   // 0x01 - insert into upper 128 bits
6317   emit_int8(imm8 & 0x01);
6318 }
6319 
6320 void Assembler::vinsertf128(XMMRegister dst, XMMRegister nds, Address src, uint8_t imm8) {
6321   assert(VM_Version::supports_avx(), "");
6322   assert(dst != xnoreg, "sanity");
6323   assert(imm8 <= 0x01, "imm8: %u", imm8);
6324   int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_256bit;
6325   InstructionMark im(this);
6326   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6327   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
6328   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6329   emit_int8(0x18);
6330   emit_operand(dst, src);
6331   // 0x00 - insert into lower 128 bits
6332   // 0x01 - insert into upper 128 bits
6333   emit_int8(imm8 & 0x01);
6334 }
6335 
6336 void Assembler::vinsertf32x4(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) {
6337   assert(VM_Version::supports_evex(), "");
6338   assert(imm8 <= 0x03, "imm8: %u", imm8);
6339   InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6340   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6341   emit_int8(0x18);
6342   emit_int8((unsigned char)(0xC0 | encode));
6343   // 0x00 - insert into q0 128 bits (0..127)
6344   // 0x01 - insert into q1 128 bits (128..255)
6345   // 0x02 - insert into q2 128 bits (256..383)
6346   // 0x03 - insert into q3 128 bits (384..511)
6347   emit_int8(imm8 & 0x03);
6348 }
6349 
6350 void Assembler::vinsertf32x4(XMMRegister dst, XMMRegister nds, Address src, uint8_t imm8) {
6351   assert(VM_Version::supports_avx(), "");
6352   assert(dst != xnoreg, "sanity");
6353   assert(imm8 <= 0x03, "imm8: %u", imm8);
6354   int vector_len = VM_Version::supports_evex() ? AVX_512bit : AVX_256bit;
6355   InstructionMark im(this);
6356   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6357   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
6358   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6359   emit_int8(0x18);
6360   emit_operand(dst, src);
6361   // 0x00 - insert into q0 128 bits (0..127)
6362   // 0x01 - insert into q1 128 bits (128..255)
6363   // 0x02 - insert into q2 128 bits (256..383)
6364   // 0x03 - insert into q3 128 bits (384..511)
6365   emit_int8(imm8 & 0x03);
6366 }
6367 
6368 void Assembler::vinsertf64x4(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) {
6369   assert(VM_Version::supports_evex(), "");
6370   assert(imm8 <= 0x01, "imm8: %u", imm8);
6371   InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6372   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6373   emit_int8(0x1A);
6374   emit_int8((unsigned char)(0xC0 | encode));
6375   // 0x00 - insert into lower 256 bits
6376   // 0x01 - insert into upper 256 bits
6377   emit_int8(imm8 & 0x01);
6378 }
6379 
6380 void Assembler::vinsertf64x4(XMMRegister dst, XMMRegister nds, Address src, uint8_t imm8) {
6381   assert(VM_Version::supports_evex(), "");
6382   assert(dst != xnoreg, "sanity");
6383   assert(imm8 <= 0x01, "imm8: %u", imm8);
6384   InstructionMark im(this);
6385   InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6386   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_64bit);
6387   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6388   emit_int8(0x1A);
6389   emit_operand(dst, src);
6390   // 0x00 - insert into lower 256 bits
6391   // 0x01 - insert into upper 256 bits
6392   emit_int8(imm8 & 0x01);
6393 }
6394 
6395 
6396 // vextracti forms
6397 
6398 void Assembler::vextracti128(XMMRegister dst, XMMRegister src, uint8_t imm8) {
6399   assert(VM_Version::supports_avx(), "");
6400   assert(imm8 <= 0x01, "imm8: %u", imm8);
6401   int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_256bit;
6402   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6403   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6404   emit_int8(0x39);
6405   emit_int8((unsigned char)(0xC0 | encode));
6406   // 0x00 - extract from lower 128 bits
6407   // 0x01 - extract from upper 128 bits
6408   emit_int8(imm8 & 0x01);
6409 }
6410 
6411 void Assembler::vextracti128(Address dst, XMMRegister src, uint8_t imm8) {
6412   assert(VM_Version::supports_avx2(), "");
6413   assert(src != xnoreg, "sanity");
6414   assert(imm8 <= 0x01, "imm8: %u", imm8);
6415   int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_256bit;
6416   InstructionMark im(this);
6417   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6418   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
6419   attributes.reset_is_clear_context();
6420   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6421   emit_int8(0x39);
6422   emit_operand(src, dst);
6423   // 0x00 - extract from lower 128 bits
6424   // 0x01 - extract from upper 128 bits
6425   emit_int8(imm8 & 0x01);
6426 }
6427 
6428 void Assembler::vextracti32x4(XMMRegister dst, XMMRegister src, uint8_t imm8) {
6429   assert(VM_Version::supports_avx(), "");
6430   assert(imm8 <= 0x03, "imm8: %u", imm8);
6431   int vector_len = VM_Version::supports_evex() ? AVX_512bit : AVX_256bit;
6432   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6433   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6434   emit_int8(0x39);
6435   emit_int8((unsigned char)(0xC0 | encode));
6436   // 0x00 - extract from bits 127:0
6437   // 0x01 - extract from bits 255:128
6438   // 0x02 - extract from bits 383:256
6439   // 0x03 - extract from bits 511:384
6440   emit_int8(imm8 & 0x03);
6441 }
6442 
6443 void Assembler::vextracti32x4(Address dst, XMMRegister src, uint8_t imm8) {
6444   assert(VM_Version::supports_evex(), "");
6445   assert(src != xnoreg, "sanity");
6446   assert(imm8 <= 0x03, "imm8: %u", imm8);
6447   InstructionMark im(this);
6448   InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6449   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
6450   attributes.reset_is_clear_context();
6451   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6452   emit_int8(0x39);
6453   emit_operand(src, dst);
6454   // 0x00 - extract from bits 127:0
6455   // 0x01 - extract from bits 255:128
6456   // 0x02 - extract from bits 383:256
6457   // 0x03 - extract from bits 511:384
6458   emit_int8(imm8 & 0x03);
6459 }
6460 
6461 void Assembler::vextracti64x2(XMMRegister dst, XMMRegister src, uint8_t imm8) {
6462   assert(VM_Version::supports_avx512dq(), "");
6463   assert(imm8 <= 0x03, "imm8: %u", imm8);
6464   InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6465   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6466   emit_int8(0x39);
6467   emit_int8((unsigned char)(0xC0 | encode));
6468   // 0x00 - extract from bits 127:0
6469   // 0x01 - extract from bits 255:128
6470   // 0x02 - extract from bits 383:256
6471   // 0x03 - extract from bits 511:384
6472   emit_int8(imm8 & 0x03);
6473 }
6474 
6475 void Assembler::vextracti64x4(XMMRegister dst, XMMRegister src, uint8_t imm8) {
6476   assert(VM_Version::supports_evex(), "");
6477   assert(imm8 <= 0x01, "imm8: %u", imm8);
6478   InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6479   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6480   emit_int8(0x3B);
6481   emit_int8((unsigned char)(0xC0 | encode));
6482   // 0x00 - extract from lower 256 bits
6483   // 0x01 - extract from upper 256 bits
6484   emit_int8(imm8 & 0x01);
6485 }
6486 
6487 
6488 // vextractf forms
6489 
6490 void Assembler::vextractf128(XMMRegister dst, XMMRegister src, uint8_t imm8) {
6491   assert(VM_Version::supports_avx(), "");
6492   assert(imm8 <= 0x01, "imm8: %u", imm8);
6493   int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_256bit;
6494   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6495   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6496   emit_int8(0x19);
6497   emit_int8((unsigned char)(0xC0 | encode));
6498   // 0x00 - extract from lower 128 bits
6499   // 0x01 - extract from upper 128 bits
6500   emit_int8(imm8 & 0x01);
6501 }
6502 
6503 void Assembler::vextractf128(Address dst, XMMRegister src, uint8_t imm8) {
6504   assert(VM_Version::supports_avx(), "");
6505   assert(src != xnoreg, "sanity");
6506   assert(imm8 <= 0x01, "imm8: %u", imm8);
6507   int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_256bit;
6508   InstructionMark im(this);
6509   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6510   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
6511   attributes.reset_is_clear_context();
6512   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6513   emit_int8(0x19);
6514   emit_operand(src, dst);
6515   // 0x00 - extract from lower 128 bits
6516   // 0x01 - extract from upper 128 bits
6517   emit_int8(imm8 & 0x01);
6518 }
6519 
6520 void Assembler::vextractf32x4(XMMRegister dst, XMMRegister src, uint8_t imm8) {
6521   assert(VM_Version::supports_avx(), "");
6522   assert(imm8 <= 0x03, "imm8: %u", imm8);
6523   int vector_len = VM_Version::supports_evex() ? AVX_512bit : AVX_256bit;
6524   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6525   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6526   emit_int8(0x19);
6527   emit_int8((unsigned char)(0xC0 | encode));
6528   // 0x00 - extract from bits 127:0
6529   // 0x01 - extract from bits 255:128
6530   // 0x02 - extract from bits 383:256
6531   // 0x03 - extract from bits 511:384
6532   emit_int8(imm8 & 0x03);
6533 }
6534 
6535 void Assembler::vextractf32x4(Address dst, XMMRegister src, uint8_t imm8) {
6536   assert(VM_Version::supports_evex(), "");
6537   assert(src != xnoreg, "sanity");
6538   assert(imm8 <= 0x03, "imm8: %u", imm8);
6539   InstructionMark im(this);
6540   InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6541   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
6542   attributes.reset_is_clear_context();
6543   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6544   emit_int8(0x19);
6545   emit_operand(src, dst);
6546   // 0x00 - extract from bits 127:0
6547   // 0x01 - extract from bits 255:128
6548   // 0x02 - extract from bits 383:256
6549   // 0x03 - extract from bits 511:384
6550   emit_int8(imm8 & 0x03);
6551 }
6552 
6553 void Assembler::vextractf64x2(XMMRegister dst, XMMRegister src, uint8_t imm8) {
6554   assert(VM_Version::supports_avx512dq(), "");
6555   assert(imm8 <= 0x03, "imm8: %u", imm8);
6556   InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6557   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6558   emit_int8(0x19);
6559   emit_int8((unsigned char)(0xC0 | encode));
6560   // 0x00 - extract from bits 127:0
6561   // 0x01 - extract from bits 255:128
6562   // 0x02 - extract from bits 383:256
6563   // 0x03 - extract from bits 511:384
6564   emit_int8(imm8 & 0x03);
6565 }
6566 
6567 void Assembler::vextractf64x4(XMMRegister dst, XMMRegister src, uint8_t imm8) {
6568   assert(VM_Version::supports_evex(), "");
6569   assert(imm8 <= 0x01, "imm8: %u", imm8);
6570   InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6571   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6572   emit_int8(0x1B);
6573   emit_int8((unsigned char)(0xC0 | encode));
6574   // 0x00 - extract from lower 256 bits
6575   // 0x01 - extract from upper 256 bits
6576   emit_int8(imm8 & 0x01);
6577 }
6578 
6579 void Assembler::vextractf64x4(Address dst, XMMRegister src, uint8_t imm8) {
6580   assert(VM_Version::supports_evex(), "");
6581   assert(src != xnoreg, "sanity");
6582   assert(imm8 <= 0x01, "imm8: %u", imm8);
6583   InstructionMark im(this);
6584   InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6585   attributes.set_address_attributes(/* tuple_type */ EVEX_T4,/* input_size_in_bits */  EVEX_64bit);
6586   attributes.reset_is_clear_context();
6587   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6588   emit_int8(0x1B);
6589   emit_operand(src, dst);
6590   // 0x00 - extract from lower 256 bits
6591   // 0x01 - extract from upper 256 bits
6592   emit_int8(imm8 & 0x01);
6593 }
6594 
6595 
6596 // legacy word/dword replicate
6597 void Assembler::vpbroadcastw(XMMRegister dst, XMMRegister src) {
6598   assert(VM_Version::supports_avx2(), "");
6599   InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
6600   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6601   emit_int8(0x79);
6602   emit_int8((unsigned char)(0xC0 | encode));
6603 }
6604 
6605 void Assembler::vpbroadcastd(XMMRegister dst, XMMRegister src) {
6606   assert(VM_Version::supports_avx2(), "");
6607   InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6608   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6609   emit_int8(0x58);
6610   emit_int8((unsigned char)(0xC0 | encode));
6611 }
6612 
6613 
6614 // xmm/mem sourced byte/word/dword/qword replicate
6615 
6616 // duplicate 1-byte integer data from src into programmed locations in dest : requires AVX512BW and AVX512VL
6617 void Assembler::evpbroadcastb(XMMRegister dst, XMMRegister src, int vector_len) {
6618   assert(VM_Version::supports_evex(), "");
6619   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
6620   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6621   emit_int8(0x78);
6622   emit_int8((unsigned char)(0xC0 | encode));
6623 }
6624 
6625 void Assembler::evpbroadcastb(XMMRegister dst, Address src, int vector_len) {
6626   assert(VM_Version::supports_evex(), "");
6627   assert(dst != xnoreg, "sanity");
6628   InstructionMark im(this);
6629   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
6630   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_8bit);
6631   // swap src<->dst for encoding
6632   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6633   emit_int8(0x78);
6634   emit_operand(dst, src);
6635 }
6636 
6637 // duplicate 2-byte integer data from src into programmed locations in dest : requires AVX512BW and AVX512VL
6638 void Assembler::evpbroadcastw(XMMRegister dst, XMMRegister src, int vector_len) {
6639   assert(VM_Version::supports_evex(), "");
6640   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
6641   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6642   emit_int8(0x79);
6643   emit_int8((unsigned char)(0xC0 | encode));
6644 }
6645 
6646 void Assembler::evpbroadcastw(XMMRegister dst, Address src, int vector_len) {
6647   assert(VM_Version::supports_evex(), "");
6648   assert(dst != xnoreg, "sanity");
6649   InstructionMark im(this);
6650   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
6651   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_16bit);
6652   // swap src<->dst for encoding
6653   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6654   emit_int8(0x79);
6655   emit_operand(dst, src);
6656 }
6657 
6658 // duplicate 4-byte integer data from src into programmed locations in dest : requires AVX512VL
6659 void Assembler::evpbroadcastd(XMMRegister dst, XMMRegister src, int vector_len) {
6660   assert(VM_Version::supports_evex(), "");
6661   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6662   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6663   emit_int8(0x58);
6664   emit_int8((unsigned char)(0xC0 | encode));
6665 }
6666 
6667 void Assembler::evpbroadcastd(XMMRegister dst, Address src, int vector_len) {
6668   assert(VM_Version::supports_evex(), "");
6669   assert(dst != xnoreg, "sanity");
6670   InstructionMark im(this);
6671   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6672   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
6673   // swap src<->dst for encoding
6674   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6675   emit_int8(0x58);
6676   emit_operand(dst, src);
6677 }
6678 
6679 // duplicate 8-byte integer data from src into programmed locations in dest : requires AVX512VL
6680 void Assembler::evpbroadcastq(XMMRegister dst, XMMRegister src, int vector_len) {
6681   assert(VM_Version::supports_evex(), "");
6682   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6683   attributes.set_rex_vex_w_reverted();
6684   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6685   emit_int8(0x59);
6686   emit_int8((unsigned char)(0xC0 | encode));
6687 }
6688 
6689 void Assembler::evpbroadcastq(XMMRegister dst, Address src, int vector_len) {
6690   assert(VM_Version::supports_evex(), "");
6691   assert(dst != xnoreg, "sanity");
6692   InstructionMark im(this);
6693   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6694   attributes.set_rex_vex_w_reverted();
6695   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
6696   // swap src<->dst for encoding
6697   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6698   emit_int8(0x59);
6699   emit_operand(dst, src);
6700 }
6701 
6702 
6703 // scalar single/double precision replicate
6704 
6705 // duplicate single precision data from src into programmed locations in dest : requires AVX512VL
6706 void Assembler::evpbroadcastss(XMMRegister dst, XMMRegister src, int vector_len) {
6707   assert(VM_Version::supports_evex(), "");
6708   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6709   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6710   emit_int8(0x18);
6711   emit_int8((unsigned char)(0xC0 | encode));
6712 }
6713 
6714 void Assembler::evpbroadcastss(XMMRegister dst, Address src, int vector_len) {
6715   assert(VM_Version::supports_evex(), "");
6716   assert(dst != xnoreg, "sanity");
6717   InstructionMark im(this);
6718   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6719   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
6720   // swap src<->dst for encoding
6721   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6722   emit_int8(0x18);
6723   emit_operand(dst, src);
6724 }
6725 
6726 // duplicate double precision data from src into programmed locations in dest : requires AVX512VL
6727 void Assembler::evpbroadcastsd(XMMRegister dst, XMMRegister src, int vector_len) {
6728   assert(VM_Version::supports_evex(), "");
6729   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6730   attributes.set_rex_vex_w_reverted();
6731   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6732   emit_int8(0x19);
6733   emit_int8((unsigned char)(0xC0 | encode));
6734 }
6735 
6736 void Assembler::evpbroadcastsd(XMMRegister dst, Address src, int vector_len) {
6737   assert(VM_Version::supports_evex(), "");
6738   assert(dst != xnoreg, "sanity");
6739   InstructionMark im(this);
6740   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6741   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
6742   attributes.set_rex_vex_w_reverted();
6743   // swap src<->dst for encoding
6744   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6745   emit_int8(0x19);
6746   emit_operand(dst, src);
6747 }
6748 
6749 
6750 // gpr source broadcast forms
6751 
6752 // duplicate 1-byte integer data from src into programmed locations in dest : requires AVX512BW and AVX512VL
6753 void Assembler::evpbroadcastb(XMMRegister dst, Register src, int vector_len) {
6754   assert(VM_Version::supports_evex(), "");
6755   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
6756   attributes.set_is_evex_instruction();
6757   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6758   emit_int8(0x7A);
6759   emit_int8((unsigned char)(0xC0 | encode));
6760 }
6761 
6762 // duplicate 2-byte integer data from src into programmed locations in dest : requires AVX512BW and AVX512VL
6763 void Assembler::evpbroadcastw(XMMRegister dst, Register src, int vector_len) {
6764   assert(VM_Version::supports_evex(), "");
6765   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
6766   attributes.set_is_evex_instruction();
6767   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6768   emit_int8(0x7B);
6769   emit_int8((unsigned char)(0xC0 | encode));
6770 }
6771 
6772 // duplicate 4-byte integer data from src into programmed locations in dest : requires AVX512VL
6773 void Assembler::evpbroadcastd(XMMRegister dst, Register src, int vector_len) {
6774   assert(VM_Version::supports_evex(), "");
6775   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6776   attributes.set_is_evex_instruction();
6777   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6778   emit_int8(0x7C);
6779   emit_int8((unsigned char)(0xC0 | encode));
6780 }
6781 
6782 // duplicate 8-byte integer data from src into programmed locations in dest : requires AVX512VL
6783 void Assembler::evpbroadcastq(XMMRegister dst, Register src, int vector_len) {
6784   assert(VM_Version::supports_evex(), "");
6785   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6786   attributes.set_is_evex_instruction();
6787   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6788   emit_int8(0x7C);
6789   emit_int8((unsigned char)(0xC0 | encode));
6790 }
6791 
6792 
6793 // Carry-Less Multiplication Quadword
6794 void Assembler::pclmulqdq(XMMRegister dst, XMMRegister src, int mask) {
6795   assert(VM_Version::supports_clmul(), "");
6796   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
6797   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6798   emit_int8(0x44);
6799   emit_int8((unsigned char)(0xC0 | encode));
6800   emit_int8((unsigned char)mask);
6801 }
6802 
6803 // Carry-Less Multiplication Quadword
6804 void Assembler::vpclmulqdq(XMMRegister dst, XMMRegister nds, XMMRegister src, int mask) {
6805   assert(VM_Version::supports_avx() && VM_Version::supports_clmul(), "");
6806   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
6807   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6808   emit_int8(0x44);
6809   emit_int8((unsigned char)(0xC0 | encode));
6810   emit_int8((unsigned char)mask);
6811 }
6812 
6813 void Assembler::vzeroupper() {
6814   if (VM_Version::supports_vzeroupper()) {
6815     InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
6816     (void)vex_prefix_and_encode(0, 0, 0, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
6817     emit_int8(0x77);
6818   }
6819 }
6820 
6821 #ifndef _LP64
6822 // 32bit only pieces of the assembler
6823 
6824 void Assembler::cmp_literal32(Register src1, int32_t imm32, RelocationHolder const& rspec) {
6825   // NO PREFIX AS NEVER 64BIT
6826   InstructionMark im(this);
6827   emit_int8((unsigned char)0x81);
6828   emit_int8((unsigned char)(0xF8 | src1->encoding()));
6829   emit_data(imm32, rspec, 0);
6830 }
6831 
6832 void Assembler::cmp_literal32(Address src1, int32_t imm32, RelocationHolder const& rspec) {
6833   // NO PREFIX AS NEVER 64BIT (not even 32bit versions of 64bit regs
6834   InstructionMark im(this);
6835   emit_int8((unsigned char)0x81);
6836   emit_operand(rdi, src1);
6837   emit_data(imm32, rspec, 0);
6838 }
6839 
6840 // The 64-bit (32bit platform) cmpxchg compares the value at adr with the contents of rdx:rax,
6841 // and stores rcx:rbx into adr if so; otherwise, the value at adr is loaded
6842 // into rdx:rax.  The ZF is set if the compared values were equal, and cleared otherwise.
6843 void Assembler::cmpxchg8(Address adr) {
6844   InstructionMark im(this);
6845   emit_int8(0x0F);
6846   emit_int8((unsigned char)0xC7);
6847   emit_operand(rcx, adr);
6848 }
6849 
6850 void Assembler::decl(Register dst) {
6851   // Don't use it directly. Use MacroAssembler::decrementl() instead.
6852  emit_int8(0x48 | dst->encoding());
6853 }
6854 
6855 #endif // _LP64
6856 
6857 // 64bit typically doesn't use the x87 but needs to for the trig funcs
6858 
6859 void Assembler::fabs() {
6860   emit_int8((unsigned char)0xD9);
6861   emit_int8((unsigned char)0xE1);
6862 }
6863 
6864 void Assembler::fadd(int i) {
6865   emit_farith(0xD8, 0xC0, i);
6866 }
6867 
6868 void Assembler::fadd_d(Address src) {
6869   InstructionMark im(this);
6870   emit_int8((unsigned char)0xDC);
6871   emit_operand32(rax, src);
6872 }
6873 
6874 void Assembler::fadd_s(Address src) {
6875   InstructionMark im(this);
6876   emit_int8((unsigned char)0xD8);
6877   emit_operand32(rax, src);
6878 }
6879 
6880 void Assembler::fadda(int i) {
6881   emit_farith(0xDC, 0xC0, i);
6882 }
6883 
6884 void Assembler::faddp(int i) {
6885   emit_farith(0xDE, 0xC0, i);
6886 }
6887 
6888 void Assembler::fchs() {
6889   emit_int8((unsigned char)0xD9);
6890   emit_int8((unsigned char)0xE0);
6891 }
6892 
6893 void Assembler::fcom(int i) {
6894   emit_farith(0xD8, 0xD0, i);
6895 }
6896 
6897 void Assembler::fcomp(int i) {
6898   emit_farith(0xD8, 0xD8, i);
6899 }
6900 
6901 void Assembler::fcomp_d(Address src) {
6902   InstructionMark im(this);
6903   emit_int8((unsigned char)0xDC);
6904   emit_operand32(rbx, src);
6905 }
6906 
6907 void Assembler::fcomp_s(Address src) {
6908   InstructionMark im(this);
6909   emit_int8((unsigned char)0xD8);
6910   emit_operand32(rbx, src);
6911 }
6912 
6913 void Assembler::fcompp() {
6914   emit_int8((unsigned char)0xDE);
6915   emit_int8((unsigned char)0xD9);
6916 }
6917 
6918 void Assembler::fcos() {
6919   emit_int8((unsigned char)0xD9);
6920   emit_int8((unsigned char)0xFF);
6921 }
6922 
6923 void Assembler::fdecstp() {
6924   emit_int8((unsigned char)0xD9);
6925   emit_int8((unsigned char)0xF6);
6926 }
6927 
6928 void Assembler::fdiv(int i) {
6929   emit_farith(0xD8, 0xF0, i);
6930 }
6931 
6932 void Assembler::fdiv_d(Address src) {
6933   InstructionMark im(this);
6934   emit_int8((unsigned char)0xDC);
6935   emit_operand32(rsi, src);
6936 }
6937 
6938 void Assembler::fdiv_s(Address src) {
6939   InstructionMark im(this);
6940   emit_int8((unsigned char)0xD8);
6941   emit_operand32(rsi, src);
6942 }
6943 
6944 void Assembler::fdiva(int i) {
6945   emit_farith(0xDC, 0xF8, i);
6946 }
6947 
6948 // Note: The Intel manual (Pentium Processor User's Manual, Vol.3, 1994)
6949 //       is erroneous for some of the floating-point instructions below.
6950 
6951 void Assembler::fdivp(int i) {
6952   emit_farith(0xDE, 0xF8, i);                    // ST(0) <- ST(0) / ST(1) and pop (Intel manual wrong)
6953 }
6954 
6955 void Assembler::fdivr(int i) {
6956   emit_farith(0xD8, 0xF8, i);
6957 }
6958 
6959 void Assembler::fdivr_d(Address src) {
6960   InstructionMark im(this);
6961   emit_int8((unsigned char)0xDC);
6962   emit_operand32(rdi, src);
6963 }
6964 
6965 void Assembler::fdivr_s(Address src) {
6966   InstructionMark im(this);
6967   emit_int8((unsigned char)0xD8);
6968   emit_operand32(rdi, src);
6969 }
6970 
6971 void Assembler::fdivra(int i) {
6972   emit_farith(0xDC, 0xF0, i);
6973 }
6974 
6975 void Assembler::fdivrp(int i) {
6976   emit_farith(0xDE, 0xF0, i);                    // ST(0) <- ST(1) / ST(0) and pop (Intel manual wrong)
6977 }
6978 
6979 void Assembler::ffree(int i) {
6980   emit_farith(0xDD, 0xC0, i);
6981 }
6982 
6983 void Assembler::fild_d(Address adr) {
6984   InstructionMark im(this);
6985   emit_int8((unsigned char)0xDF);
6986   emit_operand32(rbp, adr);
6987 }
6988 
6989 void Assembler::fild_s(Address adr) {
6990   InstructionMark im(this);
6991   emit_int8((unsigned char)0xDB);
6992   emit_operand32(rax, adr);
6993 }
6994 
6995 void Assembler::fincstp() {
6996   emit_int8((unsigned char)0xD9);
6997   emit_int8((unsigned char)0xF7);
6998 }
6999 
7000 void Assembler::finit() {
7001   emit_int8((unsigned char)0x9B);
7002   emit_int8((unsigned char)0xDB);
7003   emit_int8((unsigned char)0xE3);
7004 }
7005 
7006 void Assembler::fist_s(Address adr) {
7007   InstructionMark im(this);
7008   emit_int8((unsigned char)0xDB);
7009   emit_operand32(rdx, adr);
7010 }
7011 
7012 void Assembler::fistp_d(Address adr) {
7013   InstructionMark im(this);
7014   emit_int8((unsigned char)0xDF);
7015   emit_operand32(rdi, adr);
7016 }
7017 
7018 void Assembler::fistp_s(Address adr) {
7019   InstructionMark im(this);
7020   emit_int8((unsigned char)0xDB);
7021   emit_operand32(rbx, adr);
7022 }
7023 
7024 void Assembler::fld1() {
7025   emit_int8((unsigned char)0xD9);
7026   emit_int8((unsigned char)0xE8);
7027 }
7028 
7029 void Assembler::fld_d(Address adr) {
7030   InstructionMark im(this);
7031   emit_int8((unsigned char)0xDD);
7032   emit_operand32(rax, adr);
7033 }
7034 
7035 void Assembler::fld_s(Address adr) {
7036   InstructionMark im(this);
7037   emit_int8((unsigned char)0xD9);
7038   emit_operand32(rax, adr);
7039 }
7040 
7041 
7042 void Assembler::fld_s(int index) {
7043   emit_farith(0xD9, 0xC0, index);
7044 }
7045 
7046 void Assembler::fld_x(Address adr) {
7047   InstructionMark im(this);
7048   emit_int8((unsigned char)0xDB);
7049   emit_operand32(rbp, adr);
7050 }
7051 
7052 void Assembler::fldcw(Address src) {
7053   InstructionMark im(this);
7054   emit_int8((unsigned char)0xD9);
7055   emit_operand32(rbp, src);
7056 }
7057 
7058 void Assembler::fldenv(Address src) {
7059   InstructionMark im(this);
7060   emit_int8((unsigned char)0xD9);
7061   emit_operand32(rsp, src);
7062 }
7063 
7064 void Assembler::fldlg2() {
7065   emit_int8((unsigned char)0xD9);
7066   emit_int8((unsigned char)0xEC);
7067 }
7068 
7069 void Assembler::fldln2() {
7070   emit_int8((unsigned char)0xD9);
7071   emit_int8((unsigned char)0xED);
7072 }
7073 
7074 void Assembler::fldz() {
7075   emit_int8((unsigned char)0xD9);
7076   emit_int8((unsigned char)0xEE);
7077 }
7078 
7079 void Assembler::flog() {
7080   fldln2();
7081   fxch();
7082   fyl2x();
7083 }
7084 
7085 void Assembler::flog10() {
7086   fldlg2();
7087   fxch();
7088   fyl2x();
7089 }
7090 
7091 void Assembler::fmul(int i) {
7092   emit_farith(0xD8, 0xC8, i);
7093 }
7094 
7095 void Assembler::fmul_d(Address src) {
7096   InstructionMark im(this);
7097   emit_int8((unsigned char)0xDC);
7098   emit_operand32(rcx, src);
7099 }
7100 
7101 void Assembler::fmul_s(Address src) {
7102   InstructionMark im(this);
7103   emit_int8((unsigned char)0xD8);
7104   emit_operand32(rcx, src);
7105 }
7106 
7107 void Assembler::fmula(int i) {
7108   emit_farith(0xDC, 0xC8, i);
7109 }
7110 
7111 void Assembler::fmulp(int i) {
7112   emit_farith(0xDE, 0xC8, i);
7113 }
7114 
7115 void Assembler::fnsave(Address dst) {
7116   InstructionMark im(this);
7117   emit_int8((unsigned char)0xDD);
7118   emit_operand32(rsi, dst);
7119 }
7120 
7121 void Assembler::fnstcw(Address src) {
7122   InstructionMark im(this);
7123   emit_int8((unsigned char)0x9B);
7124   emit_int8((unsigned char)0xD9);
7125   emit_operand32(rdi, src);
7126 }
7127 
7128 void Assembler::fnstsw_ax() {
7129   emit_int8((unsigned char)0xDF);
7130   emit_int8((unsigned char)0xE0);
7131 }
7132 
7133 void Assembler::fprem() {
7134   emit_int8((unsigned char)0xD9);
7135   emit_int8((unsigned char)0xF8);
7136 }
7137 
7138 void Assembler::fprem1() {
7139   emit_int8((unsigned char)0xD9);
7140   emit_int8((unsigned char)0xF5);
7141 }
7142 
7143 void Assembler::frstor(Address src) {
7144   InstructionMark im(this);
7145   emit_int8((unsigned char)0xDD);
7146   emit_operand32(rsp, src);
7147 }
7148 
7149 void Assembler::fsin() {
7150   emit_int8((unsigned char)0xD9);
7151   emit_int8((unsigned char)0xFE);
7152 }
7153 
7154 void Assembler::fsqrt() {
7155   emit_int8((unsigned char)0xD9);
7156   emit_int8((unsigned char)0xFA);
7157 }
7158 
7159 void Assembler::fst_d(Address adr) {
7160   InstructionMark im(this);
7161   emit_int8((unsigned char)0xDD);
7162   emit_operand32(rdx, adr);
7163 }
7164 
7165 void Assembler::fst_s(Address adr) {
7166   InstructionMark im(this);
7167   emit_int8((unsigned char)0xD9);
7168   emit_operand32(rdx, adr);
7169 }
7170 
7171 void Assembler::fstp_d(Address adr) {
7172   InstructionMark im(this);
7173   emit_int8((unsigned char)0xDD);
7174   emit_operand32(rbx, adr);
7175 }
7176 
7177 void Assembler::fstp_d(int index) {
7178   emit_farith(0xDD, 0xD8, index);
7179 }
7180 
7181 void Assembler::fstp_s(Address adr) {
7182   InstructionMark im(this);
7183   emit_int8((unsigned char)0xD9);
7184   emit_operand32(rbx, adr);
7185 }
7186 
7187 void Assembler::fstp_x(Address adr) {
7188   InstructionMark im(this);
7189   emit_int8((unsigned char)0xDB);
7190   emit_operand32(rdi, adr);
7191 }
7192 
7193 void Assembler::fsub(int i) {
7194   emit_farith(0xD8, 0xE0, i);
7195 }
7196 
7197 void Assembler::fsub_d(Address src) {
7198   InstructionMark im(this);
7199   emit_int8((unsigned char)0xDC);
7200   emit_operand32(rsp, src);
7201 }
7202 
7203 void Assembler::fsub_s(Address src) {
7204   InstructionMark im(this);
7205   emit_int8((unsigned char)0xD8);
7206   emit_operand32(rsp, src);
7207 }
7208 
7209 void Assembler::fsuba(int i) {
7210   emit_farith(0xDC, 0xE8, i);
7211 }
7212 
7213 void Assembler::fsubp(int i) {
7214   emit_farith(0xDE, 0xE8, i);                    // ST(0) <- ST(0) - ST(1) and pop (Intel manual wrong)
7215 }
7216 
7217 void Assembler::fsubr(int i) {
7218   emit_farith(0xD8, 0xE8, i);
7219 }
7220 
7221 void Assembler::fsubr_d(Address src) {
7222   InstructionMark im(this);
7223   emit_int8((unsigned char)0xDC);
7224   emit_operand32(rbp, src);
7225 }
7226 
7227 void Assembler::fsubr_s(Address src) {
7228   InstructionMark im(this);
7229   emit_int8((unsigned char)0xD8);
7230   emit_operand32(rbp, src);
7231 }
7232 
7233 void Assembler::fsubra(int i) {
7234   emit_farith(0xDC, 0xE0, i);
7235 }
7236 
7237 void Assembler::fsubrp(int i) {
7238   emit_farith(0xDE, 0xE0, i);                    // ST(0) <- ST(1) - ST(0) and pop (Intel manual wrong)
7239 }
7240 
7241 void Assembler::ftan() {
7242   emit_int8((unsigned char)0xD9);
7243   emit_int8((unsigned char)0xF2);
7244   emit_int8((unsigned char)0xDD);
7245   emit_int8((unsigned char)0xD8);
7246 }
7247 
7248 void Assembler::ftst() {
7249   emit_int8((unsigned char)0xD9);
7250   emit_int8((unsigned char)0xE4);
7251 }
7252 
7253 void Assembler::fucomi(int i) {
7254   // make sure the instruction is supported (introduced for P6, together with cmov)
7255   guarantee(VM_Version::supports_cmov(), "illegal instruction");
7256   emit_farith(0xDB, 0xE8, i);
7257 }
7258 
7259 void Assembler::fucomip(int i) {
7260   // make sure the instruction is supported (introduced for P6, together with cmov)
7261   guarantee(VM_Version::supports_cmov(), "illegal instruction");
7262   emit_farith(0xDF, 0xE8, i);
7263 }
7264 
7265 void Assembler::fwait() {
7266   emit_int8((unsigned char)0x9B);
7267 }
7268 
7269 void Assembler::fxch(int i) {
7270   emit_farith(0xD9, 0xC8, i);
7271 }
7272 
7273 void Assembler::fyl2x() {
7274   emit_int8((unsigned char)0xD9);
7275   emit_int8((unsigned char)0xF1);
7276 }
7277 
7278 void Assembler::frndint() {
7279   emit_int8((unsigned char)0xD9);
7280   emit_int8((unsigned char)0xFC);
7281 }
7282 
7283 void Assembler::f2xm1() {
7284   emit_int8((unsigned char)0xD9);
7285   emit_int8((unsigned char)0xF0);
7286 }
7287 
7288 void Assembler::fldl2e() {
7289   emit_int8((unsigned char)0xD9);
7290   emit_int8((unsigned char)0xEA);
7291 }
7292 
7293 // SSE SIMD prefix byte values corresponding to VexSimdPrefix encoding.
7294 static int simd_pre[4] = { 0, 0x66, 0xF3, 0xF2 };
7295 // SSE opcode second byte values (first is 0x0F) corresponding to VexOpcode encoding.
7296 static int simd_opc[4] = { 0,    0, 0x38, 0x3A };
7297 
7298 // Generate SSE legacy REX prefix and SIMD opcode based on VEX encoding.
7299 void Assembler::rex_prefix(Address adr, XMMRegister xreg, VexSimdPrefix pre, VexOpcode opc, bool rex_w) {
7300   if (pre > 0) {
7301     emit_int8(simd_pre[pre]);
7302   }
7303   if (rex_w) {
7304     prefixq(adr, xreg);
7305   } else {
7306     prefix(adr, xreg);
7307   }
7308   if (opc > 0) {
7309     emit_int8(0x0F);
7310     int opc2 = simd_opc[opc];
7311     if (opc2 > 0) {
7312       emit_int8(opc2);
7313     }
7314   }
7315 }
7316 
7317 int Assembler::rex_prefix_and_encode(int dst_enc, int src_enc, VexSimdPrefix pre, VexOpcode opc, bool rex_w) {
7318   if (pre > 0) {
7319     emit_int8(simd_pre[pre]);
7320   }
7321   int encode = (rex_w) ? prefixq_and_encode(dst_enc, src_enc) : prefix_and_encode(dst_enc, src_enc);
7322   if (opc > 0) {
7323     emit_int8(0x0F);
7324     int opc2 = simd_opc[opc];
7325     if (opc2 > 0) {
7326       emit_int8(opc2);
7327     }
7328   }
7329   return encode;
7330 }
7331 
7332 
7333 void Assembler::vex_prefix(bool vex_r, bool vex_b, bool vex_x, int nds_enc, VexSimdPrefix pre, VexOpcode opc) {
7334   int vector_len = _attributes->get_vector_len();
7335   bool vex_w = _attributes->is_rex_vex_w();
7336   if (vex_b || vex_x || vex_w || (opc == VEX_OPCODE_0F_38) || (opc == VEX_OPCODE_0F_3A)) {
7337     prefix(VEX_3bytes);
7338 
7339     int byte1 = (vex_r ? VEX_R : 0) | (vex_x ? VEX_X : 0) | (vex_b ? VEX_B : 0);
7340     byte1 = (~byte1) & 0xE0;
7341     byte1 |= opc;
7342     emit_int8(byte1);
7343 
7344     int byte2 = ((~nds_enc) & 0xf) << 3;
7345     byte2 |= (vex_w ? VEX_W : 0) | ((vector_len > 0) ? 4 : 0) | pre;
7346     emit_int8(byte2);
7347   } else {
7348     prefix(VEX_2bytes);
7349 
7350     int byte1 = vex_r ? VEX_R : 0;
7351     byte1 = (~byte1) & 0x80;
7352     byte1 |= ((~nds_enc) & 0xf) << 3;
7353     byte1 |= ((vector_len > 0 ) ? 4 : 0) | pre;
7354     emit_int8(byte1);
7355   }
7356 }
7357 
7358 // This is a 4 byte encoding
7359 void Assembler::evex_prefix(bool vex_r, bool vex_b, bool vex_x, bool evex_r, bool evex_v, int nds_enc, VexSimdPrefix pre, VexOpcode opc){
7360   // EVEX 0x62 prefix
7361   prefix(EVEX_4bytes);
7362   bool vex_w = _attributes->is_rex_vex_w();
7363   int evex_encoding = (vex_w ? VEX_W : 0);
7364   // EVEX.b is not currently used for broadcast of single element or data rounding modes
7365   _attributes->set_evex_encoding(evex_encoding);
7366 
7367   // P0: byte 2, initialized to RXBR`00mm
7368   // instead of not'd
7369   int byte2 = (vex_r ? VEX_R : 0) | (vex_x ? VEX_X : 0) | (vex_b ? VEX_B : 0) | (evex_r ? EVEX_Rb : 0);
7370   byte2 = (~byte2) & 0xF0;
7371   // confine opc opcode extensions in mm bits to lower two bits
7372   // of form {0F, 0F_38, 0F_3A}
7373   byte2 |= opc;
7374   emit_int8(byte2);
7375 
7376   // P1: byte 3 as Wvvvv1pp
7377   int byte3 = ((~nds_enc) & 0xf) << 3;
7378   // p[10] is always 1
7379   byte3 |= EVEX_F;
7380   byte3 |= (vex_w & 1) << 7;
7381   // confine pre opcode extensions in pp bits to lower two bits
7382   // of form {66, F3, F2}
7383   byte3 |= pre;
7384   emit_int8(byte3);
7385 
7386   // P2: byte 4 as zL'Lbv'aaa
7387   // kregs are implemented in the low 3 bits as aaa (hard code k1, it will be initialized for now)
7388   int byte4 = (_attributes->is_no_reg_mask()) ?
7389               0 :
7390               _attributes->get_embedded_opmask_register_specifier();
7391   // EVEX.v` for extending EVEX.vvvv or VIDX
7392   byte4 |= (evex_v ? 0: EVEX_V);
7393   // third EXEC.b for broadcast actions
7394   byte4 |= (_attributes->is_extended_context() ? EVEX_Rb : 0);
7395   // fourth EVEX.L'L for vector length : 0 is 128, 1 is 256, 2 is 512, currently we do not support 1024
7396   byte4 |= ((_attributes->get_vector_len())& 0x3) << 5;
7397   // last is EVEX.z for zero/merge actions
7398   if (_attributes->is_no_reg_mask() == false) {
7399     byte4 |= (_attributes->is_clear_context() ? EVEX_Z : 0);
7400   }
7401   emit_int8(byte4);
7402 }
7403 
7404 void Assembler::vex_prefix(Address adr, int nds_enc, int xreg_enc, VexSimdPrefix pre, VexOpcode opc, InstructionAttr *attributes) {
7405   bool vex_r = ((xreg_enc & 8) == 8) ? 1 : 0;
7406   bool vex_b = adr.base_needs_rex();
7407   bool vex_x = adr.index_needs_rex();
7408   set_attributes(attributes);
7409   attributes->set_current_assembler(this);
7410 
7411   // if vector length is turned off, revert to AVX for vectors smaller than 512-bit
7412   if (UseAVX > 2 && _legacy_mode_vl && attributes->uses_vl()) {
7413     switch (attributes->get_vector_len()) {
7414     case AVX_128bit:
7415     case AVX_256bit:
7416       attributes->set_is_legacy_mode();
7417       break;
7418     }
7419   }
7420 
7421   // For pure EVEX check and see if this instruction
7422   // is allowed in legacy mode and has resources which will
7423   // fit in it.  Pure EVEX instructions will use set_is_evex_instruction in their definition,
7424   // else that field is set when we encode to EVEX
7425   if (UseAVX > 2 && !attributes->is_legacy_mode() &&
7426       !_is_managed && !attributes->is_evex_instruction()) {
7427     if (!_legacy_mode_vl && attributes->get_vector_len() != AVX_512bit) {
7428       bool check_register_bank = NOT_IA32(true) IA32_ONLY(false);
7429       if (check_register_bank) {
7430         // check nds_enc and xreg_enc for upper bank usage
7431         if (nds_enc < 16 && xreg_enc < 16) {
7432           attributes->set_is_legacy_mode();
7433         }
7434       } else {
7435         attributes->set_is_legacy_mode();
7436       }
7437     }
7438   }
7439 
7440   _is_managed = false;
7441   if (UseAVX > 2 && !attributes->is_legacy_mode())
7442   {
7443     bool evex_r = (xreg_enc >= 16);
7444     bool evex_v = (nds_enc >= 16);
7445     attributes->set_is_evex_instruction();
7446     evex_prefix(vex_r, vex_b, vex_x, evex_r, evex_v, nds_enc, pre, opc);
7447   } else {
7448     if (UseAVX > 2 && attributes->is_rex_vex_w_reverted()) {
7449       attributes->set_rex_vex_w(false);
7450     }
7451     vex_prefix(vex_r, vex_b, vex_x, nds_enc, pre, opc);
7452   }
7453 }
7454 
7455 int Assembler::vex_prefix_and_encode(int dst_enc, int nds_enc, int src_enc, VexSimdPrefix pre, VexOpcode opc, InstructionAttr *attributes) {
7456   bool vex_r = ((dst_enc & 8) == 8) ? 1 : 0;
7457   bool vex_b = ((src_enc & 8) == 8) ? 1 : 0;
7458   bool vex_x = false;
7459   set_attributes(attributes);
7460   attributes->set_current_assembler(this);
7461   bool check_register_bank = NOT_IA32(true) IA32_ONLY(false);
7462 
7463   // if vector length is turned off, revert to AVX for vectors smaller than 512-bit
7464   if (UseAVX > 2 && _legacy_mode_vl && attributes->uses_vl()) {
7465     switch (attributes->get_vector_len()) {
7466     case AVX_128bit:
7467     case AVX_256bit:
7468       if (check_register_bank) {
7469         if (dst_enc >= 16 || nds_enc >= 16 || src_enc >= 16) {
7470           // up propagate arithmetic instructions to meet RA requirements
7471           attributes->set_vector_len(AVX_512bit);
7472         } else {
7473           attributes->set_is_legacy_mode();
7474         }
7475       } else {
7476         attributes->set_is_legacy_mode();
7477       }
7478       break;
7479     }
7480   }
7481 
7482   // For pure EVEX check and see if this instruction
7483   // is allowed in legacy mode and has resources which will
7484   // fit in it.  Pure EVEX instructions will use set_is_evex_instruction in their definition,
7485   // else that field is set when we encode to EVEX
7486   if (UseAVX > 2 && !attributes->is_legacy_mode() &&
7487       !_is_managed && !attributes->is_evex_instruction()) {
7488     if (!_legacy_mode_vl && attributes->get_vector_len() != AVX_512bit) {
7489       if (check_register_bank) {
7490         // check dst_enc, nds_enc and src_enc for upper bank usage
7491         if (dst_enc < 16 && nds_enc < 16 && src_enc < 16) {
7492           attributes->set_is_legacy_mode();
7493         }
7494       } else {
7495         attributes->set_is_legacy_mode();
7496       }
7497     }
7498   }
7499 
7500   _is_managed = false;
7501   if (UseAVX > 2 && !attributes->is_legacy_mode())
7502   {
7503     bool evex_r = (dst_enc >= 16);
7504     bool evex_v = (nds_enc >= 16);
7505     // can use vex_x as bank extender on rm encoding
7506     vex_x = (src_enc >= 16);
7507     attributes->set_is_evex_instruction();
7508     evex_prefix(vex_r, vex_b, vex_x, evex_r, evex_v, nds_enc, pre, opc);
7509   } else {
7510     if (UseAVX > 2 && attributes->is_rex_vex_w_reverted()) {
7511       attributes->set_rex_vex_w(false);
7512     }
7513     vex_prefix(vex_r, vex_b, vex_x, nds_enc, pre, opc);
7514   }
7515 
7516   // return modrm byte components for operands
7517   return (((dst_enc & 7) << 3) | (src_enc & 7));
7518 }
7519 
7520 
7521 void Assembler::simd_prefix(XMMRegister xreg, XMMRegister nds, Address adr, VexSimdPrefix pre,
7522                             VexOpcode opc, InstructionAttr *attributes) {
7523   if (UseAVX > 0) {
7524     int xreg_enc = xreg->encoding();
7525     int nds_enc = nds->is_valid() ? nds->encoding() : 0;
7526     vex_prefix(adr, nds_enc, xreg_enc, pre, opc, attributes);
7527   } else {
7528     assert((nds == xreg) || (nds == xnoreg), "wrong sse encoding");
7529     rex_prefix(adr, xreg, pre, opc, attributes->is_rex_vex_w());
7530   }
7531 }
7532 
7533 int Assembler::simd_prefix_and_encode(XMMRegister dst, XMMRegister nds, XMMRegister src, VexSimdPrefix pre,
7534                                       VexOpcode opc, InstructionAttr *attributes) {
7535   int dst_enc = dst->encoding();
7536   int src_enc = src->encoding();
7537   if (UseAVX > 0) {
7538     int nds_enc = nds->is_valid() ? nds->encoding() : 0;
7539     return vex_prefix_and_encode(dst_enc, nds_enc, src_enc, pre, opc, attributes);
7540   } else {
7541     assert((nds == dst) || (nds == src) || (nds == xnoreg), "wrong sse encoding");
7542     return rex_prefix_and_encode(dst_enc, src_enc, pre, opc, attributes->is_rex_vex_w());
7543   }
7544 }
7545 
7546 void Assembler::cmppd(XMMRegister dst, XMMRegister nds, XMMRegister src, int cop, int vector_len) {
7547   assert(VM_Version::supports_avx(), "");
7548   assert(!VM_Version::supports_evex(), "");
7549   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
7550   int encode = simd_prefix_and_encode(dst, nds, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7551   emit_int8((unsigned char)0xC2);
7552   emit_int8((unsigned char)(0xC0 | encode));
7553   emit_int8((unsigned char)(0xF & cop));
7554 }
7555 
7556 void Assembler::blendvpd(XMMRegister dst, XMMRegister nds, XMMRegister src1, XMMRegister src2, int vector_len) {
7557   assert(VM_Version::supports_avx(), "");
7558   assert(!VM_Version::supports_evex(), "");
7559   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
7560   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src1->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
7561   emit_int8((unsigned char)0x4B);
7562   emit_int8((unsigned char)(0xC0 | encode));
7563   int src2_enc = src2->encoding();
7564   emit_int8((unsigned char)(0xF0 & src2_enc<<4));
7565 }
7566 
7567 void Assembler::cmpps(XMMRegister dst, XMMRegister nds, XMMRegister src, int cop, int vector_len) {
7568   assert(VM_Version::supports_avx(), "");
7569   assert(!VM_Version::supports_evex(), "");
7570   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
7571   int encode = simd_prefix_and_encode(dst, nds, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
7572   emit_int8((unsigned char)0xC2);
7573   emit_int8((unsigned char)(0xC0 | encode));
7574   emit_int8((unsigned char)(0xF & cop));
7575 }
7576 
7577 void Assembler::blendvps(XMMRegister dst, XMMRegister nds, XMMRegister src1, XMMRegister src2, int vector_len) {
7578   assert(VM_Version::supports_avx(), "");
7579   assert(!VM_Version::supports_evex(), "");
7580   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
7581   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src1->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
7582   emit_int8((unsigned char)0x4A);
7583   emit_int8((unsigned char)(0xC0 | encode));
7584   int src2_enc = src2->encoding();
7585   emit_int8((unsigned char)(0xF0 & src2_enc<<4));
7586 }
7587 
7588 void Assembler::vpblendd(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8, int vector_len) {
7589   assert(VM_Version::supports_avx2(), "");
7590   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
7591   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
7592   emit_int8((unsigned char)0x02);
7593   emit_int8((unsigned char)(0xC0 | encode));
7594   emit_int8((unsigned char)imm8);
7595 }
7596 
7597 void Assembler::shlxl(Register dst, Register src1, Register src2) {
7598   assert(VM_Version::supports_bmi2(), "");
7599   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
7600   int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src1->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
7601   emit_int8((unsigned char)0xF7);
7602   emit_int8((unsigned char)(0xC0 | encode));
7603 }
7604 
7605 void Assembler::shlxq(Register dst, Register src1, Register src2) {
7606   assert(VM_Version::supports_bmi2(), "");
7607   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
7608   int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src1->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
7609   emit_int8((unsigned char)0xF7);
7610   emit_int8((unsigned char)(0xC0 | encode));
7611 }
7612 
7613 #ifndef _LP64
7614 
7615 void Assembler::incl(Register dst) {
7616   // Don't use it directly. Use MacroAssembler::incrementl() instead.
7617   emit_int8(0x40 | dst->encoding());
7618 }
7619 
7620 void Assembler::lea(Register dst, Address src) {
7621   leal(dst, src);
7622 }
7623 
7624 void Assembler::mov_literal32(Address dst, int32_t imm32, RelocationHolder const& rspec) {
7625   InstructionMark im(this);
7626   emit_int8((unsigned char)0xC7);
7627   emit_operand(rax, dst);
7628   emit_data((int)imm32, rspec, 0);
7629 }
7630 
7631 void Assembler::mov_literal32(Register dst, int32_t imm32, RelocationHolder const& rspec) {
7632   InstructionMark im(this);
7633   int encode = prefix_and_encode(dst->encoding());
7634   emit_int8((unsigned char)(0xB8 | encode));
7635   emit_data((int)imm32, rspec, 0);
7636 }
7637 
7638 void Assembler::popa() { // 32bit
7639   emit_int8(0x61);
7640 }
7641 
7642 void Assembler::push_literal32(int32_t imm32, RelocationHolder const& rspec) {
7643   InstructionMark im(this);
7644   emit_int8(0x68);
7645   emit_data(imm32, rspec, 0);
7646 }
7647 
7648 void Assembler::pusha() { // 32bit
7649   emit_int8(0x60);
7650 }
7651 
7652 void Assembler::set_byte_if_not_zero(Register dst) {
7653   emit_int8(0x0F);
7654   emit_int8((unsigned char)0x95);
7655   emit_int8((unsigned char)(0xE0 | dst->encoding()));
7656 }
7657 
7658 void Assembler::shldl(Register dst, Register src) {
7659   emit_int8(0x0F);
7660   emit_int8((unsigned char)0xA5);
7661   emit_int8((unsigned char)(0xC0 | src->encoding() << 3 | dst->encoding()));
7662 }
7663 
7664 // 0F A4 / r ib
7665 void Assembler::shldl(Register dst, Register src, int8_t imm8) {
7666   emit_int8(0x0F);
7667   emit_int8((unsigned char)0xA4);
7668   emit_int8((unsigned char)(0xC0 | src->encoding() << 3 | dst->encoding()));
7669   emit_int8(imm8);
7670 }
7671 
7672 void Assembler::shrdl(Register dst, Register src) {
7673   emit_int8(0x0F);
7674   emit_int8((unsigned char)0xAD);
7675   emit_int8((unsigned char)(0xC0 | src->encoding() << 3 | dst->encoding()));
7676 }
7677 
7678 #else // LP64
7679 
7680 void Assembler::set_byte_if_not_zero(Register dst) {
7681   int enc = prefix_and_encode(dst->encoding(), true);
7682   emit_int8(0x0F);
7683   emit_int8((unsigned char)0x95);
7684   emit_int8((unsigned char)(0xE0 | enc));
7685 }
7686 
7687 // 64bit only pieces of the assembler
7688 // This should only be used by 64bit instructions that can use rip-relative
7689 // it cannot be used by instructions that want an immediate value.
7690 
7691 bool Assembler::reachable(AddressLiteral adr) {
7692   int64_t disp;
7693   // None will force a 64bit literal to the code stream. Likely a placeholder
7694   // for something that will be patched later and we need to certain it will
7695   // always be reachable.
7696   if (adr.reloc() == relocInfo::none) {
7697     return false;
7698   }
7699   if (adr.reloc() == relocInfo::internal_word_type) {
7700     // This should be rip relative and easily reachable.
7701     return true;
7702   }
7703   if (adr.reloc() == relocInfo::virtual_call_type ||
7704       adr.reloc() == relocInfo::opt_virtual_call_type ||
7705       adr.reloc() == relocInfo::static_call_type ||
7706       adr.reloc() == relocInfo::static_stub_type ) {
7707     // This should be rip relative within the code cache and easily
7708     // reachable until we get huge code caches. (At which point
7709     // ic code is going to have issues).
7710     return true;
7711   }
7712   if (adr.reloc() != relocInfo::external_word_type &&
7713       adr.reloc() != relocInfo::poll_return_type &&  // these are really external_word but need special
7714       adr.reloc() != relocInfo::poll_type &&         // relocs to identify them
7715       adr.reloc() != relocInfo::runtime_call_type ) {
7716     return false;
7717   }
7718 
7719   // Stress the correction code
7720   if (ForceUnreachable) {
7721     // Must be runtimecall reloc, see if it is in the codecache
7722     // Flipping stuff in the codecache to be unreachable causes issues
7723     // with things like inline caches where the additional instructions
7724     // are not handled.
7725     if (CodeCache::find_blob(adr._target) == NULL) {
7726       return false;
7727     }
7728   }
7729   // For external_word_type/runtime_call_type if it is reachable from where we
7730   // are now (possibly a temp buffer) and where we might end up
7731   // anywhere in the codeCache then we are always reachable.
7732   // This would have to change if we ever save/restore shared code
7733   // to be more pessimistic.
7734   disp = (int64_t)adr._target - ((int64_t)CodeCache::low_bound() + sizeof(int));
7735   if (!is_simm32(disp)) return false;
7736   disp = (int64_t)adr._target - ((int64_t)CodeCache::high_bound() + sizeof(int));
7737   if (!is_simm32(disp)) return false;
7738 
7739   disp = (int64_t)adr._target - ((int64_t)pc() + sizeof(int));
7740 
7741   // Because rip relative is a disp + address_of_next_instruction and we
7742   // don't know the value of address_of_next_instruction we apply a fudge factor
7743   // to make sure we will be ok no matter the size of the instruction we get placed into.
7744   // We don't have to fudge the checks above here because they are already worst case.
7745 
7746   // 12 == override/rex byte, opcode byte, rm byte, sib byte, a 4-byte disp , 4-byte literal
7747   // + 4 because better safe than sorry.
7748   const int fudge = 12 + 4;
7749   if (disp < 0) {
7750     disp -= fudge;
7751   } else {
7752     disp += fudge;
7753   }
7754   return is_simm32(disp);
7755 }
7756 
7757 // Check if the polling page is not reachable from the code cache using rip-relative
7758 // addressing.
7759 bool Assembler::is_polling_page_far() {
7760   intptr_t addr = (intptr_t)os::get_polling_page();
7761   return ForceUnreachable ||
7762          !is_simm32(addr - (intptr_t)CodeCache::low_bound()) ||
7763          !is_simm32(addr - (intptr_t)CodeCache::high_bound());
7764 }
7765 
7766 void Assembler::emit_data64(jlong data,
7767                             relocInfo::relocType rtype,
7768                             int format) {
7769   if (rtype == relocInfo::none) {
7770     emit_int64(data);
7771   } else {
7772     emit_data64(data, Relocation::spec_simple(rtype), format);
7773   }
7774 }
7775 
7776 void Assembler::emit_data64(jlong data,
7777                             RelocationHolder const& rspec,
7778                             int format) {
7779   assert(imm_operand == 0, "default format must be immediate in this file");
7780   assert(imm_operand == format, "must be immediate");
7781   assert(inst_mark() != NULL, "must be inside InstructionMark");
7782   // Do not use AbstractAssembler::relocate, which is not intended for
7783   // embedded words.  Instead, relocate to the enclosing instruction.
7784   code_section()->relocate(inst_mark(), rspec, format);
7785 #ifdef ASSERT
7786   check_relocation(rspec, format);
7787 #endif
7788   emit_int64(data);
7789 }
7790 
7791 int Assembler::prefix_and_encode(int reg_enc, bool byteinst) {
7792   if (reg_enc >= 8) {
7793     prefix(REX_B);
7794     reg_enc -= 8;
7795   } else if (byteinst && reg_enc >= 4) {
7796     prefix(REX);
7797   }
7798   return reg_enc;
7799 }
7800 
7801 int Assembler::prefixq_and_encode(int reg_enc) {
7802   if (reg_enc < 8) {
7803     prefix(REX_W);
7804   } else {
7805     prefix(REX_WB);
7806     reg_enc -= 8;
7807   }
7808   return reg_enc;
7809 }
7810 
7811 int Assembler::prefix_and_encode(int dst_enc, bool dst_is_byte, int src_enc, bool src_is_byte) {
7812   if (dst_enc < 8) {
7813     if (src_enc >= 8) {
7814       prefix(REX_B);
7815       src_enc -= 8;
7816     } else if ((src_is_byte && src_enc >= 4) || (dst_is_byte && dst_enc >= 4)) {
7817       prefix(REX);
7818     }
7819   } else {
7820     if (src_enc < 8) {
7821       prefix(REX_R);
7822     } else {
7823       prefix(REX_RB);
7824       src_enc -= 8;
7825     }
7826     dst_enc -= 8;
7827   }
7828   return dst_enc << 3 | src_enc;
7829 }
7830 
7831 int Assembler::prefixq_and_encode(int dst_enc, int src_enc) {
7832   if (dst_enc < 8) {
7833     if (src_enc < 8) {
7834       prefix(REX_W);
7835     } else {
7836       prefix(REX_WB);
7837       src_enc -= 8;
7838     }
7839   } else {
7840     if (src_enc < 8) {
7841       prefix(REX_WR);
7842     } else {
7843       prefix(REX_WRB);
7844       src_enc -= 8;
7845     }
7846     dst_enc -= 8;
7847   }
7848   return dst_enc << 3 | src_enc;
7849 }
7850 
7851 void Assembler::prefix(Register reg) {
7852   if (reg->encoding() >= 8) {
7853     prefix(REX_B);
7854   }
7855 }
7856 
7857 void Assembler::prefix(Register dst, Register src, Prefix p) {
7858   if (src->encoding() >= 8) {
7859     p = (Prefix)(p | REX_B);
7860   }
7861   if (dst->encoding() >= 8) {
7862     p = (Prefix)( p | REX_R);
7863   }
7864   if (p != Prefix_EMPTY) {
7865     // do not generate an empty prefix
7866     prefix(p);
7867   }
7868 }
7869 
7870 void Assembler::prefix(Register dst, Address adr, Prefix p) {
7871   if (adr.base_needs_rex()) {
7872     if (adr.index_needs_rex()) {
7873       assert(false, "prefix(Register dst, Address adr, Prefix p) does not support handling of an X");
7874     } else {
7875       prefix(REX_B);
7876     }
7877   } else {
7878     if (adr.index_needs_rex()) {
7879       assert(false, "prefix(Register dst, Address adr, Prefix p) does not support handling of an X");
7880     }
7881   }
7882   if (dst->encoding() >= 8) {
7883     p = (Prefix)(p | REX_R);
7884   }
7885   if (p != Prefix_EMPTY) {
7886     // do not generate an empty prefix
7887     prefix(p);
7888   }
7889 }
7890 
7891 void Assembler::prefix(Address adr) {
7892   if (adr.base_needs_rex()) {
7893     if (adr.index_needs_rex()) {
7894       prefix(REX_XB);
7895     } else {
7896       prefix(REX_B);
7897     }
7898   } else {
7899     if (adr.index_needs_rex()) {
7900       prefix(REX_X);
7901     }
7902   }
7903 }
7904 
7905 void Assembler::prefixq(Address adr) {
7906   if (adr.base_needs_rex()) {
7907     if (adr.index_needs_rex()) {
7908       prefix(REX_WXB);
7909     } else {
7910       prefix(REX_WB);
7911     }
7912   } else {
7913     if (adr.index_needs_rex()) {
7914       prefix(REX_WX);
7915     } else {
7916       prefix(REX_W);
7917     }
7918   }
7919 }
7920 
7921 
7922 void Assembler::prefix(Address adr, Register reg, bool byteinst) {
7923   if (reg->encoding() < 8) {
7924     if (adr.base_needs_rex()) {
7925       if (adr.index_needs_rex()) {
7926         prefix(REX_XB);
7927       } else {
7928         prefix(REX_B);
7929       }
7930     } else {
7931       if (adr.index_needs_rex()) {
7932         prefix(REX_X);
7933       } else if (byteinst && reg->encoding() >= 4 ) {
7934         prefix(REX);
7935       }
7936     }
7937   } else {
7938     if (adr.base_needs_rex()) {
7939       if (adr.index_needs_rex()) {
7940         prefix(REX_RXB);
7941       } else {
7942         prefix(REX_RB);
7943       }
7944     } else {
7945       if (adr.index_needs_rex()) {
7946         prefix(REX_RX);
7947       } else {
7948         prefix(REX_R);
7949       }
7950     }
7951   }
7952 }
7953 
7954 void Assembler::prefixq(Address adr, Register src) {
7955   if (src->encoding() < 8) {
7956     if (adr.base_needs_rex()) {
7957       if (adr.index_needs_rex()) {
7958         prefix(REX_WXB);
7959       } else {
7960         prefix(REX_WB);
7961       }
7962     } else {
7963       if (adr.index_needs_rex()) {
7964         prefix(REX_WX);
7965       } else {
7966         prefix(REX_W);
7967       }
7968     }
7969   } else {
7970     if (adr.base_needs_rex()) {
7971       if (adr.index_needs_rex()) {
7972         prefix(REX_WRXB);
7973       } else {
7974         prefix(REX_WRB);
7975       }
7976     } else {
7977       if (adr.index_needs_rex()) {
7978         prefix(REX_WRX);
7979       } else {
7980         prefix(REX_WR);
7981       }
7982     }
7983   }
7984 }
7985 
7986 void Assembler::prefix(Address adr, XMMRegister reg) {
7987   if (reg->encoding() < 8) {
7988     if (adr.base_needs_rex()) {
7989       if (adr.index_needs_rex()) {
7990         prefix(REX_XB);
7991       } else {
7992         prefix(REX_B);
7993       }
7994     } else {
7995       if (adr.index_needs_rex()) {
7996         prefix(REX_X);
7997       }
7998     }
7999   } else {
8000     if (adr.base_needs_rex()) {
8001       if (adr.index_needs_rex()) {
8002         prefix(REX_RXB);
8003       } else {
8004         prefix(REX_RB);
8005       }
8006     } else {
8007       if (adr.index_needs_rex()) {
8008         prefix(REX_RX);
8009       } else {
8010         prefix(REX_R);
8011       }
8012     }
8013   }
8014 }
8015 
8016 void Assembler::prefixq(Address adr, XMMRegister src) {
8017   if (src->encoding() < 8) {
8018     if (adr.base_needs_rex()) {
8019       if (adr.index_needs_rex()) {
8020         prefix(REX_WXB);
8021       } else {
8022         prefix(REX_WB);
8023       }
8024     } else {
8025       if (adr.index_needs_rex()) {
8026         prefix(REX_WX);
8027       } else {
8028         prefix(REX_W);
8029       }
8030     }
8031   } else {
8032     if (adr.base_needs_rex()) {
8033       if (adr.index_needs_rex()) {
8034         prefix(REX_WRXB);
8035       } else {
8036         prefix(REX_WRB);
8037       }
8038     } else {
8039       if (adr.index_needs_rex()) {
8040         prefix(REX_WRX);
8041       } else {
8042         prefix(REX_WR);
8043       }
8044     }
8045   }
8046 }
8047 
8048 void Assembler::adcq(Register dst, int32_t imm32) {
8049   (void) prefixq_and_encode(dst->encoding());
8050   emit_arith(0x81, 0xD0, dst, imm32);
8051 }
8052 
8053 void Assembler::adcq(Register dst, Address src) {
8054   InstructionMark im(this);
8055   prefixq(src, dst);
8056   emit_int8(0x13);
8057   emit_operand(dst, src);
8058 }
8059 
8060 void Assembler::adcq(Register dst, Register src) {
8061   (void) prefixq_and_encode(dst->encoding(), src->encoding());
8062   emit_arith(0x13, 0xC0, dst, src);
8063 }
8064 
8065 void Assembler::addq(Address dst, int32_t imm32) {
8066   InstructionMark im(this);
8067   prefixq(dst);
8068   emit_arith_operand(0x81, rax, dst,imm32);
8069 }
8070 
8071 void Assembler::addq(Address dst, Register src) {
8072   InstructionMark im(this);
8073   prefixq(dst, src);
8074   emit_int8(0x01);
8075   emit_operand(src, dst);
8076 }
8077 
8078 void Assembler::addq(Register dst, int32_t imm32) {
8079   (void) prefixq_and_encode(dst->encoding());
8080   emit_arith(0x81, 0xC0, dst, imm32);
8081 }
8082 
8083 void Assembler::addq(Register dst, Address src) {
8084   InstructionMark im(this);
8085   prefixq(src, dst);
8086   emit_int8(0x03);
8087   emit_operand(dst, src);
8088 }
8089 
8090 void Assembler::addq(Register dst, Register src) {
8091   (void) prefixq_and_encode(dst->encoding(), src->encoding());
8092   emit_arith(0x03, 0xC0, dst, src);
8093 }
8094 
8095 void Assembler::adcxq(Register dst, Register src) {
8096   //assert(VM_Version::supports_adx(), "adx instructions not supported");
8097   emit_int8((unsigned char)0x66);
8098   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
8099   emit_int8(0x0F);
8100   emit_int8(0x38);
8101   emit_int8((unsigned char)0xF6);
8102   emit_int8((unsigned char)(0xC0 | encode));
8103 }
8104 
8105 void Assembler::adoxq(Register dst, Register src) {
8106   //assert(VM_Version::supports_adx(), "adx instructions not supported");
8107   emit_int8((unsigned char)0xF3);
8108   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
8109   emit_int8(0x0F);
8110   emit_int8(0x38);
8111   emit_int8((unsigned char)0xF6);
8112   emit_int8((unsigned char)(0xC0 | encode));
8113 }
8114 
8115 void Assembler::andq(Address dst, int32_t imm32) {
8116   InstructionMark im(this);
8117   prefixq(dst);
8118   emit_int8((unsigned char)0x81);
8119   emit_operand(rsp, dst, 4);
8120   emit_int32(imm32);
8121 }
8122 
8123 void Assembler::andq(Register dst, int32_t imm32) {
8124   (void) prefixq_and_encode(dst->encoding());
8125   emit_arith(0x81, 0xE0, dst, imm32);
8126 }
8127 
8128 void Assembler::andq(Register dst, Address src) {
8129   InstructionMark im(this);
8130   prefixq(src, dst);
8131   emit_int8(0x23);
8132   emit_operand(dst, src);
8133 }
8134 
8135 void Assembler::andq(Register dst, Register src) {
8136   (void) prefixq_and_encode(dst->encoding(), src->encoding());
8137   emit_arith(0x23, 0xC0, dst, src);
8138 }
8139 
8140 void Assembler::andnq(Register dst, Register src1, Register src2) {
8141   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
8142   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
8143   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
8144   emit_int8((unsigned char)0xF2);
8145   emit_int8((unsigned char)(0xC0 | encode));
8146 }
8147 
8148 void Assembler::andnq(Register dst, Register src1, Address src2) {
8149   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
8150   InstructionMark im(this);
8151   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
8152   vex_prefix(src2, src1->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
8153   emit_int8((unsigned char)0xF2);
8154   emit_operand(dst, src2);
8155 }
8156 
8157 void Assembler::bsfq(Register dst, Register src) {
8158   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
8159   emit_int8(0x0F);
8160   emit_int8((unsigned char)0xBC);
8161   emit_int8((unsigned char)(0xC0 | encode));
8162 }
8163 
8164 void Assembler::bsrq(Register dst, Register src) {
8165   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
8166   emit_int8(0x0F);
8167   emit_int8((unsigned char)0xBD);
8168   emit_int8((unsigned char)(0xC0 | encode));
8169 }
8170 
8171 void Assembler::bswapq(Register reg) {
8172   int encode = prefixq_and_encode(reg->encoding());
8173   emit_int8(0x0F);
8174   emit_int8((unsigned char)(0xC8 | encode));
8175 }
8176 
8177 void Assembler::blsiq(Register dst, Register src) {
8178   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
8179   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
8180   int encode = vex_prefix_and_encode(rbx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
8181   emit_int8((unsigned char)0xF3);
8182   emit_int8((unsigned char)(0xC0 | encode));
8183 }
8184 
8185 void Assembler::blsiq(Register dst, Address src) {
8186   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
8187   InstructionMark im(this);
8188   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
8189   vex_prefix(src, dst->encoding(), rbx->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
8190   emit_int8((unsigned char)0xF3);
8191   emit_operand(rbx, src);
8192 }
8193 
8194 void Assembler::blsmskq(Register dst, Register src) {
8195   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
8196   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
8197   int encode = vex_prefix_and_encode(rdx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
8198   emit_int8((unsigned char)0xF3);
8199   emit_int8((unsigned char)(0xC0 | encode));
8200 }
8201 
8202 void Assembler::blsmskq(Register dst, Address src) {
8203   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
8204   InstructionMark im(this);
8205   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
8206   vex_prefix(src, dst->encoding(), rdx->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
8207   emit_int8((unsigned char)0xF3);
8208   emit_operand(rdx, src);
8209 }
8210 
8211 void Assembler::blsrq(Register dst, Register src) {
8212   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
8213   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
8214   int encode = vex_prefix_and_encode(rcx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
8215   emit_int8((unsigned char)0xF3);
8216   emit_int8((unsigned char)(0xC0 | encode));
8217 }
8218 
8219 void Assembler::blsrq(Register dst, Address src) {
8220   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
8221   InstructionMark im(this);
8222   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
8223   vex_prefix(src, dst->encoding(), rcx->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
8224   emit_int8((unsigned char)0xF3);
8225   emit_operand(rcx, src);
8226 }
8227 
8228 void Assembler::cdqq() {
8229   prefix(REX_W);
8230   emit_int8((unsigned char)0x99);
8231 }
8232 
8233 void Assembler::clflush(Address adr) {
8234   prefix(adr);
8235   emit_int8(0x0F);
8236   emit_int8((unsigned char)0xAE);
8237   emit_operand(rdi, adr);
8238 }
8239 
8240 void Assembler::cmovq(Condition cc, Register dst, Register src) {
8241   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
8242   emit_int8(0x0F);
8243   emit_int8(0x40 | cc);
8244   emit_int8((unsigned char)(0xC0 | encode));
8245 }
8246 
8247 void Assembler::cmovq(Condition cc, Register dst, Address src) {
8248   InstructionMark im(this);
8249   prefixq(src, dst);
8250   emit_int8(0x0F);
8251   emit_int8(0x40 | cc);
8252   emit_operand(dst, src);
8253 }
8254 
8255 void Assembler::cmpq(Address dst, int32_t imm32) {
8256   InstructionMark im(this);
8257   prefixq(dst);
8258   emit_int8((unsigned char)0x81);
8259   emit_operand(rdi, dst, 4);
8260   emit_int32(imm32);
8261 }
8262 
8263 void Assembler::cmpq(Register dst, int32_t imm32) {
8264   (void) prefixq_and_encode(dst->encoding());
8265   emit_arith(0x81, 0xF8, dst, imm32);
8266 }
8267 
8268 void Assembler::cmpq(Address dst, Register src) {
8269   InstructionMark im(this);
8270   prefixq(dst, src);
8271   emit_int8(0x3B);
8272   emit_operand(src, dst);
8273 }
8274 
8275 void Assembler::cmpq(Register dst, Register src) {
8276   (void) prefixq_and_encode(dst->encoding(), src->encoding());
8277   emit_arith(0x3B, 0xC0, dst, src);
8278 }
8279 
8280 void Assembler::cmpq(Register dst, Address  src) {
8281   InstructionMark im(this);
8282   prefixq(src, dst);
8283   emit_int8(0x3B);
8284   emit_operand(dst, src);
8285 }
8286 
8287 void Assembler::cmpxchgq(Register reg, Address adr) {
8288   InstructionMark im(this);
8289   prefixq(adr, reg);
8290   emit_int8(0x0F);
8291   emit_int8((unsigned char)0xB1);
8292   emit_operand(reg, adr);
8293 }
8294 
8295 void Assembler::cvtsi2sdq(XMMRegister dst, Register src) {
8296   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
8297   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
8298   int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
8299   emit_int8(0x2A);
8300   emit_int8((unsigned char)(0xC0 | encode));
8301 }
8302 
8303 void Assembler::cvtsi2sdq(XMMRegister dst, Address src) {
8304   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
8305   InstructionMark im(this);
8306   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
8307   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
8308   simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
8309   emit_int8(0x2A);
8310   emit_operand(dst, src);
8311 }
8312 
8313 void Assembler::cvtsi2ssq(XMMRegister dst, Address src) {
8314   NOT_LP64(assert(VM_Version::supports_sse(), ""));
8315   InstructionMark im(this);
8316   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
8317   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
8318   simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
8319   emit_int8(0x2A);
8320   emit_operand(dst, src);
8321 }
8322 
8323 void Assembler::cvttsd2siq(Register dst, XMMRegister src) {
8324   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
8325   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
8326   int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
8327   emit_int8(0x2C);
8328   emit_int8((unsigned char)(0xC0 | encode));
8329 }
8330 
8331 void Assembler::cvttss2siq(Register dst, XMMRegister src) {
8332   NOT_LP64(assert(VM_Version::supports_sse(), ""));
8333   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
8334   int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
8335   emit_int8(0x2C);
8336   emit_int8((unsigned char)(0xC0 | encode));
8337 }
8338 
8339 void Assembler::decl(Register dst) {
8340   // Don't use it directly. Use MacroAssembler::decrementl() instead.
8341   // Use two-byte form (one-byte form is a REX prefix in 64-bit mode)
8342   int encode = prefix_and_encode(dst->encoding());
8343   emit_int8((unsigned char)0xFF);
8344   emit_int8((unsigned char)(0xC8 | encode));
8345 }
8346 
8347 void Assembler::decq(Register dst) {
8348   // Don't use it directly. Use MacroAssembler::decrementq() instead.
8349   // Use two-byte form (one-byte from is a REX prefix in 64-bit mode)
8350   int encode = prefixq_and_encode(dst->encoding());
8351   emit_int8((unsigned char)0xFF);
8352   emit_int8(0xC8 | encode);
8353 }
8354 
8355 void Assembler::decq(Address dst) {
8356   // Don't use it directly. Use MacroAssembler::decrementq() instead.
8357   InstructionMark im(this);
8358   prefixq(dst);
8359   emit_int8((unsigned char)0xFF);
8360   emit_operand(rcx, dst);
8361 }
8362 
8363 void Assembler::fxrstor(Address src) {
8364   prefixq(src);
8365   emit_int8(0x0F);
8366   emit_int8((unsigned char)0xAE);
8367   emit_operand(as_Register(1), src);
8368 }
8369 
8370 void Assembler::xrstor(Address src) {
8371   prefixq(src);
8372   emit_int8(0x0F);
8373   emit_int8((unsigned char)0xAE);
8374   emit_operand(as_Register(5), src);
8375 }
8376 
8377 void Assembler::fxsave(Address dst) {
8378   prefixq(dst);
8379   emit_int8(0x0F);
8380   emit_int8((unsigned char)0xAE);
8381   emit_operand(as_Register(0), dst);
8382 }
8383 
8384 void Assembler::xsave(Address dst) {
8385   prefixq(dst);
8386   emit_int8(0x0F);
8387   emit_int8((unsigned char)0xAE);
8388   emit_operand(as_Register(4), dst);
8389 }
8390 
8391 void Assembler::idivq(Register src) {
8392   int encode = prefixq_and_encode(src->encoding());
8393   emit_int8((unsigned char)0xF7);
8394   emit_int8((unsigned char)(0xF8 | encode));
8395 }
8396 
8397 void Assembler::imulq(Register dst, Register src) {
8398   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
8399   emit_int8(0x0F);
8400   emit_int8((unsigned char)0xAF);
8401   emit_int8((unsigned char)(0xC0 | encode));
8402 }
8403 
8404 void Assembler::imulq(Register dst, Register src, int value) {
8405   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
8406   if (is8bit(value)) {
8407     emit_int8(0x6B);
8408     emit_int8((unsigned char)(0xC0 | encode));
8409     emit_int8(value & 0xFF);
8410   } else {
8411     emit_int8(0x69);
8412     emit_int8((unsigned char)(0xC0 | encode));
8413     emit_int32(value);
8414   }
8415 }
8416 
8417 void Assembler::imulq(Register dst, Address src) {
8418   InstructionMark im(this);
8419   prefixq(src, dst);
8420   emit_int8(0x0F);
8421   emit_int8((unsigned char) 0xAF);
8422   emit_operand(dst, src);
8423 }
8424 
8425 void Assembler::incl(Register dst) {
8426   // Don't use it directly. Use MacroAssembler::incrementl() instead.
8427   // Use two-byte form (one-byte from is a REX prefix in 64-bit mode)
8428   int encode = prefix_and_encode(dst->encoding());
8429   emit_int8((unsigned char)0xFF);
8430   emit_int8((unsigned char)(0xC0 | encode));
8431 }
8432 
8433 void Assembler::incq(Register dst) {
8434   // Don't use it directly. Use MacroAssembler::incrementq() instead.
8435   // Use two-byte form (one-byte from is a REX prefix in 64-bit mode)
8436   int encode = prefixq_and_encode(dst->encoding());
8437   emit_int8((unsigned char)0xFF);
8438   emit_int8((unsigned char)(0xC0 | encode));
8439 }
8440 
8441 void Assembler::incq(Address dst) {
8442   // Don't use it directly. Use MacroAssembler::incrementq() instead.
8443   InstructionMark im(this);
8444   prefixq(dst);
8445   emit_int8((unsigned char)0xFF);
8446   emit_operand(rax, dst);
8447 }
8448 
8449 void Assembler::lea(Register dst, Address src) {
8450   leaq(dst, src);
8451 }
8452 
8453 void Assembler::leaq(Register dst, Address src) {
8454   InstructionMark im(this);
8455   prefixq(src, dst);
8456   emit_int8((unsigned char)0x8D);
8457   emit_operand(dst, src);
8458 }
8459 
8460 void Assembler::mov64(Register dst, int64_t imm64) {
8461   InstructionMark im(this);
8462   int encode = prefixq_and_encode(dst->encoding());
8463   emit_int8((unsigned char)(0xB8 | encode));
8464   emit_int64(imm64);
8465 }
8466 
8467 void Assembler::mov_literal64(Register dst, intptr_t imm64, RelocationHolder const& rspec) {
8468   InstructionMark im(this);
8469   int encode = prefixq_and_encode(dst->encoding());
8470   emit_int8(0xB8 | encode);
8471   emit_data64(imm64, rspec);
8472 }
8473 
8474 void Assembler::mov_narrow_oop(Register dst, int32_t imm32, RelocationHolder const& rspec) {
8475   InstructionMark im(this);
8476   int encode = prefix_and_encode(dst->encoding());
8477   emit_int8((unsigned char)(0xB8 | encode));
8478   emit_data((int)imm32, rspec, narrow_oop_operand);
8479 }
8480 
8481 void Assembler::mov_narrow_oop(Address dst, int32_t imm32,  RelocationHolder const& rspec) {
8482   InstructionMark im(this);
8483   prefix(dst);
8484   emit_int8((unsigned char)0xC7);
8485   emit_operand(rax, dst, 4);
8486   emit_data((int)imm32, rspec, narrow_oop_operand);
8487 }
8488 
8489 void Assembler::cmp_narrow_oop(Register src1, int32_t imm32, RelocationHolder const& rspec) {
8490   InstructionMark im(this);
8491   int encode = prefix_and_encode(src1->encoding());
8492   emit_int8((unsigned char)0x81);
8493   emit_int8((unsigned char)(0xF8 | encode));
8494   emit_data((int)imm32, rspec, narrow_oop_operand);
8495 }
8496 
8497 void Assembler::cmp_narrow_oop(Address src1, int32_t imm32, RelocationHolder const& rspec) {
8498   InstructionMark im(this);
8499   prefix(src1);
8500   emit_int8((unsigned char)0x81);
8501   emit_operand(rax, src1, 4);
8502   emit_data((int)imm32, rspec, narrow_oop_operand);
8503 }
8504 
8505 void Assembler::lzcntq(Register dst, Register src) {
8506   assert(VM_Version::supports_lzcnt(), "encoding is treated as BSR");
8507   emit_int8((unsigned char)0xF3);
8508   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
8509   emit_int8(0x0F);
8510   emit_int8((unsigned char)0xBD);
8511   emit_int8((unsigned char)(0xC0 | encode));
8512 }
8513 
8514 void Assembler::movdq(XMMRegister dst, Register src) {
8515   // table D-1 says MMX/SSE2
8516   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
8517   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
8518   int encode = simd_prefix_and_encode(dst, xnoreg, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
8519   emit_int8(0x6E);
8520   emit_int8((unsigned char)(0xC0 | encode));
8521 }
8522 
8523 void Assembler::movdq(Register dst, XMMRegister src) {
8524   // table D-1 says MMX/SSE2
8525   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
8526   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
8527   // swap src/dst to get correct prefix
8528   int encode = simd_prefix_and_encode(src, xnoreg, as_XMMRegister(dst->encoding()), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
8529   emit_int8(0x7E);
8530   emit_int8((unsigned char)(0xC0 | encode));
8531 }
8532 
8533 void Assembler::movq(Register dst, Register src) {
8534   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
8535   emit_int8((unsigned char)0x8B);
8536   emit_int8((unsigned char)(0xC0 | encode));
8537 }
8538 
8539 void Assembler::movq(Register dst, Address src) {
8540   InstructionMark im(this);
8541   prefixq(src, dst);
8542   emit_int8((unsigned char)0x8B);
8543   emit_operand(dst, src);
8544 }
8545 
8546 void Assembler::movq(Address dst, Register src) {
8547   InstructionMark im(this);
8548   prefixq(dst, src);
8549   emit_int8((unsigned char)0x89);
8550   emit_operand(src, dst);
8551 }
8552 
8553 void Assembler::movsbq(Register dst, Address src) {
8554   InstructionMark im(this);
8555   prefixq(src, dst);
8556   emit_int8(0x0F);
8557   emit_int8((unsigned char)0xBE);
8558   emit_operand(dst, src);
8559 }
8560 
8561 void Assembler::movsbq(Register dst, Register src) {
8562   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
8563   emit_int8(0x0F);
8564   emit_int8((unsigned char)0xBE);
8565   emit_int8((unsigned char)(0xC0 | encode));
8566 }
8567 
8568 void Assembler::movslq(Register dst, int32_t imm32) {
8569   // dbx shows movslq(rcx, 3) as movq     $0x0000000049000000,(%rbx)
8570   // and movslq(r8, 3); as movl     $0x0000000048000000,(%rbx)
8571   // as a result we shouldn't use until tested at runtime...
8572   ShouldNotReachHere();
8573   InstructionMark im(this);
8574   int encode = prefixq_and_encode(dst->encoding());
8575   emit_int8((unsigned char)(0xC7 | encode));
8576   emit_int32(imm32);
8577 }
8578 
8579 void Assembler::movslq(Address dst, int32_t imm32) {
8580   assert(is_simm32(imm32), "lost bits");
8581   InstructionMark im(this);
8582   prefixq(dst);
8583   emit_int8((unsigned char)0xC7);
8584   emit_operand(rax, dst, 4);
8585   emit_int32(imm32);
8586 }
8587 
8588 void Assembler::movslq(Register dst, Address src) {
8589   InstructionMark im(this);
8590   prefixq(src, dst);
8591   emit_int8(0x63);
8592   emit_operand(dst, src);
8593 }
8594 
8595 void Assembler::movslq(Register dst, Register src) {
8596   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
8597   emit_int8(0x63);
8598   emit_int8((unsigned char)(0xC0 | encode));
8599 }
8600 
8601 void Assembler::movswq(Register dst, Address src) {
8602   InstructionMark im(this);
8603   prefixq(src, dst);
8604   emit_int8(0x0F);
8605   emit_int8((unsigned char)0xBF);
8606   emit_operand(dst, src);
8607 }
8608 
8609 void Assembler::movswq(Register dst, Register src) {
8610   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
8611   emit_int8((unsigned char)0x0F);
8612   emit_int8((unsigned char)0xBF);
8613   emit_int8((unsigned char)(0xC0 | encode));
8614 }
8615 
8616 void Assembler::movzbq(Register dst, Address src) {
8617   InstructionMark im(this);
8618   prefixq(src, dst);
8619   emit_int8((unsigned char)0x0F);
8620   emit_int8((unsigned char)0xB6);
8621   emit_operand(dst, src);
8622 }
8623 
8624 void Assembler::movzbq(Register dst, Register src) {
8625   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
8626   emit_int8(0x0F);
8627   emit_int8((unsigned char)0xB6);
8628   emit_int8(0xC0 | encode);
8629 }
8630 
8631 void Assembler::movzwq(Register dst, Address src) {
8632   InstructionMark im(this);
8633   prefixq(src, dst);
8634   emit_int8((unsigned char)0x0F);
8635   emit_int8((unsigned char)0xB7);
8636   emit_operand(dst, src);
8637 }
8638 
8639 void Assembler::movzwq(Register dst, Register src) {
8640   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
8641   emit_int8((unsigned char)0x0F);
8642   emit_int8((unsigned char)0xB7);
8643   emit_int8((unsigned char)(0xC0 | encode));
8644 }
8645 
8646 void Assembler::mulq(Address src) {
8647   InstructionMark im(this);
8648   prefixq(src);
8649   emit_int8((unsigned char)0xF7);
8650   emit_operand(rsp, src);
8651 }
8652 
8653 void Assembler::mulq(Register src) {
8654   int encode = prefixq_and_encode(src->encoding());
8655   emit_int8((unsigned char)0xF7);
8656   emit_int8((unsigned char)(0xE0 | encode));
8657 }
8658 
8659 void Assembler::mulxq(Register dst1, Register dst2, Register src) {
8660   assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
8661   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
8662   int encode = vex_prefix_and_encode(dst1->encoding(), dst2->encoding(), src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_38, &attributes);
8663   emit_int8((unsigned char)0xF6);
8664   emit_int8((unsigned char)(0xC0 | encode));
8665 }
8666 
8667 void Assembler::negq(Register dst) {
8668   int encode = prefixq_and_encode(dst->encoding());
8669   emit_int8((unsigned char)0xF7);
8670   emit_int8((unsigned char)(0xD8 | encode));
8671 }
8672 
8673 void Assembler::notq(Register dst) {
8674   int encode = prefixq_and_encode(dst->encoding());
8675   emit_int8((unsigned char)0xF7);
8676   emit_int8((unsigned char)(0xD0 | encode));
8677 }
8678 
8679 void Assembler::orq(Address dst, int32_t imm32) {
8680   InstructionMark im(this);
8681   prefixq(dst);
8682   emit_int8((unsigned char)0x81);
8683   emit_operand(rcx, dst, 4);
8684   emit_int32(imm32);
8685 }
8686 
8687 void Assembler::orq(Register dst, int32_t imm32) {
8688   (void) prefixq_and_encode(dst->encoding());
8689   emit_arith(0x81, 0xC8, dst, imm32);
8690 }
8691 
8692 void Assembler::orq(Register dst, Address src) {
8693   InstructionMark im(this);
8694   prefixq(src, dst);
8695   emit_int8(0x0B);
8696   emit_operand(dst, src);
8697 }
8698 
8699 void Assembler::orq(Register dst, Register src) {
8700   (void) prefixq_and_encode(dst->encoding(), src->encoding());
8701   emit_arith(0x0B, 0xC0, dst, src);
8702 }
8703 
8704 void Assembler::popa() { // 64bit
8705   movq(r15, Address(rsp, 0));
8706   movq(r14, Address(rsp, wordSize));
8707   movq(r13, Address(rsp, 2 * wordSize));
8708   movq(r12, Address(rsp, 3 * wordSize));
8709   movq(r11, Address(rsp, 4 * wordSize));
8710   movq(r10, Address(rsp, 5 * wordSize));
8711   movq(r9,  Address(rsp, 6 * wordSize));
8712   movq(r8,  Address(rsp, 7 * wordSize));
8713   movq(rdi, Address(rsp, 8 * wordSize));
8714   movq(rsi, Address(rsp, 9 * wordSize));
8715   movq(rbp, Address(rsp, 10 * wordSize));
8716   // skip rsp
8717   movq(rbx, Address(rsp, 12 * wordSize));
8718   movq(rdx, Address(rsp, 13 * wordSize));
8719   movq(rcx, Address(rsp, 14 * wordSize));
8720   movq(rax, Address(rsp, 15 * wordSize));
8721 
8722   addq(rsp, 16 * wordSize);
8723 }
8724 
8725 void Assembler::popcntq(Register dst, Address src) {
8726   assert(VM_Version::supports_popcnt(), "must support");
8727   InstructionMark im(this);
8728   emit_int8((unsigned char)0xF3);
8729   prefixq(src, dst);
8730   emit_int8((unsigned char)0x0F);
8731   emit_int8((unsigned char)0xB8);
8732   emit_operand(dst, src);
8733 }
8734 
8735 void Assembler::popcntq(Register dst, Register src) {
8736   assert(VM_Version::supports_popcnt(), "must support");
8737   emit_int8((unsigned char)0xF3);
8738   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
8739   emit_int8((unsigned char)0x0F);
8740   emit_int8((unsigned char)0xB8);
8741   emit_int8((unsigned char)(0xC0 | encode));
8742 }
8743 
8744 void Assembler::popq(Address dst) {
8745   InstructionMark im(this);
8746   prefixq(dst);
8747   emit_int8((unsigned char)0x8F);
8748   emit_operand(rax, dst);
8749 }
8750 
8751 void Assembler::pusha() { // 64bit
8752   // we have to store original rsp.  ABI says that 128 bytes
8753   // below rsp are local scratch.
8754   movq(Address(rsp, -5 * wordSize), rsp);
8755 
8756   subq(rsp, 16 * wordSize);
8757 
8758   movq(Address(rsp, 15 * wordSize), rax);
8759   movq(Address(rsp, 14 * wordSize), rcx);
8760   movq(Address(rsp, 13 * wordSize), rdx);
8761   movq(Address(rsp, 12 * wordSize), rbx);
8762   // skip rsp
8763   movq(Address(rsp, 10 * wordSize), rbp);
8764   movq(Address(rsp, 9 * wordSize), rsi);
8765   movq(Address(rsp, 8 * wordSize), rdi);
8766   movq(Address(rsp, 7 * wordSize), r8);
8767   movq(Address(rsp, 6 * wordSize), r9);
8768   movq(Address(rsp, 5 * wordSize), r10);
8769   movq(Address(rsp, 4 * wordSize), r11);
8770   movq(Address(rsp, 3 * wordSize), r12);
8771   movq(Address(rsp, 2 * wordSize), r13);
8772   movq(Address(rsp, wordSize), r14);
8773   movq(Address(rsp, 0), r15);
8774 }
8775 
8776 void Assembler::pushq(Address src) {
8777   InstructionMark im(this);
8778   prefixq(src);
8779   emit_int8((unsigned char)0xFF);
8780   emit_operand(rsi, src);
8781 }
8782 
8783 void Assembler::rclq(Register dst, int imm8) {
8784   assert(isShiftCount(imm8 >> 1), "illegal shift count");
8785   int encode = prefixq_and_encode(dst->encoding());
8786   if (imm8 == 1) {
8787     emit_int8((unsigned char)0xD1);
8788     emit_int8((unsigned char)(0xD0 | encode));
8789   } else {
8790     emit_int8((unsigned char)0xC1);
8791     emit_int8((unsigned char)(0xD0 | encode));
8792     emit_int8(imm8);
8793   }
8794 }
8795 
8796 void Assembler::rcrq(Register dst, int imm8) {
8797   assert(isShiftCount(imm8 >> 1), "illegal shift count");
8798   int encode = prefixq_and_encode(dst->encoding());
8799   if (imm8 == 1) {
8800     emit_int8((unsigned char)0xD1);
8801     emit_int8((unsigned char)(0xD8 | encode));
8802   } else {
8803     emit_int8((unsigned char)0xC1);
8804     emit_int8((unsigned char)(0xD8 | encode));
8805     emit_int8(imm8);
8806   }
8807 }
8808 
8809 void Assembler::rorq(Register dst, int imm8) {
8810   assert(isShiftCount(imm8 >> 1), "illegal shift count");
8811   int encode = prefixq_and_encode(dst->encoding());
8812   if (imm8 == 1) {
8813     emit_int8((unsigned char)0xD1);
8814     emit_int8((unsigned char)(0xC8 | encode));
8815   } else {
8816     emit_int8((unsigned char)0xC1);
8817     emit_int8((unsigned char)(0xc8 | encode));
8818     emit_int8(imm8);
8819   }
8820 }
8821 
8822 void Assembler::rorxq(Register dst, Register src, int imm8) {
8823   assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
8824   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
8825   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_3A, &attributes);
8826   emit_int8((unsigned char)0xF0);
8827   emit_int8((unsigned char)(0xC0 | encode));
8828   emit_int8(imm8);
8829 }
8830 
8831 void Assembler::rorxd(Register dst, Register src, int imm8) {
8832   assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
8833   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
8834   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_3A, &attributes);
8835   emit_int8((unsigned char)0xF0);
8836   emit_int8((unsigned char)(0xC0 | encode));
8837   emit_int8(imm8);
8838 }
8839 
8840 void Assembler::sarq(Register dst, int imm8) {
8841   assert(isShiftCount(imm8 >> 1), "illegal shift count");
8842   int encode = prefixq_and_encode(dst->encoding());
8843   if (imm8 == 1) {
8844     emit_int8((unsigned char)0xD1);
8845     emit_int8((unsigned char)(0xF8 | encode));
8846   } else {
8847     emit_int8((unsigned char)0xC1);
8848     emit_int8((unsigned char)(0xF8 | encode));
8849     emit_int8(imm8);
8850   }
8851 }
8852 
8853 void Assembler::sarq(Register dst) {
8854   int encode = prefixq_and_encode(dst->encoding());
8855   emit_int8((unsigned char)0xD3);
8856   emit_int8((unsigned char)(0xF8 | encode));
8857 }
8858 
8859 void Assembler::sbbq(Address dst, int32_t imm32) {
8860   InstructionMark im(this);
8861   prefixq(dst);
8862   emit_arith_operand(0x81, rbx, dst, imm32);
8863 }
8864 
8865 void Assembler::sbbq(Register dst, int32_t imm32) {
8866   (void) prefixq_and_encode(dst->encoding());
8867   emit_arith(0x81, 0xD8, dst, imm32);
8868 }
8869 
8870 void Assembler::sbbq(Register dst, Address src) {
8871   InstructionMark im(this);
8872   prefixq(src, dst);
8873   emit_int8(0x1B);
8874   emit_operand(dst, src);
8875 }
8876 
8877 void Assembler::sbbq(Register dst, Register src) {
8878   (void) prefixq_and_encode(dst->encoding(), src->encoding());
8879   emit_arith(0x1B, 0xC0, dst, src);
8880 }
8881 
8882 void Assembler::shlq(Register dst, int imm8) {
8883   assert(isShiftCount(imm8 >> 1), "illegal shift count");
8884   int encode = prefixq_and_encode(dst->encoding());
8885   if (imm8 == 1) {
8886     emit_int8((unsigned char)0xD1);
8887     emit_int8((unsigned char)(0xE0 | encode));
8888   } else {
8889     emit_int8((unsigned char)0xC1);
8890     emit_int8((unsigned char)(0xE0 | encode));
8891     emit_int8(imm8);
8892   }
8893 }
8894 
8895 void Assembler::shlq(Register dst) {
8896   int encode = prefixq_and_encode(dst->encoding());
8897   emit_int8((unsigned char)0xD3);
8898   emit_int8((unsigned char)(0xE0 | encode));
8899 }
8900 
8901 void Assembler::shrq(Register dst, int imm8) {
8902   assert(isShiftCount(imm8 >> 1), "illegal shift count");
8903   int encode = prefixq_and_encode(dst->encoding());
8904   emit_int8((unsigned char)0xC1);
8905   emit_int8((unsigned char)(0xE8 | encode));
8906   emit_int8(imm8);
8907 }
8908 
8909 void Assembler::shrq(Register dst) {
8910   int encode = prefixq_and_encode(dst->encoding());
8911   emit_int8((unsigned char)0xD3);
8912   emit_int8(0xE8 | encode);
8913 }
8914 
8915 void Assembler::subq(Address dst, int32_t imm32) {
8916   InstructionMark im(this);
8917   prefixq(dst);
8918   emit_arith_operand(0x81, rbp, dst, imm32);
8919 }
8920 
8921 void Assembler::subq(Address dst, Register src) {
8922   InstructionMark im(this);
8923   prefixq(dst, src);
8924   emit_int8(0x29);
8925   emit_operand(src, dst);
8926 }
8927 
8928 void Assembler::subq(Register dst, int32_t imm32) {
8929   (void) prefixq_and_encode(dst->encoding());
8930   emit_arith(0x81, 0xE8, dst, imm32);
8931 }
8932 
8933 // Force generation of a 4 byte immediate value even if it fits into 8bit
8934 void Assembler::subq_imm32(Register dst, int32_t imm32) {
8935   (void) prefixq_and_encode(dst->encoding());
8936   emit_arith_imm32(0x81, 0xE8, dst, imm32);
8937 }
8938 
8939 void Assembler::subq(Register dst, Address src) {
8940   InstructionMark im(this);
8941   prefixq(src, dst);
8942   emit_int8(0x2B);
8943   emit_operand(dst, src);
8944 }
8945 
8946 void Assembler::subq(Register dst, Register src) {
8947   (void) prefixq_and_encode(dst->encoding(), src->encoding());
8948   emit_arith(0x2B, 0xC0, dst, src);
8949 }
8950 
8951 void Assembler::testq(Register dst, int32_t imm32) {
8952   // not using emit_arith because test
8953   // doesn't support sign-extension of
8954   // 8bit operands
8955   int encode = dst->encoding();
8956   if (encode == 0) {
8957     prefix(REX_W);
8958     emit_int8((unsigned char)0xA9);
8959   } else {
8960     encode = prefixq_and_encode(encode);
8961     emit_int8((unsigned char)0xF7);
8962     emit_int8((unsigned char)(0xC0 | encode));
8963   }
8964   emit_int32(imm32);
8965 }
8966 
8967 void Assembler::testq(Register dst, Register src) {
8968   (void) prefixq_and_encode(dst->encoding(), src->encoding());
8969   emit_arith(0x85, 0xC0, dst, src);
8970 }
8971 
8972 void Assembler::xaddq(Address dst, Register src) {
8973   InstructionMark im(this);
8974   prefixq(dst, src);
8975   emit_int8(0x0F);
8976   emit_int8((unsigned char)0xC1);
8977   emit_operand(src, dst);
8978 }
8979 
8980 void Assembler::xchgq(Register dst, Address src) {
8981   InstructionMark im(this);
8982   prefixq(src, dst);
8983   emit_int8((unsigned char)0x87);
8984   emit_operand(dst, src);
8985 }
8986 
8987 void Assembler::xchgq(Register dst, Register src) {
8988   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
8989   emit_int8((unsigned char)0x87);
8990   emit_int8((unsigned char)(0xc0 | encode));
8991 }
8992 
8993 void Assembler::xorq(Register dst, Register src) {
8994   (void) prefixq_and_encode(dst->encoding(), src->encoding());
8995   emit_arith(0x33, 0xC0, dst, src);
8996 }
8997 
8998 void Assembler::xorq(Register dst, Address src) {
8999   InstructionMark im(this);
9000   prefixq(src, dst);
9001   emit_int8(0x33);
9002   emit_operand(dst, src);
9003 }
9004 
9005 #endif // !LP64