1 /*
   2  * Copyright (c) 1997, 2018, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #include "precompiled.hpp"
  26 #include "asm/assembler.hpp"
  27 #include "asm/assembler.inline.hpp"
  28 #include "gc/shared/cardTableModRefBS.hpp"
  29 #include "gc/shared/collectedHeap.inline.hpp"
  30 #include "interpreter/interpreter.hpp"
  31 #include "memory/resourceArea.hpp"
  32 #include "prims/methodHandles.hpp"
  33 #include "runtime/biasedLocking.hpp"
  34 #include "runtime/interfaceSupport.hpp"
  35 #include "runtime/objectMonitor.hpp"
  36 #include "runtime/os.hpp"
  37 #include "runtime/sharedRuntime.hpp"
  38 #include "runtime/stubRoutines.hpp"
  39 #include "utilities/macros.hpp"
  40 #if INCLUDE_ALL_GCS
  41 #include "gc/g1/g1BarrierSet.hpp"
  42 #include "gc/g1/g1CollectedHeap.inline.hpp"
  43 #include "gc/g1/heapRegion.hpp"
  44 #endif // INCLUDE_ALL_GCS
  45 
  46 #ifdef PRODUCT
  47 #define BLOCK_COMMENT(str) /* nothing */
  48 #define STOP(error) stop(error)
  49 #else
  50 #define BLOCK_COMMENT(str) block_comment(str)
  51 #define STOP(error) block_comment(error); stop(error)
  52 #endif
  53 
  54 #define BIND(label) bind(label); BLOCK_COMMENT(#label ":")
  55 // Implementation of AddressLiteral
  56 
  57 // A 2-D table for managing compressed displacement(disp8) on EVEX enabled platforms.
  58 unsigned char tuple_table[Assembler::EVEX_ETUP + 1][Assembler::AVX_512bit + 1] = {
  59   // -----------------Table 4.5 -------------------- //
  60   16, 32, 64,  // EVEX_FV(0)
  61   4,  4,  4,   // EVEX_FV(1) - with Evex.b
  62   16, 32, 64,  // EVEX_FV(2) - with Evex.w
  63   8,  8,  8,   // EVEX_FV(3) - with Evex.w and Evex.b
  64   8,  16, 32,  // EVEX_HV(0)
  65   4,  4,  4,   // EVEX_HV(1) - with Evex.b
  66   // -----------------Table 4.6 -------------------- //
  67   16, 32, 64,  // EVEX_FVM(0)
  68   1,  1,  1,   // EVEX_T1S(0)
  69   2,  2,  2,   // EVEX_T1S(1)
  70   4,  4,  4,   // EVEX_T1S(2)
  71   8,  8,  8,   // EVEX_T1S(3)
  72   4,  4,  4,   // EVEX_T1F(0)
  73   8,  8,  8,   // EVEX_T1F(1)
  74   8,  8,  8,   // EVEX_T2(0)
  75   0,  16, 16,  // EVEX_T2(1)
  76   0,  16, 16,  // EVEX_T4(0)
  77   0,  0,  32,  // EVEX_T4(1)
  78   0,  0,  32,  // EVEX_T8(0)
  79   8,  16, 32,  // EVEX_HVM(0)
  80   4,  8,  16,  // EVEX_QVM(0)
  81   2,  4,  8,   // EVEX_OVM(0)
  82   16, 16, 16,  // EVEX_M128(0)
  83   8,  32, 64,  // EVEX_DUP(0)
  84   0,  0,  0    // EVEX_NTUP
  85 };
  86 
  87 AddressLiteral::AddressLiteral(address target, relocInfo::relocType rtype) {
  88   _is_lval = false;
  89   _target = target;
  90   switch (rtype) {
  91   case relocInfo::oop_type:
  92   case relocInfo::metadata_type:
  93     // Oops are a special case. Normally they would be their own section
  94     // but in cases like icBuffer they are literals in the code stream that
  95     // we don't have a section for. We use none so that we get a literal address
  96     // which is always patchable.
  97     break;
  98   case relocInfo::external_word_type:
  99     _rspec = external_word_Relocation::spec(target);
 100     break;
 101   case relocInfo::internal_word_type:
 102     _rspec = internal_word_Relocation::spec(target);
 103     break;
 104   case relocInfo::opt_virtual_call_type:
 105     _rspec = opt_virtual_call_Relocation::spec();
 106     break;
 107   case relocInfo::static_call_type:
 108     _rspec = static_call_Relocation::spec();
 109     break;
 110   case relocInfo::runtime_call_type:
 111     _rspec = runtime_call_Relocation::spec();
 112     break;
 113   case relocInfo::poll_type:
 114   case relocInfo::poll_return_type:
 115     _rspec = Relocation::spec_simple(rtype);
 116     break;
 117   case relocInfo::none:
 118     break;
 119   default:
 120     ShouldNotReachHere();
 121     break;
 122   }
 123 }
 124 
 125 // Implementation of Address
 126 
 127 #ifdef _LP64
 128 
 129 Address Address::make_array(ArrayAddress adr) {
 130   // Not implementable on 64bit machines
 131   // Should have been handled higher up the call chain.
 132   ShouldNotReachHere();
 133   return Address();
 134 }
 135 
 136 // exceedingly dangerous constructor
 137 Address::Address(int disp, address loc, relocInfo::relocType rtype) {
 138   _base  = noreg;
 139   _index = noreg;
 140   _scale = no_scale;
 141   _disp  = disp;
 142   switch (rtype) {
 143     case relocInfo::external_word_type:
 144       _rspec = external_word_Relocation::spec(loc);
 145       break;
 146     case relocInfo::internal_word_type:
 147       _rspec = internal_word_Relocation::spec(loc);
 148       break;
 149     case relocInfo::runtime_call_type:
 150       // HMM
 151       _rspec = runtime_call_Relocation::spec();
 152       break;
 153     case relocInfo::poll_type:
 154     case relocInfo::poll_return_type:
 155       _rspec = Relocation::spec_simple(rtype);
 156       break;
 157     case relocInfo::none:
 158       break;
 159     default:
 160       ShouldNotReachHere();
 161   }
 162 }
 163 #else // LP64
 164 
 165 Address Address::make_array(ArrayAddress adr) {
 166   AddressLiteral base = adr.base();
 167   Address index = adr.index();
 168   assert(index._disp == 0, "must not have disp"); // maybe it can?
 169   Address array(index._base, index._index, index._scale, (intptr_t) base.target());
 170   array._rspec = base._rspec;
 171   return array;
 172 }
 173 
 174 // exceedingly dangerous constructor
 175 Address::Address(address loc, RelocationHolder spec) {
 176   _base  = noreg;
 177   _index = noreg;
 178   _scale = no_scale;
 179   _disp  = (intptr_t) loc;
 180   _rspec = spec;
 181 }
 182 
 183 #endif // _LP64
 184 
 185 
 186 
 187 // Convert the raw encoding form into the form expected by the constructor for
 188 // Address.  An index of 4 (rsp) corresponds to having no index, so convert
 189 // that to noreg for the Address constructor.
 190 Address Address::make_raw(int base, int index, int scale, int disp, relocInfo::relocType disp_reloc) {
 191   RelocationHolder rspec;
 192   if (disp_reloc != relocInfo::none) {
 193     rspec = Relocation::spec_simple(disp_reloc);
 194   }
 195   bool valid_index = index != rsp->encoding();
 196   if (valid_index) {
 197     Address madr(as_Register(base), as_Register(index), (Address::ScaleFactor)scale, in_ByteSize(disp));
 198     madr._rspec = rspec;
 199     return madr;
 200   } else {
 201     Address madr(as_Register(base), noreg, Address::no_scale, in_ByteSize(disp));
 202     madr._rspec = rspec;
 203     return madr;
 204   }
 205 }
 206 
 207 // Implementation of Assembler
 208 
 209 int AbstractAssembler::code_fill_byte() {
 210   return (u_char)'\xF4'; // hlt
 211 }
 212 
 213 // make this go away someday
 214 void Assembler::emit_data(jint data, relocInfo::relocType rtype, int format) {
 215   if (rtype == relocInfo::none)
 216     emit_int32(data);
 217   else
 218     emit_data(data, Relocation::spec_simple(rtype), format);
 219 }
 220 
 221 void Assembler::emit_data(jint data, RelocationHolder const& rspec, int format) {
 222   assert(imm_operand == 0, "default format must be immediate in this file");
 223   assert(inst_mark() != NULL, "must be inside InstructionMark");
 224   if (rspec.type() !=  relocInfo::none) {
 225     #ifdef ASSERT
 226       check_relocation(rspec, format);
 227     #endif
 228     // Do not use AbstractAssembler::relocate, which is not intended for
 229     // embedded words.  Instead, relocate to the enclosing instruction.
 230 
 231     // hack. call32 is too wide for mask so use disp32
 232     if (format == call32_operand)
 233       code_section()->relocate(inst_mark(), rspec, disp32_operand);
 234     else
 235       code_section()->relocate(inst_mark(), rspec, format);
 236   }
 237   emit_int32(data);
 238 }
 239 
 240 static int encode(Register r) {
 241   int enc = r->encoding();
 242   if (enc >= 8) {
 243     enc -= 8;
 244   }
 245   return enc;
 246 }
 247 
 248 void Assembler::emit_arith_b(int op1, int op2, Register dst, int imm8) {
 249   assert(dst->has_byte_register(), "must have byte register");
 250   assert(isByte(op1) && isByte(op2), "wrong opcode");
 251   assert(isByte(imm8), "not a byte");
 252   assert((op1 & 0x01) == 0, "should be 8bit operation");
 253   emit_int8(op1);
 254   emit_int8(op2 | encode(dst));
 255   emit_int8(imm8);
 256 }
 257 
 258 
 259 void Assembler::emit_arith(int op1, int op2, Register dst, int32_t imm32) {
 260   assert(isByte(op1) && isByte(op2), "wrong opcode");
 261   assert((op1 & 0x01) == 1, "should be 32bit operation");
 262   assert((op1 & 0x02) == 0, "sign-extension bit should not be set");
 263   if (is8bit(imm32)) {
 264     emit_int8(op1 | 0x02); // set sign bit
 265     emit_int8(op2 | encode(dst));
 266     emit_int8(imm32 & 0xFF);
 267   } else {
 268     emit_int8(op1);
 269     emit_int8(op2 | encode(dst));
 270     emit_int32(imm32);
 271   }
 272 }
 273 
 274 // Force generation of a 4 byte immediate value even if it fits into 8bit
 275 void Assembler::emit_arith_imm32(int op1, int op2, Register dst, int32_t imm32) {
 276   assert(isByte(op1) && isByte(op2), "wrong opcode");
 277   assert((op1 & 0x01) == 1, "should be 32bit operation");
 278   assert((op1 & 0x02) == 0, "sign-extension bit should not be set");
 279   emit_int8(op1);
 280   emit_int8(op2 | encode(dst));
 281   emit_int32(imm32);
 282 }
 283 
 284 // immediate-to-memory forms
 285 void Assembler::emit_arith_operand(int op1, Register rm, Address adr, int32_t imm32) {
 286   assert((op1 & 0x01) == 1, "should be 32bit operation");
 287   assert((op1 & 0x02) == 0, "sign-extension bit should not be set");
 288   if (is8bit(imm32)) {
 289     emit_int8(op1 | 0x02); // set sign bit
 290     emit_operand(rm, adr, 1);
 291     emit_int8(imm32 & 0xFF);
 292   } else {
 293     emit_int8(op1);
 294     emit_operand(rm, adr, 4);
 295     emit_int32(imm32);
 296   }
 297 }
 298 
 299 
 300 void Assembler::emit_arith(int op1, int op2, Register dst, Register src) {
 301   assert(isByte(op1) && isByte(op2), "wrong opcode");
 302   emit_int8(op1);
 303   emit_int8(op2 | encode(dst) << 3 | encode(src));
 304 }
 305 
 306 
 307 bool Assembler::query_compressed_disp_byte(int disp, bool is_evex_inst, int vector_len,
 308                                            int cur_tuple_type, int in_size_in_bits, int cur_encoding) {
 309   int mod_idx = 0;
 310   // We will test if the displacement fits the compressed format and if so
 311   // apply the compression to the displacment iff the result is8bit.
 312   if (VM_Version::supports_evex() && is_evex_inst) {
 313     switch (cur_tuple_type) {
 314     case EVEX_FV:
 315       if ((cur_encoding & VEX_W) == VEX_W) {
 316         mod_idx = ((cur_encoding & EVEX_Rb) == EVEX_Rb) ? 3 : 2;
 317       } else {
 318         mod_idx = ((cur_encoding & EVEX_Rb) == EVEX_Rb) ? 1 : 0;
 319       }
 320       break;
 321 
 322     case EVEX_HV:
 323       mod_idx = ((cur_encoding & EVEX_Rb) == EVEX_Rb) ? 1 : 0;
 324       break;
 325 
 326     case EVEX_FVM:
 327       break;
 328 
 329     case EVEX_T1S:
 330       switch (in_size_in_bits) {
 331       case EVEX_8bit:
 332         break;
 333 
 334       case EVEX_16bit:
 335         mod_idx = 1;
 336         break;
 337 
 338       case EVEX_32bit:
 339         mod_idx = 2;
 340         break;
 341 
 342       case EVEX_64bit:
 343         mod_idx = 3;
 344         break;
 345       }
 346       break;
 347 
 348     case EVEX_T1F:
 349     case EVEX_T2:
 350     case EVEX_T4:
 351       mod_idx = (in_size_in_bits == EVEX_64bit) ? 1 : 0;
 352       break;
 353 
 354     case EVEX_T8:
 355       break;
 356 
 357     case EVEX_HVM:
 358       break;
 359 
 360     case EVEX_QVM:
 361       break;
 362 
 363     case EVEX_OVM:
 364       break;
 365 
 366     case EVEX_M128:
 367       break;
 368 
 369     case EVEX_DUP:
 370       break;
 371 
 372     default:
 373       assert(0, "no valid evex tuple_table entry");
 374       break;
 375     }
 376 
 377     if (vector_len >= AVX_128bit && vector_len <= AVX_512bit) {
 378       int disp_factor = tuple_table[cur_tuple_type + mod_idx][vector_len];
 379       if ((disp % disp_factor) == 0) {
 380         int new_disp = disp / disp_factor;
 381         if ((-0x80 <= new_disp && new_disp < 0x80)) {
 382           disp = new_disp;
 383         }
 384       } else {
 385         return false;
 386       }
 387     }
 388   }
 389   return (-0x80 <= disp && disp < 0x80);
 390 }
 391 
 392 
 393 bool Assembler::emit_compressed_disp_byte(int &disp) {
 394   int mod_idx = 0;
 395   // We will test if the displacement fits the compressed format and if so
 396   // apply the compression to the displacment iff the result is8bit.
 397   if (VM_Version::supports_evex() && _attributes && _attributes->is_evex_instruction()) {
 398     int evex_encoding = _attributes->get_evex_encoding();
 399     int tuple_type = _attributes->get_tuple_type();
 400     switch (tuple_type) {
 401     case EVEX_FV:
 402       if ((evex_encoding & VEX_W) == VEX_W) {
 403         mod_idx = ((evex_encoding & EVEX_Rb) == EVEX_Rb) ? 3 : 2;
 404       } else {
 405         mod_idx = ((evex_encoding & EVEX_Rb) == EVEX_Rb) ? 1 : 0;
 406       }
 407       break;
 408 
 409     case EVEX_HV:
 410       mod_idx = ((evex_encoding & EVEX_Rb) == EVEX_Rb) ? 1 : 0;
 411       break;
 412 
 413     case EVEX_FVM:
 414       break;
 415 
 416     case EVEX_T1S:
 417       switch (_attributes->get_input_size()) {
 418       case EVEX_8bit:
 419         break;
 420 
 421       case EVEX_16bit:
 422         mod_idx = 1;
 423         break;
 424 
 425       case EVEX_32bit:
 426         mod_idx = 2;
 427         break;
 428 
 429       case EVEX_64bit:
 430         mod_idx = 3;
 431         break;
 432       }
 433       break;
 434 
 435     case EVEX_T1F:
 436     case EVEX_T2:
 437     case EVEX_T4:
 438       mod_idx = (_attributes->get_input_size() == EVEX_64bit) ? 1 : 0;
 439       break;
 440 
 441     case EVEX_T8:
 442       break;
 443 
 444     case EVEX_HVM:
 445       break;
 446 
 447     case EVEX_QVM:
 448       break;
 449 
 450     case EVEX_OVM:
 451       break;
 452 
 453     case EVEX_M128:
 454       break;
 455 
 456     case EVEX_DUP:
 457       break;
 458 
 459     default:
 460       assert(0, "no valid evex tuple_table entry");
 461       break;
 462     }
 463 
 464     int vector_len = _attributes->get_vector_len();
 465     if (vector_len >= AVX_128bit && vector_len <= AVX_512bit) {
 466       int disp_factor = tuple_table[tuple_type + mod_idx][vector_len];
 467       if ((disp % disp_factor) == 0) {
 468         int new_disp = disp / disp_factor;
 469         if (is8bit(new_disp)) {
 470           disp = new_disp;
 471         }
 472       } else {
 473         return false;
 474       }
 475     }
 476   }
 477   return is8bit(disp);
 478 }
 479 
 480 
 481 void Assembler::emit_operand(Register reg, Register base, Register index,
 482                              Address::ScaleFactor scale, int disp,
 483                              RelocationHolder const& rspec,
 484                              int rip_relative_correction) {
 485   relocInfo::relocType rtype = (relocInfo::relocType) rspec.type();
 486 
 487   // Encode the registers as needed in the fields they are used in
 488 
 489   int regenc = encode(reg) << 3;
 490   int indexenc = index->is_valid() ? encode(index) << 3 : 0;
 491   int baseenc = base->is_valid() ? encode(base) : 0;
 492 
 493   if (base->is_valid()) {
 494     if (index->is_valid()) {
 495       assert(scale != Address::no_scale, "inconsistent address");
 496       // [base + index*scale + disp]
 497       if (disp == 0 && rtype == relocInfo::none  &&
 498           base != rbp LP64_ONLY(&& base != r13)) {
 499         // [base + index*scale]
 500         // [00 reg 100][ss index base]
 501         assert(index != rsp, "illegal addressing mode");
 502         emit_int8(0x04 | regenc);
 503         emit_int8(scale << 6 | indexenc | baseenc);
 504       } else if (emit_compressed_disp_byte(disp) && rtype == relocInfo::none) {
 505         // [base + index*scale + imm8]
 506         // [01 reg 100][ss index base] imm8
 507         assert(index != rsp, "illegal addressing mode");
 508         emit_int8(0x44 | regenc);
 509         emit_int8(scale << 6 | indexenc | baseenc);
 510         emit_int8(disp & 0xFF);
 511       } else {
 512         // [base + index*scale + disp32]
 513         // [10 reg 100][ss index base] disp32
 514         assert(index != rsp, "illegal addressing mode");
 515         emit_int8(0x84 | regenc);
 516         emit_int8(scale << 6 | indexenc | baseenc);
 517         emit_data(disp, rspec, disp32_operand);
 518       }
 519     } else if (base == rsp LP64_ONLY(|| base == r12)) {
 520       // [rsp + disp]
 521       if (disp == 0 && rtype == relocInfo::none) {
 522         // [rsp]
 523         // [00 reg 100][00 100 100]
 524         emit_int8(0x04 | regenc);
 525         emit_int8(0x24);
 526       } else if (emit_compressed_disp_byte(disp) && rtype == relocInfo::none) {
 527         // [rsp + imm8]
 528         // [01 reg 100][00 100 100] disp8
 529         emit_int8(0x44 | regenc);
 530         emit_int8(0x24);
 531         emit_int8(disp & 0xFF);
 532       } else {
 533         // [rsp + imm32]
 534         // [10 reg 100][00 100 100] disp32
 535         emit_int8(0x84 | regenc);
 536         emit_int8(0x24);
 537         emit_data(disp, rspec, disp32_operand);
 538       }
 539     } else {
 540       // [base + disp]
 541       assert(base != rsp LP64_ONLY(&& base != r12), "illegal addressing mode");
 542       if (disp == 0 && rtype == relocInfo::none &&
 543           base != rbp LP64_ONLY(&& base != r13)) {
 544         // [base]
 545         // [00 reg base]
 546         emit_int8(0x00 | regenc | baseenc);
 547       } else if (emit_compressed_disp_byte(disp) && rtype == relocInfo::none) {
 548         // [base + disp8]
 549         // [01 reg base] disp8
 550         emit_int8(0x40 | regenc | baseenc);
 551         emit_int8(disp & 0xFF);
 552       } else {
 553         // [base + disp32]
 554         // [10 reg base] disp32
 555         emit_int8(0x80 | regenc | baseenc);
 556         emit_data(disp, rspec, disp32_operand);
 557       }
 558     }
 559   } else {
 560     if (index->is_valid()) {
 561       assert(scale != Address::no_scale, "inconsistent address");
 562       // [index*scale + disp]
 563       // [00 reg 100][ss index 101] disp32
 564       assert(index != rsp, "illegal addressing mode");
 565       emit_int8(0x04 | regenc);
 566       emit_int8(scale << 6 | indexenc | 0x05);
 567       emit_data(disp, rspec, disp32_operand);
 568     } else if (rtype != relocInfo::none ) {
 569       // [disp] (64bit) RIP-RELATIVE (32bit) abs
 570       // [00 000 101] disp32
 571 
 572       emit_int8(0x05 | regenc);
 573       // Note that the RIP-rel. correction applies to the generated
 574       // disp field, but _not_ to the target address in the rspec.
 575 
 576       // disp was created by converting the target address minus the pc
 577       // at the start of the instruction. That needs more correction here.
 578       // intptr_t disp = target - next_ip;
 579       assert(inst_mark() != NULL, "must be inside InstructionMark");
 580       address next_ip = pc() + sizeof(int32_t) + rip_relative_correction;
 581       int64_t adjusted = disp;
 582       // Do rip-rel adjustment for 64bit
 583       LP64_ONLY(adjusted -=  (next_ip - inst_mark()));
 584       assert(is_simm32(adjusted),
 585              "must be 32bit offset (RIP relative address)");
 586       emit_data((int32_t) adjusted, rspec, disp32_operand);
 587 
 588     } else {
 589       // 32bit never did this, did everything as the rip-rel/disp code above
 590       // [disp] ABSOLUTE
 591       // [00 reg 100][00 100 101] disp32
 592       emit_int8(0x04 | regenc);
 593       emit_int8(0x25);
 594       emit_data(disp, rspec, disp32_operand);
 595     }
 596   }
 597 }
 598 
 599 void Assembler::emit_operand(XMMRegister reg, Register base, Register index,
 600                              Address::ScaleFactor scale, int disp,
 601                              RelocationHolder const& rspec) {
 602   if (UseAVX > 2) {
 603     int xreg_enc = reg->encoding();
 604     if (xreg_enc > 15) {
 605       XMMRegister new_reg = as_XMMRegister(xreg_enc & 0xf);
 606       emit_operand((Register)new_reg, base, index, scale, disp, rspec);
 607       return;
 608     }
 609   }
 610   emit_operand((Register)reg, base, index, scale, disp, rspec);
 611 }
 612 
 613 // Secret local extension to Assembler::WhichOperand:
 614 #define end_pc_operand (_WhichOperand_limit)
 615 
 616 address Assembler::locate_operand(address inst, WhichOperand which) {
 617   // Decode the given instruction, and return the address of
 618   // an embedded 32-bit operand word.
 619 
 620   // If "which" is disp32_operand, selects the displacement portion
 621   // of an effective address specifier.
 622   // If "which" is imm64_operand, selects the trailing immediate constant.
 623   // If "which" is call32_operand, selects the displacement of a call or jump.
 624   // Caller is responsible for ensuring that there is such an operand,
 625   // and that it is 32/64 bits wide.
 626 
 627   // If "which" is end_pc_operand, find the end of the instruction.
 628 
 629   address ip = inst;
 630   bool is_64bit = false;
 631 
 632   debug_only(bool has_disp32 = false);
 633   int tail_size = 0; // other random bytes (#32, #16, etc.) at end of insn
 634 
 635   again_after_prefix:
 636   switch (0xFF & *ip++) {
 637 
 638   // These convenience macros generate groups of "case" labels for the switch.
 639 #define REP4(x) (x)+0: case (x)+1: case (x)+2: case (x)+3
 640 #define REP8(x) (x)+0: case (x)+1: case (x)+2: case (x)+3: \
 641              case (x)+4: case (x)+5: case (x)+6: case (x)+7
 642 #define REP16(x) REP8((x)+0): \
 643               case REP8((x)+8)
 644 
 645   case CS_segment:
 646   case SS_segment:
 647   case DS_segment:
 648   case ES_segment:
 649   case FS_segment:
 650   case GS_segment:
 651     // Seems dubious
 652     LP64_ONLY(assert(false, "shouldn't have that prefix"));
 653     assert(ip == inst+1, "only one prefix allowed");
 654     goto again_after_prefix;
 655 
 656   case 0x67:
 657   case REX:
 658   case REX_B:
 659   case REX_X:
 660   case REX_XB:
 661   case REX_R:
 662   case REX_RB:
 663   case REX_RX:
 664   case REX_RXB:
 665     NOT_LP64(assert(false, "64bit prefixes"));
 666     goto again_after_prefix;
 667 
 668   case REX_W:
 669   case REX_WB:
 670   case REX_WX:
 671   case REX_WXB:
 672   case REX_WR:
 673   case REX_WRB:
 674   case REX_WRX:
 675   case REX_WRXB:
 676     NOT_LP64(assert(false, "64bit prefixes"));
 677     is_64bit = true;
 678     goto again_after_prefix;
 679 
 680   case 0xFF: // pushq a; decl a; incl a; call a; jmp a
 681   case 0x88: // movb a, r
 682   case 0x89: // movl a, r
 683   case 0x8A: // movb r, a
 684   case 0x8B: // movl r, a
 685   case 0x8F: // popl a
 686     debug_only(has_disp32 = true);
 687     break;
 688 
 689   case 0x68: // pushq #32
 690     if (which == end_pc_operand) {
 691       return ip + 4;
 692     }
 693     assert(which == imm_operand && !is_64bit, "pushl has no disp32 or 64bit immediate");
 694     return ip;                  // not produced by emit_operand
 695 
 696   case 0x66: // movw ... (size prefix)
 697     again_after_size_prefix2:
 698     switch (0xFF & *ip++) {
 699     case REX:
 700     case REX_B:
 701     case REX_X:
 702     case REX_XB:
 703     case REX_R:
 704     case REX_RB:
 705     case REX_RX:
 706     case REX_RXB:
 707     case REX_W:
 708     case REX_WB:
 709     case REX_WX:
 710     case REX_WXB:
 711     case REX_WR:
 712     case REX_WRB:
 713     case REX_WRX:
 714     case REX_WRXB:
 715       NOT_LP64(assert(false, "64bit prefix found"));
 716       goto again_after_size_prefix2;
 717     case 0x8B: // movw r, a
 718     case 0x89: // movw a, r
 719       debug_only(has_disp32 = true);
 720       break;
 721     case 0xC7: // movw a, #16
 722       debug_only(has_disp32 = true);
 723       tail_size = 2;  // the imm16
 724       break;
 725     case 0x0F: // several SSE/SSE2 variants
 726       ip--;    // reparse the 0x0F
 727       goto again_after_prefix;
 728     default:
 729       ShouldNotReachHere();
 730     }
 731     break;
 732 
 733   case REP8(0xB8): // movl/q r, #32/#64(oop?)
 734     if (which == end_pc_operand)  return ip + (is_64bit ? 8 : 4);
 735     // these asserts are somewhat nonsensical
 736 #ifndef _LP64
 737     assert(which == imm_operand || which == disp32_operand,
 738            "which %d is_64_bit %d ip " INTPTR_FORMAT, which, is_64bit, p2i(ip));
 739 #else
 740     assert((which == call32_operand || which == imm_operand) && is_64bit ||
 741            which == narrow_oop_operand && !is_64bit,
 742            "which %d is_64_bit %d ip " INTPTR_FORMAT, which, is_64bit, p2i(ip));
 743 #endif // _LP64
 744     return ip;
 745 
 746   case 0x69: // imul r, a, #32
 747   case 0xC7: // movl a, #32(oop?)
 748     tail_size = 4;
 749     debug_only(has_disp32 = true); // has both kinds of operands!
 750     break;
 751 
 752   case 0x0F: // movx..., etc.
 753     switch (0xFF & *ip++) {
 754     case 0x3A: // pcmpestri
 755       tail_size = 1;
 756     case 0x38: // ptest, pmovzxbw
 757       ip++; // skip opcode
 758       debug_only(has_disp32 = true); // has both kinds of operands!
 759       break;
 760 
 761     case 0x70: // pshufd r, r/a, #8
 762       debug_only(has_disp32 = true); // has both kinds of operands!
 763     case 0x73: // psrldq r, #8
 764       tail_size = 1;
 765       break;
 766 
 767     case 0x12: // movlps
 768     case 0x28: // movaps
 769     case 0x2E: // ucomiss
 770     case 0x2F: // comiss
 771     case 0x54: // andps
 772     case 0x55: // andnps
 773     case 0x56: // orps
 774     case 0x57: // xorps
 775     case 0x58: // addpd
 776     case 0x59: // mulpd
 777     case 0x6E: // movd
 778     case 0x7E: // movd
 779     case 0xAE: // ldmxcsr, stmxcsr, fxrstor, fxsave, clflush
 780     case 0xFE: // paddd
 781       debug_only(has_disp32 = true);
 782       break;
 783 
 784     case 0xAD: // shrd r, a, %cl
 785     case 0xAF: // imul r, a
 786     case 0xBE: // movsbl r, a (movsxb)
 787     case 0xBF: // movswl r, a (movsxw)
 788     case 0xB6: // movzbl r, a (movzxb)
 789     case 0xB7: // movzwl r, a (movzxw)
 790     case REP16(0x40): // cmovl cc, r, a
 791     case 0xB0: // cmpxchgb
 792     case 0xB1: // cmpxchg
 793     case 0xC1: // xaddl
 794     case 0xC7: // cmpxchg8
 795     case REP16(0x90): // setcc a
 796       debug_only(has_disp32 = true);
 797       // fall out of the switch to decode the address
 798       break;
 799 
 800     case 0xC4: // pinsrw r, a, #8
 801       debug_only(has_disp32 = true);
 802     case 0xC5: // pextrw r, r, #8
 803       tail_size = 1;  // the imm8
 804       break;
 805 
 806     case 0xAC: // shrd r, a, #8
 807       debug_only(has_disp32 = true);
 808       tail_size = 1;  // the imm8
 809       break;
 810 
 811     case REP16(0x80): // jcc rdisp32
 812       if (which == end_pc_operand)  return ip + 4;
 813       assert(which == call32_operand, "jcc has no disp32 or imm");
 814       return ip;
 815     default:
 816       ShouldNotReachHere();
 817     }
 818     break;
 819 
 820   case 0x81: // addl a, #32; addl r, #32
 821     // also: orl, adcl, sbbl, andl, subl, xorl, cmpl
 822     // on 32bit in the case of cmpl, the imm might be an oop
 823     tail_size = 4;
 824     debug_only(has_disp32 = true); // has both kinds of operands!
 825     break;
 826 
 827   case 0x83: // addl a, #8; addl r, #8
 828     // also: orl, adcl, sbbl, andl, subl, xorl, cmpl
 829     debug_only(has_disp32 = true); // has both kinds of operands!
 830     tail_size = 1;
 831     break;
 832 
 833   case 0x9B:
 834     switch (0xFF & *ip++) {
 835     case 0xD9: // fnstcw a
 836       debug_only(has_disp32 = true);
 837       break;
 838     default:
 839       ShouldNotReachHere();
 840     }
 841     break;
 842 
 843   case REP4(0x00): // addb a, r; addl a, r; addb r, a; addl r, a
 844   case REP4(0x10): // adc...
 845   case REP4(0x20): // and...
 846   case REP4(0x30): // xor...
 847   case REP4(0x08): // or...
 848   case REP4(0x18): // sbb...
 849   case REP4(0x28): // sub...
 850   case 0xF7: // mull a
 851   case 0x8D: // lea r, a
 852   case 0x87: // xchg r, a
 853   case REP4(0x38): // cmp...
 854   case 0x85: // test r, a
 855     debug_only(has_disp32 = true); // has both kinds of operands!
 856     break;
 857 
 858   case 0xC1: // sal a, #8; sar a, #8; shl a, #8; shr a, #8
 859   case 0xC6: // movb a, #8
 860   case 0x80: // cmpb a, #8
 861   case 0x6B: // imul r, a, #8
 862     debug_only(has_disp32 = true); // has both kinds of operands!
 863     tail_size = 1; // the imm8
 864     break;
 865 
 866   case 0xC4: // VEX_3bytes
 867   case 0xC5: // VEX_2bytes
 868     assert((UseAVX > 0), "shouldn't have VEX prefix");
 869     assert(ip == inst+1, "no prefixes allowed");
 870     // C4 and C5 are also used as opcodes for PINSRW and PEXTRW instructions
 871     // but they have prefix 0x0F and processed when 0x0F processed above.
 872     //
 873     // In 32-bit mode the VEX first byte C4 and C5 alias onto LDS and LES
 874     // instructions (these instructions are not supported in 64-bit mode).
 875     // To distinguish them bits [7:6] are set in the VEX second byte since
 876     // ModRM byte can not be of the form 11xxxxxx in 32-bit mode. To set
 877     // those VEX bits REX and vvvv bits are inverted.
 878     //
 879     // Fortunately C2 doesn't generate these instructions so we don't need
 880     // to check for them in product version.
 881 
 882     // Check second byte
 883     NOT_LP64(assert((0xC0 & *ip) == 0xC0, "shouldn't have LDS and LES instructions"));
 884 
 885     int vex_opcode;
 886     // First byte
 887     if ((0xFF & *inst) == VEX_3bytes) {
 888       vex_opcode = VEX_OPCODE_MASK & *ip;
 889       ip++; // third byte
 890       is_64bit = ((VEX_W & *ip) == VEX_W);
 891     } else {
 892       vex_opcode = VEX_OPCODE_0F;
 893     }
 894     ip++; // opcode
 895     // To find the end of instruction (which == end_pc_operand).
 896     switch (vex_opcode) {
 897       case VEX_OPCODE_0F:
 898         switch (0xFF & *ip) {
 899         case 0x70: // pshufd r, r/a, #8
 900         case 0x71: // ps[rl|ra|ll]w r, #8
 901         case 0x72: // ps[rl|ra|ll]d r, #8
 902         case 0x73: // ps[rl|ra|ll]q r, #8
 903         case 0xC2: // cmp[ps|pd|ss|sd] r, r, r/a, #8
 904         case 0xC4: // pinsrw r, r, r/a, #8
 905         case 0xC5: // pextrw r/a, r, #8
 906         case 0xC6: // shufp[s|d] r, r, r/a, #8
 907           tail_size = 1;  // the imm8
 908           break;
 909         }
 910         break;
 911       case VEX_OPCODE_0F_3A:
 912         tail_size = 1;
 913         break;
 914     }
 915     ip++; // skip opcode
 916     debug_only(has_disp32 = true); // has both kinds of operands!
 917     break;
 918 
 919   case 0x62: // EVEX_4bytes
 920     assert(VM_Version::supports_evex(), "shouldn't have EVEX prefix");
 921     assert(ip == inst+1, "no prefixes allowed");
 922     // no EVEX collisions, all instructions that have 0x62 opcodes
 923     // have EVEX versions and are subopcodes of 0x66
 924     ip++; // skip P0 and exmaine W in P1
 925     is_64bit = ((VEX_W & *ip) == VEX_W);
 926     ip++; // move to P2
 927     ip++; // skip P2, move to opcode
 928     // To find the end of instruction (which == end_pc_operand).
 929     switch (0xFF & *ip) {
 930     case 0x22: // pinsrd r, r/a, #8
 931     case 0x61: // pcmpestri r, r/a, #8
 932     case 0x70: // pshufd r, r/a, #8
 933     case 0x73: // psrldq r, #8
 934       tail_size = 1;  // the imm8
 935       break;
 936     default:
 937       break;
 938     }
 939     ip++; // skip opcode
 940     debug_only(has_disp32 = true); // has both kinds of operands!
 941     break;
 942 
 943   case 0xD1: // sal a, 1; sar a, 1; shl a, 1; shr a, 1
 944   case 0xD3: // sal a, %cl; sar a, %cl; shl a, %cl; shr a, %cl
 945   case 0xD9: // fld_s a; fst_s a; fstp_s a; fldcw a
 946   case 0xDD: // fld_d a; fst_d a; fstp_d a
 947   case 0xDB: // fild_s a; fistp_s a; fld_x a; fstp_x a
 948   case 0xDF: // fild_d a; fistp_d a
 949   case 0xD8: // fadd_s a; fsubr_s a; fmul_s a; fdivr_s a; fcomp_s a
 950   case 0xDC: // fadd_d a; fsubr_d a; fmul_d a; fdivr_d a; fcomp_d a
 951   case 0xDE: // faddp_d a; fsubrp_d a; fmulp_d a; fdivrp_d a; fcompp_d a
 952     debug_only(has_disp32 = true);
 953     break;
 954 
 955   case 0xE8: // call rdisp32
 956   case 0xE9: // jmp  rdisp32
 957     if (which == end_pc_operand)  return ip + 4;
 958     assert(which == call32_operand, "call has no disp32 or imm");
 959     return ip;
 960 
 961   case 0xF0:                    // Lock
 962     assert(os::is_MP(), "only on MP");
 963     goto again_after_prefix;
 964 
 965   case 0xF3:                    // For SSE
 966   case 0xF2:                    // For SSE2
 967     switch (0xFF & *ip++) {
 968     case REX:
 969     case REX_B:
 970     case REX_X:
 971     case REX_XB:
 972     case REX_R:
 973     case REX_RB:
 974     case REX_RX:
 975     case REX_RXB:
 976     case REX_W:
 977     case REX_WB:
 978     case REX_WX:
 979     case REX_WXB:
 980     case REX_WR:
 981     case REX_WRB:
 982     case REX_WRX:
 983     case REX_WRXB:
 984       NOT_LP64(assert(false, "found 64bit prefix"));
 985       ip++;
 986     default:
 987       ip++;
 988     }
 989     debug_only(has_disp32 = true); // has both kinds of operands!
 990     break;
 991 
 992   default:
 993     ShouldNotReachHere();
 994 
 995 #undef REP8
 996 #undef REP16
 997   }
 998 
 999   assert(which != call32_operand, "instruction is not a call, jmp, or jcc");
1000 #ifdef _LP64
1001   assert(which != imm_operand, "instruction is not a movq reg, imm64");
1002 #else
1003   // assert(which != imm_operand || has_imm32, "instruction has no imm32 field");
1004   assert(which != imm_operand || has_disp32, "instruction has no imm32 field");
1005 #endif // LP64
1006   assert(which != disp32_operand || has_disp32, "instruction has no disp32 field");
1007 
1008   // parse the output of emit_operand
1009   int op2 = 0xFF & *ip++;
1010   int base = op2 & 0x07;
1011   int op3 = -1;
1012   const int b100 = 4;
1013   const int b101 = 5;
1014   if (base == b100 && (op2 >> 6) != 3) {
1015     op3 = 0xFF & *ip++;
1016     base = op3 & 0x07;   // refetch the base
1017   }
1018   // now ip points at the disp (if any)
1019 
1020   switch (op2 >> 6) {
1021   case 0:
1022     // [00 reg  100][ss index base]
1023     // [00 reg  100][00   100  esp]
1024     // [00 reg base]
1025     // [00 reg  100][ss index  101][disp32]
1026     // [00 reg  101]               [disp32]
1027 
1028     if (base == b101) {
1029       if (which == disp32_operand)
1030         return ip;              // caller wants the disp32
1031       ip += 4;                  // skip the disp32
1032     }
1033     break;
1034 
1035   case 1:
1036     // [01 reg  100][ss index base][disp8]
1037     // [01 reg  100][00   100  esp][disp8]
1038     // [01 reg base]               [disp8]
1039     ip += 1;                    // skip the disp8
1040     break;
1041 
1042   case 2:
1043     // [10 reg  100][ss index base][disp32]
1044     // [10 reg  100][00   100  esp][disp32]
1045     // [10 reg base]               [disp32]
1046     if (which == disp32_operand)
1047       return ip;                // caller wants the disp32
1048     ip += 4;                    // skip the disp32
1049     break;
1050 
1051   case 3:
1052     // [11 reg base]  (not a memory addressing mode)
1053     break;
1054   }
1055 
1056   if (which == end_pc_operand) {
1057     return ip + tail_size;
1058   }
1059 
1060 #ifdef _LP64
1061   assert(which == narrow_oop_operand && !is_64bit, "instruction is not a movl adr, imm32");
1062 #else
1063   assert(which == imm_operand, "instruction has only an imm field");
1064 #endif // LP64
1065   return ip;
1066 }
1067 
1068 address Assembler::locate_next_instruction(address inst) {
1069   // Secretly share code with locate_operand:
1070   return locate_operand(inst, end_pc_operand);
1071 }
1072 
1073 
1074 #ifdef ASSERT
1075 void Assembler::check_relocation(RelocationHolder const& rspec, int format) {
1076   address inst = inst_mark();
1077   assert(inst != NULL && inst < pc(), "must point to beginning of instruction");
1078   address opnd;
1079 
1080   Relocation* r = rspec.reloc();
1081   if (r->type() == relocInfo::none) {
1082     return;
1083   } else if (r->is_call() || format == call32_operand) {
1084     // assert(format == imm32_operand, "cannot specify a nonzero format");
1085     opnd = locate_operand(inst, call32_operand);
1086   } else if (r->is_data()) {
1087     assert(format == imm_operand || format == disp32_operand
1088            LP64_ONLY(|| format == narrow_oop_operand), "format ok");
1089     opnd = locate_operand(inst, (WhichOperand)format);
1090   } else {
1091     assert(format == imm_operand, "cannot specify a format");
1092     return;
1093   }
1094   assert(opnd == pc(), "must put operand where relocs can find it");
1095 }
1096 #endif // ASSERT
1097 
1098 void Assembler::emit_operand32(Register reg, Address adr) {
1099   assert(reg->encoding() < 8, "no extended registers");
1100   assert(!adr.base_needs_rex() && !adr.index_needs_rex(), "no extended registers");
1101   emit_operand(reg, adr._base, adr._index, adr._scale, adr._disp,
1102                adr._rspec);
1103 }
1104 
1105 void Assembler::emit_operand(Register reg, Address adr,
1106                              int rip_relative_correction) {
1107   emit_operand(reg, adr._base, adr._index, adr._scale, adr._disp,
1108                adr._rspec,
1109                rip_relative_correction);
1110 }
1111 
1112 void Assembler::emit_operand(XMMRegister reg, Address adr) {
1113   emit_operand(reg, adr._base, adr._index, adr._scale, adr._disp,
1114                adr._rspec);
1115 }
1116 
1117 // MMX operations
1118 void Assembler::emit_operand(MMXRegister reg, Address adr) {
1119   assert(!adr.base_needs_rex() && !adr.index_needs_rex(), "no extended registers");
1120   emit_operand((Register)reg, adr._base, adr._index, adr._scale, adr._disp, adr._rspec);
1121 }
1122 
1123 // work around gcc (3.2.1-7a) bug
1124 void Assembler::emit_operand(Address adr, MMXRegister reg) {
1125   assert(!adr.base_needs_rex() && !adr.index_needs_rex(), "no extended registers");
1126   emit_operand((Register)reg, adr._base, adr._index, adr._scale, adr._disp, adr._rspec);
1127 }
1128 
1129 
1130 void Assembler::emit_farith(int b1, int b2, int i) {
1131   assert(isByte(b1) && isByte(b2), "wrong opcode");
1132   assert(0 <= i &&  i < 8, "illegal stack offset");
1133   emit_int8(b1);
1134   emit_int8(b2 + i);
1135 }
1136 
1137 
1138 // Now the Assembler instructions (identical for 32/64 bits)
1139 
1140 void Assembler::adcl(Address dst, int32_t imm32) {
1141   InstructionMark im(this);
1142   prefix(dst);
1143   emit_arith_operand(0x81, rdx, dst, imm32);
1144 }
1145 
1146 void Assembler::adcl(Address dst, Register src) {
1147   InstructionMark im(this);
1148   prefix(dst, src);
1149   emit_int8(0x11);
1150   emit_operand(src, dst);
1151 }
1152 
1153 void Assembler::adcl(Register dst, int32_t imm32) {
1154   prefix(dst);
1155   emit_arith(0x81, 0xD0, dst, imm32);
1156 }
1157 
1158 void Assembler::adcl(Register dst, Address src) {
1159   InstructionMark im(this);
1160   prefix(src, dst);
1161   emit_int8(0x13);
1162   emit_operand(dst, src);
1163 }
1164 
1165 void Assembler::adcl(Register dst, Register src) {
1166   (void) prefix_and_encode(dst->encoding(), src->encoding());
1167   emit_arith(0x13, 0xC0, dst, src);
1168 }
1169 
1170 void Assembler::addl(Address dst, int32_t imm32) {
1171   InstructionMark im(this);
1172   prefix(dst);
1173   emit_arith_operand(0x81, rax, dst, imm32);
1174 }
1175 
1176 void Assembler::addb(Address dst, int imm8) {
1177   InstructionMark im(this);
1178   prefix(dst);
1179   emit_int8((unsigned char)0x80);
1180   emit_operand(rax, dst, 1);
1181   emit_int8(imm8);
1182 }
1183 
1184 void Assembler::addw(Address dst, int imm16) {
1185   InstructionMark im(this);
1186   emit_int8(0x66);
1187   prefix(dst);
1188   emit_int8((unsigned char)0x81);
1189   emit_operand(rax, dst, 2);
1190   emit_int16(imm16);
1191 }
1192 
1193 void Assembler::addl(Address dst, Register src) {
1194   InstructionMark im(this);
1195   prefix(dst, src);
1196   emit_int8(0x01);
1197   emit_operand(src, dst);
1198 }
1199 
1200 void Assembler::addl(Register dst, int32_t imm32) {
1201   prefix(dst);
1202   emit_arith(0x81, 0xC0, dst, imm32);
1203 }
1204 
1205 void Assembler::addl(Register dst, Address src) {
1206   InstructionMark im(this);
1207   prefix(src, dst);
1208   emit_int8(0x03);
1209   emit_operand(dst, src);
1210 }
1211 
1212 void Assembler::addl(Register dst, Register src) {
1213   (void) prefix_and_encode(dst->encoding(), src->encoding());
1214   emit_arith(0x03, 0xC0, dst, src);
1215 }
1216 
1217 void Assembler::addr_nop_4() {
1218   assert(UseAddressNop, "no CPU support");
1219   // 4 bytes: NOP DWORD PTR [EAX+0]
1220   emit_int8(0x0F);
1221   emit_int8(0x1F);
1222   emit_int8(0x40); // emit_rm(cbuf, 0x1, EAX_enc, EAX_enc);
1223   emit_int8(0);    // 8-bits offset (1 byte)
1224 }
1225 
1226 void Assembler::addr_nop_5() {
1227   assert(UseAddressNop, "no CPU support");
1228   // 5 bytes: NOP DWORD PTR [EAX+EAX*0+0] 8-bits offset
1229   emit_int8(0x0F);
1230   emit_int8(0x1F);
1231   emit_int8(0x44); // emit_rm(cbuf, 0x1, EAX_enc, 0x4);
1232   emit_int8(0x00); // emit_rm(cbuf, 0x0, EAX_enc, EAX_enc);
1233   emit_int8(0);    // 8-bits offset (1 byte)
1234 }
1235 
1236 void Assembler::addr_nop_7() {
1237   assert(UseAddressNop, "no CPU support");
1238   // 7 bytes: NOP DWORD PTR [EAX+0] 32-bits offset
1239   emit_int8(0x0F);
1240   emit_int8(0x1F);
1241   emit_int8((unsigned char)0x80);
1242                    // emit_rm(cbuf, 0x2, EAX_enc, EAX_enc);
1243   emit_int32(0);   // 32-bits offset (4 bytes)
1244 }
1245 
1246 void Assembler::addr_nop_8() {
1247   assert(UseAddressNop, "no CPU support");
1248   // 8 bytes: NOP DWORD PTR [EAX+EAX*0+0] 32-bits offset
1249   emit_int8(0x0F);
1250   emit_int8(0x1F);
1251   emit_int8((unsigned char)0x84);
1252                    // emit_rm(cbuf, 0x2, EAX_enc, 0x4);
1253   emit_int8(0x00); // emit_rm(cbuf, 0x0, EAX_enc, EAX_enc);
1254   emit_int32(0);   // 32-bits offset (4 bytes)
1255 }
1256 
1257 void Assembler::addsd(XMMRegister dst, XMMRegister src) {
1258   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1259   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1260   attributes.set_rex_vex_w_reverted();
1261   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
1262   emit_int8(0x58);
1263   emit_int8((unsigned char)(0xC0 | encode));
1264 }
1265 
1266 void Assembler::addsd(XMMRegister dst, Address src) {
1267   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1268   InstructionMark im(this);
1269   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1270   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
1271   attributes.set_rex_vex_w_reverted();
1272   simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
1273   emit_int8(0x58);
1274   emit_operand(dst, src);
1275 }
1276 
1277 void Assembler::addss(XMMRegister dst, XMMRegister src) {
1278   NOT_LP64(assert(VM_Version::supports_sse(), ""));
1279   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1280   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
1281   emit_int8(0x58);
1282   emit_int8((unsigned char)(0xC0 | encode));
1283 }
1284 
1285 void Assembler::addss(XMMRegister dst, Address src) {
1286   NOT_LP64(assert(VM_Version::supports_sse(), ""));
1287   InstructionMark im(this);
1288   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1289   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
1290   simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
1291   emit_int8(0x58);
1292   emit_operand(dst, src);
1293 }
1294 
1295 void Assembler::aesdec(XMMRegister dst, Address src) {
1296   assert(VM_Version::supports_aes(), "");
1297   InstructionMark im(this);
1298   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
1299   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
1300   emit_int8((unsigned char)0xDE);
1301   emit_operand(dst, src);
1302 }
1303 
1304 void Assembler::aesdec(XMMRegister dst, XMMRegister src) {
1305   assert(VM_Version::supports_aes(), "");
1306   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
1307   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
1308   emit_int8((unsigned char)0xDE);
1309   emit_int8(0xC0 | encode);
1310 }
1311 
1312 void Assembler::aesdeclast(XMMRegister dst, Address src) {
1313   assert(VM_Version::supports_aes(), "");
1314   InstructionMark im(this);
1315   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
1316   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
1317   emit_int8((unsigned char)0xDF);
1318   emit_operand(dst, src);
1319 }
1320 
1321 void Assembler::aesdeclast(XMMRegister dst, XMMRegister src) {
1322   assert(VM_Version::supports_aes(), "");
1323   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
1324   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
1325   emit_int8((unsigned char)0xDF);
1326   emit_int8((unsigned char)(0xC0 | encode));
1327 }
1328 
1329 void Assembler::aesenc(XMMRegister dst, Address src) {
1330   assert(VM_Version::supports_aes(), "");
1331   InstructionMark im(this);
1332   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
1333   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
1334   emit_int8((unsigned char)0xDC);
1335   emit_operand(dst, src);
1336 }
1337 
1338 void Assembler::aesenc(XMMRegister dst, XMMRegister src) {
1339   assert(VM_Version::supports_aes(), "");
1340   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
1341   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
1342   emit_int8((unsigned char)0xDC);
1343   emit_int8(0xC0 | encode);
1344 }
1345 
1346 void Assembler::aesenclast(XMMRegister dst, Address src) {
1347   assert(VM_Version::supports_aes(), "");
1348   InstructionMark im(this);
1349   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
1350   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
1351   emit_int8((unsigned char)0xDD);
1352   emit_operand(dst, src);
1353 }
1354 
1355 void Assembler::aesenclast(XMMRegister dst, XMMRegister src) {
1356   assert(VM_Version::supports_aes(), "");
1357   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
1358   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
1359   emit_int8((unsigned char)0xDD);
1360   emit_int8((unsigned char)(0xC0 | encode));
1361 }
1362 
1363 void Assembler::andl(Address dst, int32_t imm32) {
1364   InstructionMark im(this);
1365   prefix(dst);
1366   emit_int8((unsigned char)0x81);
1367   emit_operand(rsp, dst, 4);
1368   emit_int32(imm32);
1369 }
1370 
1371 void Assembler::andl(Register dst, int32_t imm32) {
1372   prefix(dst);
1373   emit_arith(0x81, 0xE0, dst, imm32);
1374 }
1375 
1376 void Assembler::andl(Register dst, Address src) {
1377   InstructionMark im(this);
1378   prefix(src, dst);
1379   emit_int8(0x23);
1380   emit_operand(dst, src);
1381 }
1382 
1383 void Assembler::andl(Register dst, Register src) {
1384   (void) prefix_and_encode(dst->encoding(), src->encoding());
1385   emit_arith(0x23, 0xC0, dst, src);
1386 }
1387 
1388 void Assembler::andnl(Register dst, Register src1, Register src2) {
1389   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1390   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
1391   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
1392   emit_int8((unsigned char)0xF2);
1393   emit_int8((unsigned char)(0xC0 | encode));
1394 }
1395 
1396 void Assembler::andnl(Register dst, Register src1, Address src2) {
1397   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1398   InstructionMark im(this);
1399   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
1400   vex_prefix(src2, src1->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
1401   emit_int8((unsigned char)0xF2);
1402   emit_operand(dst, src2);
1403 }
1404 
1405 void Assembler::bsfl(Register dst, Register src) {
1406   int encode = prefix_and_encode(dst->encoding(), src->encoding());
1407   emit_int8(0x0F);
1408   emit_int8((unsigned char)0xBC);
1409   emit_int8((unsigned char)(0xC0 | encode));
1410 }
1411 
1412 void Assembler::bsrl(Register dst, Register src) {
1413   int encode = prefix_and_encode(dst->encoding(), src->encoding());
1414   emit_int8(0x0F);
1415   emit_int8((unsigned char)0xBD);
1416   emit_int8((unsigned char)(0xC0 | encode));
1417 }
1418 
1419 void Assembler::bswapl(Register reg) { // bswap
1420   int encode = prefix_and_encode(reg->encoding());
1421   emit_int8(0x0F);
1422   emit_int8((unsigned char)(0xC8 | encode));
1423 }
1424 
1425 void Assembler::blsil(Register dst, Register src) {
1426   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1427   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
1428   int encode = vex_prefix_and_encode(rbx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
1429   emit_int8((unsigned char)0xF3);
1430   emit_int8((unsigned char)(0xC0 | encode));
1431 }
1432 
1433 void Assembler::blsil(Register dst, Address src) {
1434   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1435   InstructionMark im(this);
1436   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
1437   vex_prefix(src, dst->encoding(), rbx->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
1438   emit_int8((unsigned char)0xF3);
1439   emit_operand(rbx, src);
1440 }
1441 
1442 void Assembler::blsmskl(Register dst, Register src) {
1443   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1444   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
1445   int encode = vex_prefix_and_encode(rdx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
1446   emit_int8((unsigned char)0xF3);
1447   emit_int8((unsigned char)(0xC0 | encode));
1448 }
1449 
1450 void Assembler::blsmskl(Register dst, Address src) {
1451   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1452   InstructionMark im(this);
1453   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
1454   vex_prefix(src, dst->encoding(), rdx->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
1455   emit_int8((unsigned char)0xF3);
1456   emit_operand(rdx, src);
1457 }
1458 
1459 void Assembler::blsrl(Register dst, Register src) {
1460   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1461   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
1462   int encode = vex_prefix_and_encode(rcx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
1463   emit_int8((unsigned char)0xF3);
1464   emit_int8((unsigned char)(0xC0 | encode));
1465 }
1466 
1467 void Assembler::blsrl(Register dst, Address src) {
1468   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1469   InstructionMark im(this);
1470   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
1471   vex_prefix(src, dst->encoding(), rcx->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
1472   emit_int8((unsigned char)0xF3);
1473   emit_operand(rcx, src);
1474 }
1475 
1476 void Assembler::call(Label& L, relocInfo::relocType rtype) {
1477   // suspect disp32 is always good
1478   int operand = LP64_ONLY(disp32_operand) NOT_LP64(imm_operand);
1479 
1480   if (L.is_bound()) {
1481     const int long_size = 5;
1482     int offs = (int)( target(L) - pc() );
1483     assert(offs <= 0, "assembler error");
1484     InstructionMark im(this);
1485     // 1110 1000 #32-bit disp
1486     emit_int8((unsigned char)0xE8);
1487     emit_data(offs - long_size, rtype, operand);
1488   } else {
1489     InstructionMark im(this);
1490     // 1110 1000 #32-bit disp
1491     L.add_patch_at(code(), locator());
1492 
1493     emit_int8((unsigned char)0xE8);
1494     emit_data(int(0), rtype, operand);
1495   }
1496 }
1497 
1498 void Assembler::call(Register dst) {
1499   int encode = prefix_and_encode(dst->encoding());
1500   emit_int8((unsigned char)0xFF);
1501   emit_int8((unsigned char)(0xD0 | encode));
1502 }
1503 
1504 
1505 void Assembler::call(Address adr) {
1506   InstructionMark im(this);
1507   prefix(adr);
1508   emit_int8((unsigned char)0xFF);
1509   emit_operand(rdx, adr);
1510 }
1511 
1512 void Assembler::call_literal(address entry, RelocationHolder const& rspec) {
1513   InstructionMark im(this);
1514   emit_int8((unsigned char)0xE8);
1515   intptr_t disp = entry - (pc() + sizeof(int32_t));
1516   // Entry is NULL in case of a scratch emit.
1517   assert(entry == NULL || is_simm32(disp), "disp=" INTPTR_FORMAT " must be 32bit offset (call2)", disp);
1518   // Technically, should use call32_operand, but this format is
1519   // implied by the fact that we're emitting a call instruction.
1520 
1521   int operand = LP64_ONLY(disp32_operand) NOT_LP64(call32_operand);
1522   emit_data((int) disp, rspec, operand);
1523 }
1524 
1525 void Assembler::cdql() {
1526   emit_int8((unsigned char)0x99);
1527 }
1528 
1529 void Assembler::cld() {
1530   emit_int8((unsigned char)0xFC);
1531 }
1532 
1533 void Assembler::cmovl(Condition cc, Register dst, Register src) {
1534   NOT_LP64(guarantee(VM_Version::supports_cmov(), "illegal instruction"));
1535   int encode = prefix_and_encode(dst->encoding(), src->encoding());
1536   emit_int8(0x0F);
1537   emit_int8(0x40 | cc);
1538   emit_int8((unsigned char)(0xC0 | encode));
1539 }
1540 
1541 
1542 void Assembler::cmovl(Condition cc, Register dst, Address src) {
1543   NOT_LP64(guarantee(VM_Version::supports_cmov(), "illegal instruction"));
1544   prefix(src, dst);
1545   emit_int8(0x0F);
1546   emit_int8(0x40 | cc);
1547   emit_operand(dst, src);
1548 }
1549 
1550 void Assembler::cmpb(Address dst, int imm8) {
1551   InstructionMark im(this);
1552   prefix(dst);
1553   emit_int8((unsigned char)0x80);
1554   emit_operand(rdi, dst, 1);
1555   emit_int8(imm8);
1556 }
1557 
1558 void Assembler::cmpl(Address dst, int32_t imm32) {
1559   InstructionMark im(this);
1560   prefix(dst);
1561   emit_int8((unsigned char)0x81);
1562   emit_operand(rdi, dst, 4);
1563   emit_int32(imm32);
1564 }
1565 
1566 void Assembler::cmpl(Register dst, int32_t imm32) {
1567   prefix(dst);
1568   emit_arith(0x81, 0xF8, dst, imm32);
1569 }
1570 
1571 void Assembler::cmpl(Register dst, Register src) {
1572   (void) prefix_and_encode(dst->encoding(), src->encoding());
1573   emit_arith(0x3B, 0xC0, dst, src);
1574 }
1575 
1576 void Assembler::cmpl(Register dst, Address  src) {
1577   InstructionMark im(this);
1578   prefix(src, dst);
1579   emit_int8((unsigned char)0x3B);
1580   emit_operand(dst, src);
1581 }
1582 
1583 void Assembler::cmpw(Address dst, int imm16) {
1584   InstructionMark im(this);
1585   assert(!dst.base_needs_rex() && !dst.index_needs_rex(), "no extended registers");
1586   emit_int8(0x66);
1587   emit_int8((unsigned char)0x81);
1588   emit_operand(rdi, dst, 2);
1589   emit_int16(imm16);
1590 }
1591 
1592 // The 32-bit cmpxchg compares the value at adr with the contents of rax,
1593 // and stores reg into adr if so; otherwise, the value at adr is loaded into rax,.
1594 // The ZF is set if the compared values were equal, and cleared otherwise.
1595 void Assembler::cmpxchgl(Register reg, Address adr) { // cmpxchg
1596   InstructionMark im(this);
1597   prefix(adr, reg);
1598   emit_int8(0x0F);
1599   emit_int8((unsigned char)0xB1);
1600   emit_operand(reg, adr);
1601 }
1602 
1603 // The 8-bit cmpxchg compares the value at adr with the contents of rax,
1604 // and stores reg into adr if so; otherwise, the value at adr is loaded into rax,.
1605 // The ZF is set if the compared values were equal, and cleared otherwise.
1606 void Assembler::cmpxchgb(Register reg, Address adr) { // cmpxchg
1607   InstructionMark im(this);
1608   prefix(adr, reg, true);
1609   emit_int8(0x0F);
1610   emit_int8((unsigned char)0xB0);
1611   emit_operand(reg, adr);
1612 }
1613 
1614 void Assembler::comisd(XMMRegister dst, Address src) {
1615   // NOTE: dbx seems to decode this as comiss even though the
1616   // 0x66 is there. Strangly ucomisd comes out correct
1617   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1618   InstructionMark im(this);
1619   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);;
1620   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
1621   attributes.set_rex_vex_w_reverted();
1622   simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
1623   emit_int8(0x2F);
1624   emit_operand(dst, src);
1625 }
1626 
1627 void Assembler::comisd(XMMRegister dst, XMMRegister src) {
1628   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1629   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1630   attributes.set_rex_vex_w_reverted();
1631   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
1632   emit_int8(0x2F);
1633   emit_int8((unsigned char)(0xC0 | encode));
1634 }
1635 
1636 void Assembler::comiss(XMMRegister dst, Address src) {
1637   NOT_LP64(assert(VM_Version::supports_sse(), ""));
1638   InstructionMark im(this);
1639   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1640   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
1641   simd_prefix(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
1642   emit_int8(0x2F);
1643   emit_operand(dst, src);
1644 }
1645 
1646 void Assembler::comiss(XMMRegister dst, XMMRegister src) {
1647   NOT_LP64(assert(VM_Version::supports_sse(), ""));
1648   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1649   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
1650   emit_int8(0x2F);
1651   emit_int8((unsigned char)(0xC0 | encode));
1652 }
1653 
1654 void Assembler::cpuid() {
1655   emit_int8(0x0F);
1656   emit_int8((unsigned char)0xA2);
1657 }
1658 
1659 // Opcode / Instruction                      Op /  En  64 - Bit Mode     Compat / Leg Mode Description                  Implemented
1660 // F2 0F 38 F0 / r       CRC32 r32, r / m8   RM        Valid             Valid             Accumulate CRC32 on r / m8.  v
1661 // F2 REX 0F 38 F0 / r   CRC32 r32, r / m8*  RM        Valid             N.E.              Accumulate CRC32 on r / m8.  -
1662 // F2 REX.W 0F 38 F0 / r CRC32 r64, r / m8   RM        Valid             N.E.              Accumulate CRC32 on r / m8.  -
1663 //
1664 // F2 0F 38 F1 / r       CRC32 r32, r / m16  RM        Valid             Valid             Accumulate CRC32 on r / m16. v
1665 //
1666 // F2 0F 38 F1 / r       CRC32 r32, r / m32  RM        Valid             Valid             Accumulate CRC32 on r / m32. v
1667 //
1668 // F2 REX.W 0F 38 F1 / r CRC32 r64, r / m64  RM        Valid             N.E.              Accumulate CRC32 on r / m64. v
1669 void Assembler::crc32(Register crc, Register v, int8_t sizeInBytes) {
1670   assert(VM_Version::supports_sse4_2(), "");
1671   int8_t w = 0x01;
1672   Prefix p = Prefix_EMPTY;
1673 
1674   emit_int8((int8_t)0xF2);
1675   switch (sizeInBytes) {
1676   case 1:
1677     w = 0;
1678     break;
1679   case 2:
1680   case 4:
1681     break;
1682   LP64_ONLY(case 8:)
1683     // This instruction is not valid in 32 bits
1684     // Note:
1685     // http://www.intel.com/content/dam/www/public/us/en/documents/manuals/64-ia-32-architectures-software-developer-instruction-set-reference-manual-325383.pdf
1686     //
1687     // Page B - 72   Vol. 2C says
1688     // qwreg2 to qwreg            1111 0010 : 0100 1R0B : 0000 1111 : 0011 1000 : 1111 0000 : 11 qwreg1 qwreg2
1689     // mem64 to qwreg             1111 0010 : 0100 1R0B : 0000 1111 : 0011 1000 : 1111 0000 : mod qwreg r / m
1690     //                                                                            F0!!!
1691     // while 3 - 208 Vol. 2A
1692     // F2 REX.W 0F 38 F1 / r       CRC32 r64, r / m64             RM         Valid      N.E.Accumulate CRC32 on r / m64.
1693     //
1694     // the 0 on a last bit is reserved for a different flavor of this instruction :
1695     // F2 REX.W 0F 38 F0 / r       CRC32 r64, r / m8              RM         Valid      N.E.Accumulate CRC32 on r / m8.
1696     p = REX_W;
1697     break;
1698   default:
1699     assert(0, "Unsupported value for a sizeInBytes argument");
1700     break;
1701   }
1702   LP64_ONLY(prefix(crc, v, p);)
1703   emit_int8((int8_t)0x0F);
1704   emit_int8(0x38);
1705   emit_int8((int8_t)(0xF0 | w));
1706   emit_int8(0xC0 | ((crc->encoding() & 0x7) << 3) | (v->encoding() & 7));
1707 }
1708 
1709 void Assembler::crc32(Register crc, Address adr, int8_t sizeInBytes) {
1710   assert(VM_Version::supports_sse4_2(), "");
1711   InstructionMark im(this);
1712   int8_t w = 0x01;
1713   Prefix p = Prefix_EMPTY;
1714 
1715   emit_int8((int8_t)0xF2);
1716   switch (sizeInBytes) {
1717   case 1:
1718     w = 0;
1719     break;
1720   case 2:
1721   case 4:
1722     break;
1723   LP64_ONLY(case 8:)
1724     // This instruction is not valid in 32 bits
1725     p = REX_W;
1726     break;
1727   default:
1728     assert(0, "Unsupported value for a sizeInBytes argument");
1729     break;
1730   }
1731   LP64_ONLY(prefix(crc, adr, p);)
1732   emit_int8((int8_t)0x0F);
1733   emit_int8(0x38);
1734   emit_int8((int8_t)(0xF0 | w));
1735   emit_operand(crc, adr);
1736 }
1737 
1738 void Assembler::cvtdq2pd(XMMRegister dst, XMMRegister src) {
1739   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1740   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
1741   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
1742   emit_int8((unsigned char)0xE6);
1743   emit_int8((unsigned char)(0xC0 | encode));
1744 }
1745 
1746 void Assembler::cvtdq2ps(XMMRegister dst, XMMRegister src) {
1747   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1748   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
1749   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
1750   emit_int8(0x5B);
1751   emit_int8((unsigned char)(0xC0 | encode));
1752 }
1753 
1754 void Assembler::cvtsd2ss(XMMRegister dst, XMMRegister src) {
1755   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1756   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1757   attributes.set_rex_vex_w_reverted();
1758   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
1759   emit_int8(0x5A);
1760   emit_int8((unsigned char)(0xC0 | encode));
1761 }
1762 
1763 void Assembler::cvtsd2ss(XMMRegister dst, Address src) {
1764   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1765   InstructionMark im(this);
1766   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1767   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
1768   attributes.set_rex_vex_w_reverted();
1769   simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
1770   emit_int8(0x5A);
1771   emit_operand(dst, src);
1772 }
1773 
1774 void Assembler::cvtsi2sdl(XMMRegister dst, Register src) {
1775   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1776   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1777   int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
1778   emit_int8(0x2A);
1779   emit_int8((unsigned char)(0xC0 | encode));
1780 }
1781 
1782 void Assembler::cvtsi2sdl(XMMRegister dst, Address src) {
1783   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1784   InstructionMark im(this);
1785   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1786   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
1787   simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
1788   emit_int8(0x2A);
1789   emit_operand(dst, src);
1790 }
1791 
1792 void Assembler::cvtsi2ssl(XMMRegister dst, Register src) {
1793   NOT_LP64(assert(VM_Version::supports_sse(), ""));
1794   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1795   int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
1796   emit_int8(0x2A);
1797   emit_int8((unsigned char)(0xC0 | encode));
1798 }
1799 
1800 void Assembler::cvtsi2ssl(XMMRegister dst, Address src) {
1801   NOT_LP64(assert(VM_Version::supports_sse(), ""));
1802   InstructionMark im(this);
1803   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1804   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
1805   simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
1806   emit_int8(0x2A);
1807   emit_operand(dst, src);
1808 }
1809 
1810 void Assembler::cvtsi2ssq(XMMRegister dst, Register src) {
1811   NOT_LP64(assert(VM_Version::supports_sse(), ""));
1812   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1813   int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
1814   emit_int8(0x2A);
1815   emit_int8((unsigned char)(0xC0 | encode));
1816 }
1817 
1818 void Assembler::cvtss2sd(XMMRegister dst, XMMRegister src) {
1819   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1820   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1821   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
1822   emit_int8(0x5A);
1823   emit_int8((unsigned char)(0xC0 | encode));
1824 }
1825 
1826 void Assembler::cvtss2sd(XMMRegister dst, Address src) {
1827   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1828   InstructionMark im(this);
1829   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1830   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
1831   simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
1832   emit_int8(0x5A);
1833   emit_operand(dst, src);
1834 }
1835 
1836 
1837 void Assembler::cvttsd2sil(Register dst, XMMRegister src) {
1838   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1839   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1840   int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
1841   emit_int8(0x2C);
1842   emit_int8((unsigned char)(0xC0 | encode));
1843 }
1844 
1845 void Assembler::cvttss2sil(Register dst, XMMRegister src) {
1846   NOT_LP64(assert(VM_Version::supports_sse(), ""));
1847   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1848   int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
1849   emit_int8(0x2C);
1850   emit_int8((unsigned char)(0xC0 | encode));
1851 }
1852 
1853 void Assembler::cvttpd2dq(XMMRegister dst, XMMRegister src) {
1854   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1855   int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_128bit;
1856   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
1857   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
1858   emit_int8((unsigned char)0xE6);
1859   emit_int8((unsigned char)(0xC0 | encode));
1860 }
1861 
1862 void Assembler::decl(Address dst) {
1863   // Don't use it directly. Use MacroAssembler::decrement() instead.
1864   InstructionMark im(this);
1865   prefix(dst);
1866   emit_int8((unsigned char)0xFF);
1867   emit_operand(rcx, dst);
1868 }
1869 
1870 void Assembler::divsd(XMMRegister dst, Address src) {
1871   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1872   InstructionMark im(this);
1873   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1874   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
1875   attributes.set_rex_vex_w_reverted();
1876   simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
1877   emit_int8(0x5E);
1878   emit_operand(dst, src);
1879 }
1880 
1881 void Assembler::divsd(XMMRegister dst, XMMRegister src) {
1882   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1883   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1884   attributes.set_rex_vex_w_reverted();
1885   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
1886   emit_int8(0x5E);
1887   emit_int8((unsigned char)(0xC0 | encode));
1888 }
1889 
1890 void Assembler::divss(XMMRegister dst, Address src) {
1891   NOT_LP64(assert(VM_Version::supports_sse(), ""));
1892   InstructionMark im(this);
1893   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1894   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
1895   simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
1896   emit_int8(0x5E);
1897   emit_operand(dst, src);
1898 }
1899 
1900 void Assembler::divss(XMMRegister dst, XMMRegister src) {
1901   NOT_LP64(assert(VM_Version::supports_sse(), ""));
1902   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1903   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
1904   emit_int8(0x5E);
1905   emit_int8((unsigned char)(0xC0 | encode));
1906 }
1907 
1908 void Assembler::emms() {
1909   NOT_LP64(assert(VM_Version::supports_mmx(), ""));
1910   emit_int8(0x0F);
1911   emit_int8(0x77);
1912 }
1913 
1914 void Assembler::hlt() {
1915   emit_int8((unsigned char)0xF4);
1916 }
1917 
1918 void Assembler::idivl(Register src) {
1919   int encode = prefix_and_encode(src->encoding());
1920   emit_int8((unsigned char)0xF7);
1921   emit_int8((unsigned char)(0xF8 | encode));
1922 }
1923 
1924 void Assembler::divl(Register src) { // Unsigned
1925   int encode = prefix_and_encode(src->encoding());
1926   emit_int8((unsigned char)0xF7);
1927   emit_int8((unsigned char)(0xF0 | encode));
1928 }
1929 
1930 void Assembler::imull(Register src) {
1931   int encode = prefix_and_encode(src->encoding());
1932   emit_int8((unsigned char)0xF7);
1933   emit_int8((unsigned char)(0xE8 | encode));
1934 }
1935 
1936 void Assembler::imull(Register dst, Register src) {
1937   int encode = prefix_and_encode(dst->encoding(), src->encoding());
1938   emit_int8(0x0F);
1939   emit_int8((unsigned char)0xAF);
1940   emit_int8((unsigned char)(0xC0 | encode));
1941 }
1942 
1943 
1944 void Assembler::imull(Register dst, Register src, int value) {
1945   int encode = prefix_and_encode(dst->encoding(), src->encoding());
1946   if (is8bit(value)) {
1947     emit_int8(0x6B);
1948     emit_int8((unsigned char)(0xC0 | encode));
1949     emit_int8(value & 0xFF);
1950   } else {
1951     emit_int8(0x69);
1952     emit_int8((unsigned char)(0xC0 | encode));
1953     emit_int32(value);
1954   }
1955 }
1956 
1957 void Assembler::imull(Register dst, Address src) {
1958   InstructionMark im(this);
1959   prefix(src, dst);
1960   emit_int8(0x0F);
1961   emit_int8((unsigned char) 0xAF);
1962   emit_operand(dst, src);
1963 }
1964 
1965 
1966 void Assembler::incl(Address dst) {
1967   // Don't use it directly. Use MacroAssembler::increment() instead.
1968   InstructionMark im(this);
1969   prefix(dst);
1970   emit_int8((unsigned char)0xFF);
1971   emit_operand(rax, dst);
1972 }
1973 
1974 void Assembler::jcc(Condition cc, Label& L, bool maybe_short) {
1975   InstructionMark im(this);
1976   assert((0 <= cc) && (cc < 16), "illegal cc");
1977   if (L.is_bound()) {
1978     address dst = target(L);
1979     assert(dst != NULL, "jcc most probably wrong");
1980 
1981     const int short_size = 2;
1982     const int long_size = 6;
1983     intptr_t offs = (intptr_t)dst - (intptr_t)pc();
1984     if (maybe_short && is8bit(offs - short_size)) {
1985       // 0111 tttn #8-bit disp
1986       emit_int8(0x70 | cc);
1987       emit_int8((offs - short_size) & 0xFF);
1988     } else {
1989       // 0000 1111 1000 tttn #32-bit disp
1990       assert(is_simm32(offs - long_size),
1991              "must be 32bit offset (call4)");
1992       emit_int8(0x0F);
1993       emit_int8((unsigned char)(0x80 | cc));
1994       emit_int32(offs - long_size);
1995     }
1996   } else {
1997     // Note: could eliminate cond. jumps to this jump if condition
1998     //       is the same however, seems to be rather unlikely case.
1999     // Note: use jccb() if label to be bound is very close to get
2000     //       an 8-bit displacement
2001     L.add_patch_at(code(), locator());
2002     emit_int8(0x0F);
2003     emit_int8((unsigned char)(0x80 | cc));
2004     emit_int32(0);
2005   }
2006 }
2007 
2008 void Assembler::jccb(Condition cc, Label& L) {
2009   if (L.is_bound()) {
2010     const int short_size = 2;
2011     address entry = target(L);
2012 #ifdef ASSERT
2013     intptr_t dist = (intptr_t)entry - ((intptr_t)pc() + short_size);
2014     intptr_t delta = short_branch_delta();
2015     if (delta != 0) {
2016       dist += (dist < 0 ? (-delta) :delta);
2017     }
2018     assert(is8bit(dist), "Dispacement too large for a short jmp");
2019 #endif
2020     intptr_t offs = (intptr_t)entry - (intptr_t)pc();
2021     // 0111 tttn #8-bit disp
2022     emit_int8(0x70 | cc);
2023     emit_int8((offs - short_size) & 0xFF);
2024   } else {
2025     InstructionMark im(this);
2026     L.add_patch_at(code(), locator());
2027     emit_int8(0x70 | cc);
2028     emit_int8(0);
2029   }
2030 }
2031 
2032 void Assembler::jmp(Address adr) {
2033   InstructionMark im(this);
2034   prefix(adr);
2035   emit_int8((unsigned char)0xFF);
2036   emit_operand(rsp, adr);
2037 }
2038 
2039 void Assembler::jmp(Label& L, bool maybe_short) {
2040   if (L.is_bound()) {
2041     address entry = target(L);
2042     assert(entry != NULL, "jmp most probably wrong");
2043     InstructionMark im(this);
2044     const int short_size = 2;
2045     const int long_size = 5;
2046     intptr_t offs = entry - pc();
2047     if (maybe_short && is8bit(offs - short_size)) {
2048       emit_int8((unsigned char)0xEB);
2049       emit_int8((offs - short_size) & 0xFF);
2050     } else {
2051       emit_int8((unsigned char)0xE9);
2052       emit_int32(offs - long_size);
2053     }
2054   } else {
2055     // By default, forward jumps are always 32-bit displacements, since
2056     // we can't yet know where the label will be bound.  If you're sure that
2057     // the forward jump will not run beyond 256 bytes, use jmpb to
2058     // force an 8-bit displacement.
2059     InstructionMark im(this);
2060     L.add_patch_at(code(), locator());
2061     emit_int8((unsigned char)0xE9);
2062     emit_int32(0);
2063   }
2064 }
2065 
2066 void Assembler::jmp(Register entry) {
2067   int encode = prefix_and_encode(entry->encoding());
2068   emit_int8((unsigned char)0xFF);
2069   emit_int8((unsigned char)(0xE0 | encode));
2070 }
2071 
2072 void Assembler::jmp_literal(address dest, RelocationHolder const& rspec) {
2073   InstructionMark im(this);
2074   emit_int8((unsigned char)0xE9);
2075   assert(dest != NULL, "must have a target");
2076   intptr_t disp = dest - (pc() + sizeof(int32_t));
2077   assert(is_simm32(disp), "must be 32bit offset (jmp)");
2078   emit_data(disp, rspec.reloc(), call32_operand);
2079 }
2080 
2081 void Assembler::jmpb(Label& L) {
2082   if (L.is_bound()) {
2083     const int short_size = 2;
2084     address entry = target(L);
2085     assert(entry != NULL, "jmp most probably wrong");
2086 #ifdef ASSERT
2087     intptr_t dist = (intptr_t)entry - ((intptr_t)pc() + short_size);
2088     intptr_t delta = short_branch_delta();
2089     if (delta != 0) {
2090       dist += (dist < 0 ? (-delta) :delta);
2091     }
2092     assert(is8bit(dist), "Dispacement too large for a short jmp");
2093 #endif
2094     intptr_t offs = entry - pc();
2095     emit_int8((unsigned char)0xEB);
2096     emit_int8((offs - short_size) & 0xFF);
2097   } else {
2098     InstructionMark im(this);
2099     L.add_patch_at(code(), locator());
2100     emit_int8((unsigned char)0xEB);
2101     emit_int8(0);
2102   }
2103 }
2104 
2105 void Assembler::ldmxcsr( Address src) {
2106   if (UseAVX > 0 ) {
2107     InstructionMark im(this);
2108     InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2109     vex_prefix(src, 0, 0, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2110     emit_int8((unsigned char)0xAE);
2111     emit_operand(as_Register(2), src);
2112   } else {
2113     NOT_LP64(assert(VM_Version::supports_sse(), ""));
2114     InstructionMark im(this);
2115     prefix(src);
2116     emit_int8(0x0F);
2117     emit_int8((unsigned char)0xAE);
2118     emit_operand(as_Register(2), src);
2119   }
2120 }
2121 
2122 void Assembler::leal(Register dst, Address src) {
2123   InstructionMark im(this);
2124 #ifdef _LP64
2125   emit_int8(0x67); // addr32
2126   prefix(src, dst);
2127 #endif // LP64
2128   emit_int8((unsigned char)0x8D);
2129   emit_operand(dst, src);
2130 }
2131 
2132 void Assembler::lfence() {
2133   emit_int8(0x0F);
2134   emit_int8((unsigned char)0xAE);
2135   emit_int8((unsigned char)0xE8);
2136 }
2137 
2138 void Assembler::lock() {
2139   emit_int8((unsigned char)0xF0);
2140 }
2141 
2142 void Assembler::lzcntl(Register dst, Register src) {
2143   assert(VM_Version::supports_lzcnt(), "encoding is treated as BSR");
2144   emit_int8((unsigned char)0xF3);
2145   int encode = prefix_and_encode(dst->encoding(), src->encoding());
2146   emit_int8(0x0F);
2147   emit_int8((unsigned char)0xBD);
2148   emit_int8((unsigned char)(0xC0 | encode));
2149 }
2150 
2151 // Emit mfence instruction
2152 void Assembler::mfence() {
2153   NOT_LP64(assert(VM_Version::supports_sse2(), "unsupported");)
2154   emit_int8(0x0F);
2155   emit_int8((unsigned char)0xAE);
2156   emit_int8((unsigned char)0xF0);
2157 }
2158 
2159 void Assembler::mov(Register dst, Register src) {
2160   LP64_ONLY(movq(dst, src)) NOT_LP64(movl(dst, src));
2161 }
2162 
2163 void Assembler::movapd(XMMRegister dst, XMMRegister src) {
2164   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2165   int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_128bit;
2166   InstructionAttr attributes(vector_len, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
2167   attributes.set_rex_vex_w_reverted();
2168   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2169   emit_int8(0x28);
2170   emit_int8((unsigned char)(0xC0 | encode));
2171 }
2172 
2173 void Assembler::movaps(XMMRegister dst, XMMRegister src) {
2174   NOT_LP64(assert(VM_Version::supports_sse(), ""));
2175   int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_128bit;
2176   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
2177   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2178   emit_int8(0x28);
2179   emit_int8((unsigned char)(0xC0 | encode));
2180 }
2181 
2182 void Assembler::movlhps(XMMRegister dst, XMMRegister src) {
2183   NOT_LP64(assert(VM_Version::supports_sse(), ""));
2184   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2185   int encode = simd_prefix_and_encode(dst, src, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2186   emit_int8(0x16);
2187   emit_int8((unsigned char)(0xC0 | encode));
2188 }
2189 
2190 void Assembler::movb(Register dst, Address src) {
2191   NOT_LP64(assert(dst->has_byte_register(), "must have byte register"));
2192   InstructionMark im(this);
2193   prefix(src, dst, true);
2194   emit_int8((unsigned char)0x8A);
2195   emit_operand(dst, src);
2196 }
2197 
2198 void Assembler::movddup(XMMRegister dst, XMMRegister src) {
2199   NOT_LP64(assert(VM_Version::supports_sse3(), ""));
2200   int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_128bit;
2201   InstructionAttr attributes(vector_len, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2202   attributes.set_rex_vex_w_reverted();
2203   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2204   emit_int8(0x12);
2205   emit_int8(0xC0 | encode);
2206 }
2207 
2208 void Assembler::kmovbl(KRegister dst, Register src) {
2209   assert(VM_Version::supports_avx512dq(), "");
2210   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2211   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2212   emit_int8((unsigned char)0x92);
2213   emit_int8((unsigned char)(0xC0 | encode));
2214 }
2215 
2216 void Assembler::kmovbl(Register dst, KRegister src) {
2217   assert(VM_Version::supports_avx512dq(), "");
2218   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2219   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2220   emit_int8((unsigned char)0x93);
2221   emit_int8((unsigned char)(0xC0 | encode));
2222 }
2223 
2224 void Assembler::kmovwl(KRegister dst, Register src) {
2225   assert(VM_Version::supports_evex(), "");
2226   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2227   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2228   emit_int8((unsigned char)0x92);
2229   emit_int8((unsigned char)(0xC0 | encode));
2230 }
2231 
2232 void Assembler::kmovwl(Register dst, KRegister src) {
2233   assert(VM_Version::supports_evex(), "");
2234   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2235   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2236   emit_int8((unsigned char)0x93);
2237   emit_int8((unsigned char)(0xC0 | encode));
2238 }
2239 
2240 void Assembler::kmovwl(KRegister dst, Address src) {
2241   assert(VM_Version::supports_evex(), "");
2242   InstructionMark im(this);
2243   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2244   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2245   emit_int8((unsigned char)0x90);
2246   emit_operand((Register)dst, src);
2247 }
2248 
2249 void Assembler::kmovdl(KRegister dst, Register src) {
2250   assert(VM_Version::supports_avx512bw(), "");
2251   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2252   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2253   emit_int8((unsigned char)0x92);
2254   emit_int8((unsigned char)(0xC0 | encode));
2255 }
2256 
2257 void Assembler::kmovdl(Register dst, KRegister src) {
2258   assert(VM_Version::supports_avx512bw(), "");
2259   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2260   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2261   emit_int8((unsigned char)0x93);
2262   emit_int8((unsigned char)(0xC0 | encode));
2263 }
2264 
2265 void Assembler::kmovql(KRegister dst, KRegister src) {
2266   assert(VM_Version::supports_avx512bw(), "");
2267   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2268   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2269   emit_int8((unsigned char)0x90);
2270   emit_int8((unsigned char)(0xC0 | encode));
2271 }
2272 
2273 void Assembler::kmovql(KRegister dst, Address src) {
2274   assert(VM_Version::supports_avx512bw(), "");
2275   InstructionMark im(this);
2276   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2277   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2278   emit_int8((unsigned char)0x90);
2279   emit_operand((Register)dst, src);
2280 }
2281 
2282 void Assembler::kmovql(Address dst, KRegister src) {
2283   assert(VM_Version::supports_avx512bw(), "");
2284   InstructionMark im(this);
2285   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2286   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2287   emit_int8((unsigned char)0x90);
2288   emit_operand((Register)src, dst);
2289 }
2290 
2291 void Assembler::kmovql(KRegister dst, Register src) {
2292   assert(VM_Version::supports_avx512bw(), "");
2293   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2294   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2295   emit_int8((unsigned char)0x92);
2296   emit_int8((unsigned char)(0xC0 | encode));
2297 }
2298 
2299 void Assembler::kmovql(Register dst, KRegister src) {
2300   assert(VM_Version::supports_avx512bw(), "");
2301   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2302   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2303   emit_int8((unsigned char)0x93);
2304   emit_int8((unsigned char)(0xC0 | encode));
2305 }
2306 
2307 void Assembler::knotwl(KRegister dst, KRegister src) {
2308   assert(VM_Version::supports_evex(), "");
2309   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2310   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2311   emit_int8((unsigned char)0x44);
2312   emit_int8((unsigned char)(0xC0 | encode));
2313 }
2314 
2315 // This instruction produces ZF or CF flags
2316 void Assembler::kortestbl(KRegister src1, KRegister src2) {
2317   assert(VM_Version::supports_avx512dq(), "");
2318   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2319   int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2320   emit_int8((unsigned char)0x98);
2321   emit_int8((unsigned char)(0xC0 | encode));
2322 }
2323 
2324 // This instruction produces ZF or CF flags
2325 void Assembler::kortestwl(KRegister src1, KRegister src2) {
2326   assert(VM_Version::supports_evex(), "");
2327   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2328   int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2329   emit_int8((unsigned char)0x98);
2330   emit_int8((unsigned char)(0xC0 | encode));
2331 }
2332 
2333 // This instruction produces ZF or CF flags
2334 void Assembler::kortestdl(KRegister src1, KRegister src2) {
2335   assert(VM_Version::supports_avx512bw(), "");
2336   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2337   int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2338   emit_int8((unsigned char)0x98);
2339   emit_int8((unsigned char)(0xC0 | encode));
2340 }
2341 
2342 // This instruction produces ZF or CF flags
2343 void Assembler::kortestql(KRegister src1, KRegister src2) {
2344   assert(VM_Version::supports_avx512bw(), "");
2345   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2346   int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2347   emit_int8((unsigned char)0x98);
2348   emit_int8((unsigned char)(0xC0 | encode));
2349 }
2350 
2351 // This instruction produces ZF or CF flags
2352 void Assembler::ktestql(KRegister src1, KRegister src2) {
2353   assert(VM_Version::supports_avx512bw(), "");
2354   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2355   int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2356   emit_int8((unsigned char)0x99);
2357   emit_int8((unsigned char)(0xC0 | encode));
2358 }
2359 
2360 void Assembler::ktestq(KRegister src1, KRegister src2) {
2361   assert(VM_Version::supports_avx512bw(), "");
2362   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2363   int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2364   emit_int8((unsigned char)0x99);
2365   emit_int8((unsigned char)(0xC0 | encode));
2366 }
2367 
2368 void Assembler::ktestd(KRegister src1, KRegister src2) {
2369   assert(VM_Version::supports_avx512bw(), "");
2370   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2371   int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2372   emit_int8((unsigned char)0x99);
2373   emit_int8((unsigned char)(0xC0 | encode));
2374 }
2375 
2376 void Assembler::movb(Address dst, int imm8) {
2377   InstructionMark im(this);
2378    prefix(dst);
2379   emit_int8((unsigned char)0xC6);
2380   emit_operand(rax, dst, 1);
2381   emit_int8(imm8);
2382 }
2383 
2384 
2385 void Assembler::movb(Address dst, Register src) {
2386   assert(src->has_byte_register(), "must have byte register");
2387   InstructionMark im(this);
2388   prefix(dst, src, true);
2389   emit_int8((unsigned char)0x88);
2390   emit_operand(src, dst);
2391 }
2392 
2393 void Assembler::movdl(XMMRegister dst, Register src) {
2394   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2395   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2396   int encode = simd_prefix_and_encode(dst, xnoreg, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2397   emit_int8(0x6E);
2398   emit_int8((unsigned char)(0xC0 | encode));
2399 }
2400 
2401 void Assembler::movdl(Register dst, XMMRegister src) {
2402   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2403   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2404   // swap src/dst to get correct prefix
2405   int encode = simd_prefix_and_encode(src, xnoreg, as_XMMRegister(dst->encoding()), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2406   emit_int8(0x7E);
2407   emit_int8((unsigned char)(0xC0 | encode));
2408 }
2409 
2410 void Assembler::movdl(XMMRegister dst, Address src) {
2411   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2412   InstructionMark im(this);
2413   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2414   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
2415   simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2416   emit_int8(0x6E);
2417   emit_operand(dst, src);
2418 }
2419 
2420 void Assembler::movdl(Address dst, XMMRegister src) {
2421   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2422   InstructionMark im(this);
2423   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2424   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
2425   simd_prefix(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2426   emit_int8(0x7E);
2427   emit_operand(src, dst);
2428 }
2429 
2430 void Assembler::movdqa(XMMRegister dst, XMMRegister src) {
2431   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2432   int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_128bit;
2433   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2434   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2435   emit_int8(0x6F);
2436   emit_int8((unsigned char)(0xC0 | encode));
2437 }
2438 
2439 void Assembler::movdqa(XMMRegister dst, Address src) {
2440   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2441   InstructionMark im(this);
2442   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2443   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2444   simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2445   emit_int8(0x6F);
2446   emit_operand(dst, src);
2447 }
2448 
2449 void Assembler::movdqu(XMMRegister dst, Address src) {
2450   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2451   InstructionMark im(this);
2452   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2453   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2454   simd_prefix(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2455   emit_int8(0x6F);
2456   emit_operand(dst, src);
2457 }
2458 
2459 void Assembler::movdqu(XMMRegister dst, XMMRegister src) {
2460   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2461   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2462   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2463   emit_int8(0x6F);
2464   emit_int8((unsigned char)(0xC0 | encode));
2465 }
2466 
2467 void Assembler::movdqu(Address dst, XMMRegister src) {
2468   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2469   InstructionMark im(this);
2470   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2471   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2472   attributes.reset_is_clear_context();
2473   simd_prefix(src, xnoreg, dst, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2474   emit_int8(0x7F);
2475   emit_operand(src, dst);
2476 }
2477 
2478 // Move Unaligned 256bit Vector
2479 void Assembler::vmovdqu(XMMRegister dst, XMMRegister src) {
2480   assert(UseAVX > 0, "");
2481   InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2482   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2483   emit_int8(0x6F);
2484   emit_int8((unsigned char)(0xC0 | encode));
2485 }
2486 
2487 void Assembler::vmovdqu(XMMRegister dst, Address src) {
2488   assert(UseAVX > 0, "");
2489   InstructionMark im(this);
2490   InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2491   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2492   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2493   emit_int8(0x6F);
2494   emit_operand(dst, src);
2495 }
2496 
2497 void Assembler::vmovdqu(Address dst, XMMRegister src) {
2498   assert(UseAVX > 0, "");
2499   InstructionMark im(this);
2500   InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2501   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2502   attributes.reset_is_clear_context();
2503   // swap src<->dst for encoding
2504   assert(src != xnoreg, "sanity");
2505   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2506   emit_int8(0x7F);
2507   emit_operand(src, dst);
2508 }
2509 
2510 // Move Unaligned EVEX enabled Vector (programmable : 8,16,32,64)
2511 void Assembler::evmovdqub(XMMRegister dst, XMMRegister src, int vector_len) {
2512   assert(VM_Version::supports_evex(), "");
2513   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
2514   attributes.set_is_evex_instruction();
2515   int prefix = (_legacy_mode_bw) ? VEX_SIMD_F2 : VEX_SIMD_F3;
2516   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), (Assembler::VexSimdPrefix)prefix, VEX_OPCODE_0F, &attributes);
2517   emit_int8(0x6F);
2518   emit_int8((unsigned char)(0xC0 | encode));
2519 }
2520 
2521 void Assembler::evmovdqub(XMMRegister dst, Address src, int vector_len) {
2522   assert(VM_Version::supports_evex(), "");
2523   InstructionMark im(this);
2524   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
2525   int prefix = (_legacy_mode_bw) ? VEX_SIMD_F2 : VEX_SIMD_F3;
2526   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2527   attributes.set_is_evex_instruction();
2528   vex_prefix(src, 0, dst->encoding(), (Assembler::VexSimdPrefix)prefix, VEX_OPCODE_0F, &attributes);
2529   emit_int8(0x6F);
2530   emit_operand(dst, src);
2531 }
2532 
2533 void Assembler::evmovdqub(Address dst, XMMRegister src, int vector_len) {
2534   assert(VM_Version::supports_evex(), "");
2535   assert(src != xnoreg, "sanity");
2536   InstructionMark im(this);
2537   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
2538   int prefix = (_legacy_mode_bw) ? VEX_SIMD_F2 : VEX_SIMD_F3;
2539   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2540   attributes.set_is_evex_instruction();
2541   vex_prefix(dst, 0, src->encoding(), (Assembler::VexSimdPrefix)prefix, VEX_OPCODE_0F, &attributes);
2542   emit_int8(0x7F);
2543   emit_operand(src, dst);
2544 }
2545 
2546 void Assembler::evmovdqub(XMMRegister dst, KRegister mask, Address src, int vector_len) {
2547   assert(VM_Version::supports_avx512vlbw(), "");
2548   assert(is_vector_masking(), "");    // For stub code use only
2549   InstructionMark im(this);
2550   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
2551   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2552   attributes.set_embedded_opmask_register_specifier(mask);
2553   attributes.set_is_evex_instruction();
2554   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2555   emit_int8(0x6F);
2556   emit_operand(dst, src);
2557 }
2558 
2559 void Assembler::evmovdquw(XMMRegister dst, Address src, int vector_len) {
2560   assert(VM_Version::supports_evex(), "");
2561   InstructionMark im(this);
2562   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
2563   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2564   attributes.set_is_evex_instruction();
2565   int prefix = (_legacy_mode_bw) ? VEX_SIMD_F2 : VEX_SIMD_F3;
2566   vex_prefix(src, 0, dst->encoding(), (Assembler::VexSimdPrefix)prefix, VEX_OPCODE_0F, &attributes);
2567   emit_int8(0x6F);
2568   emit_operand(dst, src);
2569 }
2570 
2571 void Assembler::evmovdquw(XMMRegister dst, KRegister mask, Address src, int vector_len) {
2572   assert(is_vector_masking(), "");
2573   assert(VM_Version::supports_avx512vlbw(), "");
2574   InstructionMark im(this);
2575   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
2576   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2577   attributes.set_embedded_opmask_register_specifier(mask);
2578   attributes.set_is_evex_instruction();
2579   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2580   emit_int8(0x6F);
2581   emit_operand(dst, src);
2582 }
2583 
2584 void Assembler::evmovdquw(Address dst, XMMRegister src, int vector_len) {
2585   assert(VM_Version::supports_evex(), "");
2586   assert(src != xnoreg, "sanity");
2587   InstructionMark im(this);
2588   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
2589   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2590   attributes.set_is_evex_instruction();
2591   int prefix = (_legacy_mode_bw) ? VEX_SIMD_F2 : VEX_SIMD_F3;
2592   vex_prefix(dst, 0, src->encoding(), (Assembler::VexSimdPrefix)prefix, VEX_OPCODE_0F, &attributes);
2593   emit_int8(0x7F);
2594   emit_operand(src, dst);
2595 }
2596 
2597 void Assembler::evmovdquw(Address dst, KRegister mask, XMMRegister src, int vector_len) {
2598   assert(VM_Version::supports_avx512vlbw(), "");
2599   assert(src != xnoreg, "sanity");
2600   InstructionMark im(this);
2601   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2602   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2603   attributes.reset_is_clear_context();
2604   attributes.set_embedded_opmask_register_specifier(mask);
2605   attributes.set_is_evex_instruction();
2606   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2607   emit_int8(0x7F);
2608   emit_operand(src, dst);
2609 }
2610 
2611 void Assembler::evmovdqul(XMMRegister dst, XMMRegister src, int vector_len) {
2612   assert(VM_Version::supports_evex(), "");
2613   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2614   attributes.set_is_evex_instruction();
2615   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2616   emit_int8(0x6F);
2617   emit_int8((unsigned char)(0xC0 | encode));
2618 }
2619 
2620 void Assembler::evmovdqul(XMMRegister dst, Address src, int vector_len) {
2621   assert(VM_Version::supports_evex(), "");
2622   InstructionMark im(this);
2623   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false , /* uses_vl */ true);
2624   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2625   attributes.set_is_evex_instruction();
2626   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2627   emit_int8(0x6F);
2628   emit_operand(dst, src);
2629 }
2630 
2631 void Assembler::evmovdqul(Address dst, XMMRegister src, int vector_len) {
2632   assert(VM_Version::supports_evex(), "");
2633   assert(src != xnoreg, "sanity");
2634   InstructionMark im(this);
2635   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2636   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2637   attributes.reset_is_clear_context();
2638   attributes.set_is_evex_instruction();
2639   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2640   emit_int8(0x7F);
2641   emit_operand(src, dst);
2642 }
2643 
2644 void Assembler::evmovdquq(XMMRegister dst, XMMRegister src, int vector_len) {
2645   assert(VM_Version::supports_evex(), "");
2646   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2647   attributes.set_is_evex_instruction();
2648   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2649   emit_int8(0x6F);
2650   emit_int8((unsigned char)(0xC0 | encode));
2651 }
2652 
2653 void Assembler::evmovdquq(XMMRegister dst, Address src, int vector_len) {
2654   assert(VM_Version::supports_evex(), "");
2655   InstructionMark im(this);
2656   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2657   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2658   attributes.set_is_evex_instruction();
2659   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2660   emit_int8(0x6F);
2661   emit_operand(dst, src);
2662 }
2663 
2664 void Assembler::evmovdquq(Address dst, XMMRegister src, int vector_len) {
2665   assert(VM_Version::supports_evex(), "");
2666   assert(src != xnoreg, "sanity");
2667   InstructionMark im(this);
2668   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2669   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2670   attributes.reset_is_clear_context();
2671   attributes.set_is_evex_instruction();
2672   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2673   emit_int8(0x7F);
2674   emit_operand(src, dst);
2675 }
2676 
2677 // Uses zero extension on 64bit
2678 
2679 void Assembler::movl(Register dst, int32_t imm32) {
2680   int encode = prefix_and_encode(dst->encoding());
2681   emit_int8((unsigned char)(0xB8 | encode));
2682   emit_int32(imm32);
2683 }
2684 
2685 void Assembler::movl(Register dst, Register src) {
2686   int encode = prefix_and_encode(dst->encoding(), src->encoding());
2687   emit_int8((unsigned char)0x8B);
2688   emit_int8((unsigned char)(0xC0 | encode));
2689 }
2690 
2691 void Assembler::movl(Register dst, Address src) {
2692   InstructionMark im(this);
2693   prefix(src, dst);
2694   emit_int8((unsigned char)0x8B);
2695   emit_operand(dst, src);
2696 }
2697 
2698 void Assembler::movl(Address dst, int32_t imm32) {
2699   InstructionMark im(this);
2700   prefix(dst);
2701   emit_int8((unsigned char)0xC7);
2702   emit_operand(rax, dst, 4);
2703   emit_int32(imm32);
2704 }
2705 
2706 void Assembler::movl(Address dst, Register src) {
2707   InstructionMark im(this);
2708   prefix(dst, src);
2709   emit_int8((unsigned char)0x89);
2710   emit_operand(src, dst);
2711 }
2712 
2713 // New cpus require to use movsd and movss to avoid partial register stall
2714 // when loading from memory. But for old Opteron use movlpd instead of movsd.
2715 // The selection is done in MacroAssembler::movdbl() and movflt().
2716 void Assembler::movlpd(XMMRegister dst, Address src) {
2717   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2718   InstructionMark im(this);
2719   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2720   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
2721   attributes.set_rex_vex_w_reverted();
2722   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2723   emit_int8(0x12);
2724   emit_operand(dst, src);
2725 }
2726 
2727 void Assembler::movq( MMXRegister dst, Address src ) {
2728   assert( VM_Version::supports_mmx(), "" );
2729   emit_int8(0x0F);
2730   emit_int8(0x6F);
2731   emit_operand(dst, src);
2732 }
2733 
2734 void Assembler::movq( Address dst, MMXRegister src ) {
2735   assert( VM_Version::supports_mmx(), "" );
2736   emit_int8(0x0F);
2737   emit_int8(0x7F);
2738   // workaround gcc (3.2.1-7a) bug
2739   // In that version of gcc with only an emit_operand(MMX, Address)
2740   // gcc will tail jump and try and reverse the parameters completely
2741   // obliterating dst in the process. By having a version available
2742   // that doesn't need to swap the args at the tail jump the bug is
2743   // avoided.
2744   emit_operand(dst, src);
2745 }
2746 
2747 void Assembler::movq(XMMRegister dst, Address src) {
2748   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2749   InstructionMark im(this);
2750   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2751   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
2752   attributes.set_rex_vex_w_reverted();
2753   simd_prefix(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2754   emit_int8(0x7E);
2755   emit_operand(dst, src);
2756 }
2757 
2758 void Assembler::movq(Address dst, XMMRegister src) {
2759   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2760   InstructionMark im(this);
2761   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2762   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
2763   attributes.set_rex_vex_w_reverted();
2764   simd_prefix(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2765   emit_int8((unsigned char)0xD6);
2766   emit_operand(src, dst);
2767 }
2768 
2769 void Assembler::movsbl(Register dst, Address src) { // movsxb
2770   InstructionMark im(this);
2771   prefix(src, dst);
2772   emit_int8(0x0F);
2773   emit_int8((unsigned char)0xBE);
2774   emit_operand(dst, src);
2775 }
2776 
2777 void Assembler::movsbl(Register dst, Register src) { // movsxb
2778   NOT_LP64(assert(src->has_byte_register(), "must have byte register"));
2779   int encode = prefix_and_encode(dst->encoding(), false, src->encoding(), true);
2780   emit_int8(0x0F);
2781   emit_int8((unsigned char)0xBE);
2782   emit_int8((unsigned char)(0xC0 | encode));
2783 }
2784 
2785 void Assembler::movsd(XMMRegister dst, XMMRegister src) {
2786   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2787   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2788   attributes.set_rex_vex_w_reverted();
2789   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2790   emit_int8(0x10);
2791   emit_int8((unsigned char)(0xC0 | encode));
2792 }
2793 
2794 void Assembler::movsd(XMMRegister dst, Address src) {
2795   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2796   InstructionMark im(this);
2797   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2798   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
2799   attributes.set_rex_vex_w_reverted();
2800   simd_prefix(dst, xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2801   emit_int8(0x10);
2802   emit_operand(dst, src);
2803 }
2804 
2805 void Assembler::movsd(Address dst, XMMRegister src) {
2806   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2807   InstructionMark im(this);
2808   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2809   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
2810   attributes.reset_is_clear_context();
2811   attributes.set_rex_vex_w_reverted();
2812   simd_prefix(src, xnoreg, dst, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2813   emit_int8(0x11);
2814   emit_operand(src, dst);
2815 }
2816 
2817 void Assembler::movss(XMMRegister dst, XMMRegister src) {
2818   NOT_LP64(assert(VM_Version::supports_sse(), ""));
2819   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2820   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2821   emit_int8(0x10);
2822   emit_int8((unsigned char)(0xC0 | encode));
2823 }
2824 
2825 void Assembler::movss(XMMRegister dst, Address src) {
2826   NOT_LP64(assert(VM_Version::supports_sse(), ""));
2827   InstructionMark im(this);
2828   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2829   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
2830   simd_prefix(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2831   emit_int8(0x10);
2832   emit_operand(dst, src);
2833 }
2834 
2835 void Assembler::movss(Address dst, XMMRegister src) {
2836   NOT_LP64(assert(VM_Version::supports_sse(), ""));
2837   InstructionMark im(this);
2838   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2839   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
2840   attributes.reset_is_clear_context();
2841   simd_prefix(src, xnoreg, dst, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2842   emit_int8(0x11);
2843   emit_operand(src, dst);
2844 }
2845 
2846 void Assembler::movswl(Register dst, Address src) { // movsxw
2847   InstructionMark im(this);
2848   prefix(src, dst);
2849   emit_int8(0x0F);
2850   emit_int8((unsigned char)0xBF);
2851   emit_operand(dst, src);
2852 }
2853 
2854 void Assembler::movswl(Register dst, Register src) { // movsxw
2855   int encode = prefix_and_encode(dst->encoding(), src->encoding());
2856   emit_int8(0x0F);
2857   emit_int8((unsigned char)0xBF);
2858   emit_int8((unsigned char)(0xC0 | encode));
2859 }
2860 
2861 void Assembler::movw(Address dst, int imm16) {
2862   InstructionMark im(this);
2863 
2864   emit_int8(0x66); // switch to 16-bit mode
2865   prefix(dst);
2866   emit_int8((unsigned char)0xC7);
2867   emit_operand(rax, dst, 2);
2868   emit_int16(imm16);
2869 }
2870 
2871 void Assembler::movw(Register dst, Address src) {
2872   InstructionMark im(this);
2873   emit_int8(0x66);
2874   prefix(src, dst);
2875   emit_int8((unsigned char)0x8B);
2876   emit_operand(dst, src);
2877 }
2878 
2879 void Assembler::movw(Address dst, Register src) {
2880   InstructionMark im(this);
2881   emit_int8(0x66);
2882   prefix(dst, src);
2883   emit_int8((unsigned char)0x89);
2884   emit_operand(src, dst);
2885 }
2886 
2887 void Assembler::movzbl(Register dst, Address src) { // movzxb
2888   InstructionMark im(this);
2889   prefix(src, dst);
2890   emit_int8(0x0F);
2891   emit_int8((unsigned char)0xB6);
2892   emit_operand(dst, src);
2893 }
2894 
2895 void Assembler::movzbl(Register dst, Register src) { // movzxb
2896   NOT_LP64(assert(src->has_byte_register(), "must have byte register"));
2897   int encode = prefix_and_encode(dst->encoding(), false, src->encoding(), true);
2898   emit_int8(0x0F);
2899   emit_int8((unsigned char)0xB6);
2900   emit_int8(0xC0 | encode);
2901 }
2902 
2903 void Assembler::movzwl(Register dst, Address src) { // movzxw
2904   InstructionMark im(this);
2905   prefix(src, dst);
2906   emit_int8(0x0F);
2907   emit_int8((unsigned char)0xB7);
2908   emit_operand(dst, src);
2909 }
2910 
2911 void Assembler::movzwl(Register dst, Register src) { // movzxw
2912   int encode = prefix_and_encode(dst->encoding(), src->encoding());
2913   emit_int8(0x0F);
2914   emit_int8((unsigned char)0xB7);
2915   emit_int8(0xC0 | encode);
2916 }
2917 
2918 void Assembler::mull(Address src) {
2919   InstructionMark im(this);
2920   prefix(src);
2921   emit_int8((unsigned char)0xF7);
2922   emit_operand(rsp, src);
2923 }
2924 
2925 void Assembler::mull(Register src) {
2926   int encode = prefix_and_encode(src->encoding());
2927   emit_int8((unsigned char)0xF7);
2928   emit_int8((unsigned char)(0xE0 | encode));
2929 }
2930 
2931 void Assembler::mulsd(XMMRegister dst, Address src) {
2932   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2933   InstructionMark im(this);
2934   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2935   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
2936   attributes.set_rex_vex_w_reverted();
2937   simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2938   emit_int8(0x59);
2939   emit_operand(dst, src);
2940 }
2941 
2942 void Assembler::mulsd(XMMRegister dst, XMMRegister src) {
2943   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2944   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2945   attributes.set_rex_vex_w_reverted();
2946   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2947   emit_int8(0x59);
2948   emit_int8((unsigned char)(0xC0 | encode));
2949 }
2950 
2951 void Assembler::mulss(XMMRegister dst, Address src) {
2952   NOT_LP64(assert(VM_Version::supports_sse(), ""));
2953   InstructionMark im(this);
2954   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2955   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
2956   simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2957   emit_int8(0x59);
2958   emit_operand(dst, src);
2959 }
2960 
2961 void Assembler::mulss(XMMRegister dst, XMMRegister src) {
2962   NOT_LP64(assert(VM_Version::supports_sse(), ""));
2963   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2964   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2965   emit_int8(0x59);
2966   emit_int8((unsigned char)(0xC0 | encode));
2967 }
2968 
2969 void Assembler::negl(Register dst) {
2970   int encode = prefix_and_encode(dst->encoding());
2971   emit_int8((unsigned char)0xF7);
2972   emit_int8((unsigned char)(0xD8 | encode));
2973 }
2974 
2975 void Assembler::nop(int i) {
2976 #ifdef ASSERT
2977   assert(i > 0, " ");
2978   // The fancy nops aren't currently recognized by debuggers making it a
2979   // pain to disassemble code while debugging. If asserts are on clearly
2980   // speed is not an issue so simply use the single byte traditional nop
2981   // to do alignment.
2982 
2983   for (; i > 0 ; i--) emit_int8((unsigned char)0x90);
2984   return;
2985 
2986 #endif // ASSERT
2987 
2988   if (UseAddressNop && VM_Version::is_intel()) {
2989     //
2990     // Using multi-bytes nops "0x0F 0x1F [address]" for Intel
2991     //  1: 0x90
2992     //  2: 0x66 0x90
2993     //  3: 0x66 0x66 0x90 (don't use "0x0F 0x1F 0x00" - need patching safe padding)
2994     //  4: 0x0F 0x1F 0x40 0x00
2995     //  5: 0x0F 0x1F 0x44 0x00 0x00
2996     //  6: 0x66 0x0F 0x1F 0x44 0x00 0x00
2997     //  7: 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00
2998     //  8: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
2999     //  9: 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
3000     // 10: 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
3001     // 11: 0x66 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
3002 
3003     // The rest coding is Intel specific - don't use consecutive address nops
3004 
3005     // 12: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
3006     // 13: 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
3007     // 14: 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
3008     // 15: 0x66 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
3009 
3010     while(i >= 15) {
3011       // For Intel don't generate consecutive addess nops (mix with regular nops)
3012       i -= 15;
3013       emit_int8(0x66);   // size prefix
3014       emit_int8(0x66);   // size prefix
3015       emit_int8(0x66);   // size prefix
3016       addr_nop_8();
3017       emit_int8(0x66);   // size prefix
3018       emit_int8(0x66);   // size prefix
3019       emit_int8(0x66);   // size prefix
3020       emit_int8((unsigned char)0x90);
3021                          // nop
3022     }
3023     switch (i) {
3024       case 14:
3025         emit_int8(0x66); // size prefix
3026       case 13:
3027         emit_int8(0x66); // size prefix
3028       case 12:
3029         addr_nop_8();
3030         emit_int8(0x66); // size prefix
3031         emit_int8(0x66); // size prefix
3032         emit_int8(0x66); // size prefix
3033         emit_int8((unsigned char)0x90);
3034                          // nop
3035         break;
3036       case 11:
3037         emit_int8(0x66); // size prefix
3038       case 10:
3039         emit_int8(0x66); // size prefix
3040       case 9:
3041         emit_int8(0x66); // size prefix
3042       case 8:
3043         addr_nop_8();
3044         break;
3045       case 7:
3046         addr_nop_7();
3047         break;
3048       case 6:
3049         emit_int8(0x66); // size prefix
3050       case 5:
3051         addr_nop_5();
3052         break;
3053       case 4:
3054         addr_nop_4();
3055         break;
3056       case 3:
3057         // Don't use "0x0F 0x1F 0x00" - need patching safe padding
3058         emit_int8(0x66); // size prefix
3059       case 2:
3060         emit_int8(0x66); // size prefix
3061       case 1:
3062         emit_int8((unsigned char)0x90);
3063                          // nop
3064         break;
3065       default:
3066         assert(i == 0, " ");
3067     }
3068     return;
3069   }
3070   if (UseAddressNop && VM_Version::is_amd()) {
3071     //
3072     // Using multi-bytes nops "0x0F 0x1F [address]" for AMD.
3073     //  1: 0x90
3074     //  2: 0x66 0x90
3075     //  3: 0x66 0x66 0x90 (don't use "0x0F 0x1F 0x00" - need patching safe padding)
3076     //  4: 0x0F 0x1F 0x40 0x00
3077     //  5: 0x0F 0x1F 0x44 0x00 0x00
3078     //  6: 0x66 0x0F 0x1F 0x44 0x00 0x00
3079     //  7: 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00
3080     //  8: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
3081     //  9: 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
3082     // 10: 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
3083     // 11: 0x66 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
3084 
3085     // The rest coding is AMD specific - use consecutive address nops
3086 
3087     // 12: 0x66 0x0F 0x1F 0x44 0x00 0x00 0x66 0x0F 0x1F 0x44 0x00 0x00
3088     // 13: 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00 0x66 0x0F 0x1F 0x44 0x00 0x00
3089     // 14: 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00
3090     // 15: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00
3091     // 16: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
3092     //     Size prefixes (0x66) are added for larger sizes
3093 
3094     while(i >= 22) {
3095       i -= 11;
3096       emit_int8(0x66); // size prefix
3097       emit_int8(0x66); // size prefix
3098       emit_int8(0x66); // size prefix
3099       addr_nop_8();
3100     }
3101     // Generate first nop for size between 21-12
3102     switch (i) {
3103       case 21:
3104         i -= 1;
3105         emit_int8(0x66); // size prefix
3106       case 20:
3107       case 19:
3108         i -= 1;
3109         emit_int8(0x66); // size prefix
3110       case 18:
3111       case 17:
3112         i -= 1;
3113         emit_int8(0x66); // size prefix
3114       case 16:
3115       case 15:
3116         i -= 8;
3117         addr_nop_8();
3118         break;
3119       case 14:
3120       case 13:
3121         i -= 7;
3122         addr_nop_7();
3123         break;
3124       case 12:
3125         i -= 6;
3126         emit_int8(0x66); // size prefix
3127         addr_nop_5();
3128         break;
3129       default:
3130         assert(i < 12, " ");
3131     }
3132 
3133     // Generate second nop for size between 11-1
3134     switch (i) {
3135       case 11:
3136         emit_int8(0x66); // size prefix
3137       case 10:
3138         emit_int8(0x66); // size prefix
3139       case 9:
3140         emit_int8(0x66); // size prefix
3141       case 8:
3142         addr_nop_8();
3143         break;
3144       case 7:
3145         addr_nop_7();
3146         break;
3147       case 6:
3148         emit_int8(0x66); // size prefix
3149       case 5:
3150         addr_nop_5();
3151         break;
3152       case 4:
3153         addr_nop_4();
3154         break;
3155       case 3:
3156         // Don't use "0x0F 0x1F 0x00" - need patching safe padding
3157         emit_int8(0x66); // size prefix
3158       case 2:
3159         emit_int8(0x66); // size prefix
3160       case 1:
3161         emit_int8((unsigned char)0x90);
3162                          // nop
3163         break;
3164       default:
3165         assert(i == 0, " ");
3166     }
3167     return;
3168   }
3169 
3170   if (UseAddressNop && VM_Version::is_zx()) {
3171     //
3172     // Using multi-bytes nops "0x0F 0x1F [address]" for ZX
3173     //  1: 0x90
3174     //  2: 0x66 0x90
3175     //  3: 0x66 0x66 0x90 (don't use "0x0F 0x1F 0x00" - need patching safe padding)
3176     //  4: 0x0F 0x1F 0x40 0x00
3177     //  5: 0x0F 0x1F 0x44 0x00 0x00
3178     //  6: 0x66 0x0F 0x1F 0x44 0x00 0x00
3179     //  7: 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00
3180     //  8: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
3181     //  9: 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
3182     // 10: 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
3183     // 11: 0x66 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
3184 
3185     // The rest coding is ZX specific - don't use consecutive address nops
3186 
3187     // 12: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
3188     // 13: 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
3189     // 14: 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
3190     // 15: 0x66 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
3191 
3192     while (i >= 15) {
3193       // For ZX don't generate consecutive addess nops (mix with regular nops)
3194       i -= 15;
3195       emit_int8(0x66);   // size prefix
3196       emit_int8(0x66);   // size prefix
3197       emit_int8(0x66);   // size prefix
3198       addr_nop_8();
3199       emit_int8(0x66);   // size prefix
3200       emit_int8(0x66);   // size prefix
3201       emit_int8(0x66);   // size prefix
3202       emit_int8((unsigned char)0x90);
3203                          // nop
3204     }
3205     switch (i) {
3206       case 14:
3207         emit_int8(0x66); // size prefix
3208       case 13:
3209         emit_int8(0x66); // size prefix
3210       case 12:
3211         addr_nop_8();
3212         emit_int8(0x66); // size prefix
3213         emit_int8(0x66); // size prefix
3214         emit_int8(0x66); // size prefix
3215         emit_int8((unsigned char)0x90);
3216                          // nop
3217         break;
3218       case 11:
3219         emit_int8(0x66); // size prefix
3220       case 10:
3221         emit_int8(0x66); // size prefix
3222       case 9:
3223         emit_int8(0x66); // size prefix
3224       case 8:
3225         addr_nop_8();
3226         break;
3227       case 7:
3228         addr_nop_7();
3229         break;
3230       case 6:
3231         emit_int8(0x66); // size prefix
3232       case 5:
3233         addr_nop_5();
3234         break;
3235       case 4:
3236         addr_nop_4();
3237         break;
3238       case 3:
3239         // Don't use "0x0F 0x1F 0x00" - need patching safe padding
3240         emit_int8(0x66); // size prefix
3241       case 2:
3242         emit_int8(0x66); // size prefix
3243       case 1:
3244         emit_int8((unsigned char)0x90);
3245                          // nop
3246         break;
3247       default:
3248         assert(i == 0, " ");
3249     }
3250     return;
3251   }
3252 
3253   // Using nops with size prefixes "0x66 0x90".
3254   // From AMD Optimization Guide:
3255   //  1: 0x90
3256   //  2: 0x66 0x90
3257   //  3: 0x66 0x66 0x90
3258   //  4: 0x66 0x66 0x66 0x90
3259   //  5: 0x66 0x66 0x90 0x66 0x90
3260   //  6: 0x66 0x66 0x90 0x66 0x66 0x90
3261   //  7: 0x66 0x66 0x66 0x90 0x66 0x66 0x90
3262   //  8: 0x66 0x66 0x66 0x90 0x66 0x66 0x66 0x90
3263   //  9: 0x66 0x66 0x90 0x66 0x66 0x90 0x66 0x66 0x90
3264   // 10: 0x66 0x66 0x66 0x90 0x66 0x66 0x90 0x66 0x66 0x90
3265   //
3266   while(i > 12) {
3267     i -= 4;
3268     emit_int8(0x66); // size prefix
3269     emit_int8(0x66);
3270     emit_int8(0x66);
3271     emit_int8((unsigned char)0x90);
3272                      // nop
3273   }
3274   // 1 - 12 nops
3275   if(i > 8) {
3276     if(i > 9) {
3277       i -= 1;
3278       emit_int8(0x66);
3279     }
3280     i -= 3;
3281     emit_int8(0x66);
3282     emit_int8(0x66);
3283     emit_int8((unsigned char)0x90);
3284   }
3285   // 1 - 8 nops
3286   if(i > 4) {
3287     if(i > 6) {
3288       i -= 1;
3289       emit_int8(0x66);
3290     }
3291     i -= 3;
3292     emit_int8(0x66);
3293     emit_int8(0x66);
3294     emit_int8((unsigned char)0x90);
3295   }
3296   switch (i) {
3297     case 4:
3298       emit_int8(0x66);
3299     case 3:
3300       emit_int8(0x66);
3301     case 2:
3302       emit_int8(0x66);
3303     case 1:
3304       emit_int8((unsigned char)0x90);
3305       break;
3306     default:
3307       assert(i == 0, " ");
3308   }
3309 }
3310 
3311 void Assembler::notl(Register dst) {
3312   int encode = prefix_and_encode(dst->encoding());
3313   emit_int8((unsigned char)0xF7);
3314   emit_int8((unsigned char)(0xD0 | encode));
3315 }
3316 
3317 void Assembler::orl(Address dst, int32_t imm32) {
3318   InstructionMark im(this);
3319   prefix(dst);
3320   emit_arith_operand(0x81, rcx, dst, imm32);
3321 }
3322 
3323 void Assembler::orl(Register dst, int32_t imm32) {
3324   prefix(dst);
3325   emit_arith(0x81, 0xC8, dst, imm32);
3326 }
3327 
3328 void Assembler::orl(Register dst, Address src) {
3329   InstructionMark im(this);
3330   prefix(src, dst);
3331   emit_int8(0x0B);
3332   emit_operand(dst, src);
3333 }
3334 
3335 void Assembler::orl(Register dst, Register src) {
3336   (void) prefix_and_encode(dst->encoding(), src->encoding());
3337   emit_arith(0x0B, 0xC0, dst, src);
3338 }
3339 
3340 void Assembler::orl(Address dst, Register src) {
3341   InstructionMark im(this);
3342   prefix(dst, src);
3343   emit_int8(0x09);
3344   emit_operand(src, dst);
3345 }
3346 
3347 void Assembler::packuswb(XMMRegister dst, Address src) {
3348   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3349   assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
3350   InstructionMark im(this);
3351   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
3352   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
3353   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3354   emit_int8(0x67);
3355   emit_operand(dst, src);
3356 }
3357 
3358 void Assembler::packuswb(XMMRegister dst, XMMRegister src) {
3359   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3360   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
3361   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3362   emit_int8(0x67);
3363   emit_int8((unsigned char)(0xC0 | encode));
3364 }
3365 
3366 void Assembler::vpackuswb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
3367   assert(UseAVX > 0, "some form of AVX must be enabled");
3368   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
3369   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3370   emit_int8(0x67);
3371   emit_int8((unsigned char)(0xC0 | encode));
3372 }
3373 
3374 void Assembler::vpermq(XMMRegister dst, XMMRegister src, int imm8, int vector_len) {
3375   assert(VM_Version::supports_avx2(), "");
3376   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
3377   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3378   emit_int8(0x00);
3379   emit_int8(0xC0 | encode);
3380   emit_int8(imm8);
3381 }
3382 
3383 void Assembler::vperm2i128(XMMRegister dst,  XMMRegister nds, XMMRegister src, int imm8) {
3384   assert(VM_Version::supports_avx2(), "");
3385   InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3386   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3387   emit_int8(0x46);
3388   emit_int8(0xC0 | encode);
3389   emit_int8(imm8);
3390 }
3391 
3392 void Assembler::vperm2f128(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8) {
3393   assert(VM_Version::supports_avx(), "");
3394   InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3395   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3396   emit_int8(0x06);
3397   emit_int8(0xC0 | encode);
3398   emit_int8(imm8);
3399 }
3400 
3401 
3402 void Assembler::pause() {
3403   emit_int8((unsigned char)0xF3);
3404   emit_int8((unsigned char)0x90);
3405 }
3406 
3407 void Assembler::ud2() {
3408   emit_int8(0x0F);
3409   emit_int8(0x0B);
3410 }
3411 
3412 void Assembler::pcmpestri(XMMRegister dst, Address src, int imm8) {
3413   assert(VM_Version::supports_sse4_2(), "");
3414   InstructionMark im(this);
3415   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3416   simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3417   emit_int8(0x61);
3418   emit_operand(dst, src);
3419   emit_int8(imm8);
3420 }
3421 
3422 void Assembler::pcmpestri(XMMRegister dst, XMMRegister src, int imm8) {
3423   assert(VM_Version::supports_sse4_2(), "");
3424   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3425   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3426   emit_int8(0x61);
3427   emit_int8((unsigned char)(0xC0 | encode));
3428   emit_int8(imm8);
3429 }
3430 
3431 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
3432 void Assembler::pcmpeqb(XMMRegister dst, XMMRegister src) {
3433   assert(VM_Version::supports_sse2(), "");
3434   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3435   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3436   emit_int8(0x74);
3437   emit_int8((unsigned char)(0xC0 | encode));
3438 }
3439 
3440 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
3441 void Assembler::vpcmpeqb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
3442   assert(VM_Version::supports_avx(), "");
3443   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3444   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3445   emit_int8(0x74);
3446   emit_int8((unsigned char)(0xC0 | encode));
3447 }
3448 
3449 // In this context, kdst is written the mask used to process the equal components
3450 void Assembler::evpcmpeqb(KRegister kdst, XMMRegister nds, XMMRegister src, int vector_len) {
3451   assert(VM_Version::supports_avx512bw(), "");
3452   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
3453   attributes.set_is_evex_instruction();
3454   int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3455   emit_int8(0x74);
3456   emit_int8((unsigned char)(0xC0 | encode));
3457 }
3458 
3459 void Assembler::evpcmpgtb(KRegister kdst, XMMRegister nds, Address src, int vector_len) {
3460   assert(VM_Version::supports_avx512vlbw(), "");
3461   InstructionMark im(this);
3462   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3463   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
3464   attributes.set_is_evex_instruction();
3465   int dst_enc = kdst->encoding();
3466   vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3467   emit_int8(0x64);
3468   emit_operand(as_Register(dst_enc), src);
3469 }
3470 
3471 void Assembler::evpcmpgtb(KRegister kdst, KRegister mask, XMMRegister nds, Address src, int vector_len) {
3472   assert(is_vector_masking(), "");
3473   assert(VM_Version::supports_avx512vlbw(), "");
3474   InstructionMark im(this);
3475   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
3476   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
3477   attributes.reset_is_clear_context();
3478   attributes.set_embedded_opmask_register_specifier(mask);
3479   attributes.set_is_evex_instruction();
3480   int dst_enc = kdst->encoding();
3481   vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3482   emit_int8(0x64);
3483   emit_operand(as_Register(dst_enc), src);
3484 }
3485 
3486 void Assembler::evpcmpuw(KRegister kdst, XMMRegister nds, XMMRegister src, ComparisonPredicate vcc, int vector_len) {
3487   assert(VM_Version::supports_avx512vlbw(), "");
3488   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3489   attributes.set_is_evex_instruction();
3490   int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3491   emit_int8(0x3E);
3492   emit_int8((unsigned char)(0xC0 | encode));
3493   emit_int8(vcc);
3494 }
3495 
3496 void Assembler::evpcmpuw(KRegister kdst, KRegister mask, XMMRegister nds, XMMRegister src, ComparisonPredicate vcc, int vector_len) {
3497   assert(is_vector_masking(), "");
3498   assert(VM_Version::supports_avx512vlbw(), "");
3499   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
3500   attributes.reset_is_clear_context();
3501   attributes.set_embedded_opmask_register_specifier(mask);
3502   attributes.set_is_evex_instruction();
3503   int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3504   emit_int8(0x3E);
3505   emit_int8((unsigned char)(0xC0 | encode));
3506   emit_int8(vcc);
3507 }
3508 
3509 void Assembler::evpcmpuw(KRegister kdst, XMMRegister nds, Address src, ComparisonPredicate vcc, int vector_len) {
3510   assert(VM_Version::supports_avx512vlbw(), "");
3511   InstructionMark im(this);
3512   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3513   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
3514   attributes.set_is_evex_instruction();
3515   int dst_enc = kdst->encoding();
3516   vex_prefix(src, nds->encoding(), kdst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3517   emit_int8(0x3E);
3518   emit_operand(as_Register(dst_enc), src);
3519   emit_int8(vcc);
3520 }
3521 
3522 void Assembler::evpcmpeqb(KRegister kdst, XMMRegister nds, Address src, int vector_len) {
3523   assert(VM_Version::supports_avx512bw(), "");
3524   InstructionMark im(this);
3525   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3526   attributes.set_is_evex_instruction();
3527   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
3528   int dst_enc = kdst->encoding();
3529   vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3530   emit_int8(0x74);
3531   emit_operand(as_Register(dst_enc), src);
3532 }
3533 
3534 void Assembler::evpcmpeqb(KRegister kdst, KRegister mask, XMMRegister nds, Address src, int vector_len) {
3535   assert(VM_Version::supports_avx512vlbw(), "");
3536   assert(is_vector_masking(), "");    // For stub code use only
3537   InstructionMark im(this);
3538   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_reg_mask */ false, /* uses_vl */ false);
3539   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
3540   attributes.reset_is_clear_context();
3541   attributes.set_embedded_opmask_register_specifier(mask);
3542   attributes.set_is_evex_instruction();
3543   vex_prefix(src, nds->encoding(), kdst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3544   emit_int8(0x74);
3545   emit_operand(as_Register(kdst->encoding()), src);
3546 }
3547 
3548 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
3549 void Assembler::pcmpeqw(XMMRegister dst, XMMRegister src) {
3550   assert(VM_Version::supports_sse2(), "");
3551   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3552   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3553   emit_int8(0x75);
3554   emit_int8((unsigned char)(0xC0 | encode));
3555 }
3556 
3557 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
3558 void Assembler::vpcmpeqw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
3559   assert(VM_Version::supports_avx(), "");
3560   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3561   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3562   emit_int8(0x75);
3563   emit_int8((unsigned char)(0xC0 | encode));
3564 }
3565 
3566 // In this context, kdst is written the mask used to process the equal components
3567 void Assembler::evpcmpeqw(KRegister kdst, XMMRegister nds, XMMRegister src, int vector_len) {
3568   assert(VM_Version::supports_avx512bw(), "");
3569   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
3570   attributes.set_is_evex_instruction();
3571   int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3572   emit_int8(0x75);
3573   emit_int8((unsigned char)(0xC0 | encode));
3574 }
3575 
3576 void Assembler::evpcmpeqw(KRegister kdst, XMMRegister nds, Address src, int vector_len) {
3577   assert(VM_Version::supports_avx512bw(), "");
3578   InstructionMark im(this);
3579   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
3580   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
3581   attributes.set_is_evex_instruction();
3582   int dst_enc = kdst->encoding();
3583   vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3584   emit_int8(0x75);
3585   emit_operand(as_Register(dst_enc), src);
3586 }
3587 
3588 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
3589 void Assembler::pcmpeqd(XMMRegister dst, XMMRegister src) {
3590   assert(VM_Version::supports_sse2(), "");
3591   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3592   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3593   emit_int8(0x76);
3594   emit_int8((unsigned char)(0xC0 | encode));
3595 }
3596 
3597 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
3598 void Assembler::vpcmpeqd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
3599   assert(VM_Version::supports_avx(), "");
3600   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3601   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3602   emit_int8(0x76);
3603   emit_int8((unsigned char)(0xC0 | encode));
3604 }
3605 
3606 // In this context, kdst is written the mask used to process the equal components
3607 void Assembler::evpcmpeqd(KRegister kdst, XMMRegister nds, XMMRegister src, int vector_len) {
3608   assert(VM_Version::supports_evex(), "");
3609   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
3610   attributes.set_is_evex_instruction();
3611   attributes.reset_is_clear_context();
3612   int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3613   emit_int8(0x76);
3614   emit_int8((unsigned char)(0xC0 | encode));
3615 }
3616 
3617 void Assembler::evpcmpeqd(KRegister kdst, XMMRegister nds, Address src, int vector_len) {
3618   assert(VM_Version::supports_evex(), "");
3619   InstructionMark im(this);
3620   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
3621   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
3622   attributes.reset_is_clear_context();
3623   attributes.set_is_evex_instruction();
3624   int dst_enc = kdst->encoding();
3625   vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3626   emit_int8(0x76);
3627   emit_operand(as_Register(dst_enc), src);
3628 }
3629 
3630 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
3631 void Assembler::pcmpeqq(XMMRegister dst, XMMRegister src) {
3632   assert(VM_Version::supports_sse4_1(), "");
3633   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3634   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3635   emit_int8(0x29);
3636   emit_int8((unsigned char)(0xC0 | encode));
3637 }
3638 
3639 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
3640 void Assembler::vpcmpeqq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
3641   assert(VM_Version::supports_avx(), "");
3642   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3643   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3644   emit_int8(0x29);
3645   emit_int8((unsigned char)(0xC0 | encode));
3646 }
3647 
3648 // In this context, kdst is written the mask used to process the equal components
3649 void Assembler::evpcmpeqq(KRegister kdst, XMMRegister nds, XMMRegister src, int vector_len) {
3650   assert(VM_Version::supports_evex(), "");
3651   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
3652   attributes.reset_is_clear_context();
3653   attributes.set_is_evex_instruction();
3654   int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3655   emit_int8(0x29);
3656   emit_int8((unsigned char)(0xC0 | encode));
3657 }
3658 
3659 // In this context, kdst is written the mask used to process the equal components
3660 void Assembler::evpcmpeqq(KRegister kdst, XMMRegister nds, Address src, int vector_len) {
3661   assert(VM_Version::supports_evex(), "");
3662   InstructionMark im(this);
3663   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
3664   attributes.reset_is_clear_context();
3665   attributes.set_is_evex_instruction();
3666   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
3667   int dst_enc = kdst->encoding();
3668   vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3669   emit_int8(0x29);
3670   emit_operand(as_Register(dst_enc), src);
3671 }
3672 
3673 void Assembler::pmovmskb(Register dst, XMMRegister src) {
3674   assert(VM_Version::supports_sse2(), "");
3675   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3676   int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3677   emit_int8((unsigned char)0xD7);
3678   emit_int8((unsigned char)(0xC0 | encode));
3679 }
3680 
3681 void Assembler::vpmovmskb(Register dst, XMMRegister src) {
3682   assert(VM_Version::supports_avx2(), "");
3683   InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3684   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3685   emit_int8((unsigned char)0xD7);
3686   emit_int8((unsigned char)(0xC0 | encode));
3687 }
3688 
3689 void Assembler::pextrd(Register dst, XMMRegister src, int imm8) {
3690   assert(VM_Version::supports_sse4_1(), "");
3691   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
3692   int encode = simd_prefix_and_encode(src, xnoreg, as_XMMRegister(dst->encoding()), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3693   emit_int8(0x16);
3694   emit_int8((unsigned char)(0xC0 | encode));
3695   emit_int8(imm8);
3696 }
3697 
3698 void Assembler::pextrd(Address dst, XMMRegister src, int imm8) {
3699   assert(VM_Version::supports_sse4_1(), "");
3700   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
3701   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
3702   simd_prefix(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3703   emit_int8(0x16);
3704   emit_operand(src, dst);
3705   emit_int8(imm8);
3706 }
3707 
3708 void Assembler::pextrq(Register dst, XMMRegister src, int imm8) {
3709   assert(VM_Version::supports_sse4_1(), "");
3710   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
3711   int encode = simd_prefix_and_encode(src, xnoreg, as_XMMRegister(dst->encoding()), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3712   emit_int8(0x16);
3713   emit_int8((unsigned char)(0xC0 | encode));
3714   emit_int8(imm8);
3715 }
3716 
3717 void Assembler::pextrq(Address dst, XMMRegister src, int imm8) {
3718   assert(VM_Version::supports_sse4_1(), "");
3719   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
3720   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
3721   simd_prefix(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3722   emit_int8(0x16);
3723   emit_operand(src, dst);
3724   emit_int8(imm8);
3725 }
3726 
3727 void Assembler::pextrw(Register dst, XMMRegister src, int imm8) {
3728   assert(VM_Version::supports_sse2(), "");
3729   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
3730   int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3731   emit_int8((unsigned char)0xC5);
3732   emit_int8((unsigned char)(0xC0 | encode));
3733   emit_int8(imm8);
3734 }
3735 
3736 void Assembler::pextrw(Address dst, XMMRegister src, int imm8) {
3737   assert(VM_Version::supports_sse4_1(), "");
3738   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
3739   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_16bit);
3740   simd_prefix(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3741   emit_int8((unsigned char)0x15);
3742   emit_operand(src, dst);
3743   emit_int8(imm8);
3744 }
3745 
3746 void Assembler::pextrb(Address dst, XMMRegister src, int imm8) {
3747   assert(VM_Version::supports_sse4_1(), "");
3748   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
3749   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_8bit);
3750   simd_prefix(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3751   emit_int8(0x14);
3752   emit_operand(src, dst);
3753   emit_int8(imm8);
3754 }
3755 
3756 void Assembler::pinsrd(XMMRegister dst, Register src, int imm8) {
3757   assert(VM_Version::supports_sse4_1(), "");
3758   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
3759   int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3760   emit_int8(0x22);
3761   emit_int8((unsigned char)(0xC0 | encode));
3762   emit_int8(imm8);
3763 }
3764 
3765 void Assembler::pinsrd(XMMRegister dst, Address src, int imm8) {
3766   assert(VM_Version::supports_sse4_1(), "");
3767   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
3768   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
3769   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3770   emit_int8(0x22);
3771   emit_operand(dst,src);
3772   emit_int8(imm8);
3773 }
3774 
3775 void Assembler::pinsrq(XMMRegister dst, Register src, int imm8) {
3776   assert(VM_Version::supports_sse4_1(), "");
3777   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
3778   int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3779   emit_int8(0x22);
3780   emit_int8((unsigned char)(0xC0 | encode));
3781   emit_int8(imm8);
3782 }
3783 
3784 void Assembler::pinsrq(XMMRegister dst, Address src, int imm8) {
3785   assert(VM_Version::supports_sse4_1(), "");
3786   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
3787   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
3788   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3789   emit_int8(0x22);
3790   emit_operand(dst, src);
3791   emit_int8(imm8);
3792 }
3793 
3794 void Assembler::pinsrw(XMMRegister dst, Register src, int imm8) {
3795   assert(VM_Version::supports_sse2(), "");
3796   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
3797   int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3798   emit_int8((unsigned char)0xC4);
3799   emit_int8((unsigned char)(0xC0 | encode));
3800   emit_int8(imm8);
3801 }
3802 
3803 void Assembler::pinsrw(XMMRegister dst, Address src, int imm8) {
3804   assert(VM_Version::supports_sse2(), "");
3805   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
3806   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_16bit);
3807   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3808   emit_int8((unsigned char)0xC4);
3809   emit_operand(dst, src);
3810   emit_int8(imm8);
3811 }
3812 
3813 void Assembler::pinsrb(XMMRegister dst, Address src, int imm8) {
3814   assert(VM_Version::supports_sse4_1(), "");
3815   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
3816   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_8bit);
3817   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3818   emit_int8(0x20);
3819   emit_operand(dst, src);
3820   emit_int8(imm8);
3821 }
3822 
3823 void Assembler::pmovzxbw(XMMRegister dst, Address src) {
3824   assert(VM_Version::supports_sse4_1(), "");
3825   InstructionMark im(this);
3826   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
3827   attributes.set_address_attributes(/* tuple_type */ EVEX_HVM, /* input_size_in_bits */ EVEX_NObit);
3828   simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3829   emit_int8(0x30);
3830   emit_operand(dst, src);
3831 }
3832 
3833 void Assembler::pmovzxbw(XMMRegister dst, XMMRegister src) {
3834   assert(VM_Version::supports_sse4_1(), "");
3835   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
3836   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3837   emit_int8(0x30);
3838   emit_int8((unsigned char)(0xC0 | encode));
3839 }
3840 
3841 void Assembler::vpmovzxbw(XMMRegister dst, Address src, int vector_len) {
3842   assert(VM_Version::supports_avx(), "");
3843   InstructionMark im(this);
3844   assert(dst != xnoreg, "sanity");
3845   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
3846   attributes.set_address_attributes(/* tuple_type */ EVEX_HVM, /* input_size_in_bits */ EVEX_NObit);
3847   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3848   emit_int8(0x30);
3849   emit_operand(dst, src);
3850 }
3851 
3852 void Assembler::evpmovzxbw(XMMRegister dst, KRegister mask, Address src, int vector_len) {
3853   assert(is_vector_masking(), "");
3854   assert(VM_Version::supports_avx512vlbw(), "");
3855   assert(dst != xnoreg, "sanity");
3856   InstructionMark im(this);
3857   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
3858   attributes.set_address_attributes(/* tuple_type */ EVEX_HVM, /* input_size_in_bits */ EVEX_NObit);
3859   attributes.set_embedded_opmask_register_specifier(mask);
3860   attributes.set_is_evex_instruction();
3861   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3862   emit_int8(0x30);
3863   emit_operand(dst, src);
3864 }
3865 
3866 void Assembler::evpmovwb(Address dst, XMMRegister src, int vector_len) {
3867   assert(VM_Version::supports_avx512vlbw(), "");
3868   assert(src != xnoreg, "sanity");
3869   InstructionMark im(this);
3870   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3871   attributes.set_address_attributes(/* tuple_type */ EVEX_HVM, /* input_size_in_bits */ EVEX_NObit);
3872   attributes.set_is_evex_instruction();
3873   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
3874   emit_int8(0x30);
3875   emit_operand(src, dst);
3876 }
3877 
3878 void Assembler::evpmovwb(Address dst, KRegister mask, XMMRegister src, int vector_len) {
3879   assert(is_vector_masking(), "");
3880   assert(VM_Version::supports_avx512vlbw(), "");
3881   assert(src != xnoreg, "sanity");
3882   InstructionMark im(this);
3883   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
3884   attributes.set_address_attributes(/* tuple_type */ EVEX_HVM, /* input_size_in_bits */ EVEX_NObit);
3885   attributes.reset_is_clear_context();
3886   attributes.set_embedded_opmask_register_specifier(mask);
3887   attributes.set_is_evex_instruction();
3888   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
3889   emit_int8(0x30);
3890   emit_operand(src, dst);
3891 }
3892 
3893 // generic
3894 void Assembler::pop(Register dst) {
3895   int encode = prefix_and_encode(dst->encoding());
3896   emit_int8(0x58 | encode);
3897 }
3898 
3899 void Assembler::popcntl(Register dst, Address src) {
3900   assert(VM_Version::supports_popcnt(), "must support");
3901   InstructionMark im(this);
3902   emit_int8((unsigned char)0xF3);
3903   prefix(src, dst);
3904   emit_int8(0x0F);
3905   emit_int8((unsigned char)0xB8);
3906   emit_operand(dst, src);
3907 }
3908 
3909 void Assembler::popcntl(Register dst, Register src) {
3910   assert(VM_Version::supports_popcnt(), "must support");
3911   emit_int8((unsigned char)0xF3);
3912   int encode = prefix_and_encode(dst->encoding(), src->encoding());
3913   emit_int8(0x0F);
3914   emit_int8((unsigned char)0xB8);
3915   emit_int8((unsigned char)(0xC0 | encode));
3916 }
3917 
3918 void Assembler::popf() {
3919   emit_int8((unsigned char)0x9D);
3920 }
3921 
3922 #ifndef _LP64 // no 32bit push/pop on amd64
3923 void Assembler::popl(Address dst) {
3924   // NOTE: this will adjust stack by 8byte on 64bits
3925   InstructionMark im(this);
3926   prefix(dst);
3927   emit_int8((unsigned char)0x8F);
3928   emit_operand(rax, dst);
3929 }
3930 #endif
3931 
3932 void Assembler::prefetch_prefix(Address src) {
3933   prefix(src);
3934   emit_int8(0x0F);
3935 }
3936 
3937 void Assembler::prefetchnta(Address src) {
3938   NOT_LP64(assert(VM_Version::supports_sse(), "must support"));
3939   InstructionMark im(this);
3940   prefetch_prefix(src);
3941   emit_int8(0x18);
3942   emit_operand(rax, src); // 0, src
3943 }
3944 
3945 void Assembler::prefetchr(Address src) {
3946   assert(VM_Version::supports_3dnow_prefetch(), "must support");
3947   InstructionMark im(this);
3948   prefetch_prefix(src);
3949   emit_int8(0x0D);
3950   emit_operand(rax, src); // 0, src
3951 }
3952 
3953 void Assembler::prefetcht0(Address src) {
3954   NOT_LP64(assert(VM_Version::supports_sse(), "must support"));
3955   InstructionMark im(this);
3956   prefetch_prefix(src);
3957   emit_int8(0x18);
3958   emit_operand(rcx, src); // 1, src
3959 }
3960 
3961 void Assembler::prefetcht1(Address src) {
3962   NOT_LP64(assert(VM_Version::supports_sse(), "must support"));
3963   InstructionMark im(this);
3964   prefetch_prefix(src);
3965   emit_int8(0x18);
3966   emit_operand(rdx, src); // 2, src
3967 }
3968 
3969 void Assembler::prefetcht2(Address src) {
3970   NOT_LP64(assert(VM_Version::supports_sse(), "must support"));
3971   InstructionMark im(this);
3972   prefetch_prefix(src);
3973   emit_int8(0x18);
3974   emit_operand(rbx, src); // 3, src
3975 }
3976 
3977 void Assembler::prefetchw(Address src) {
3978   assert(VM_Version::supports_3dnow_prefetch(), "must support");
3979   InstructionMark im(this);
3980   prefetch_prefix(src);
3981   emit_int8(0x0D);
3982   emit_operand(rcx, src); // 1, src
3983 }
3984 
3985 void Assembler::prefix(Prefix p) {
3986   emit_int8(p);
3987 }
3988 
3989 void Assembler::pshufb(XMMRegister dst, XMMRegister src) {
3990   assert(VM_Version::supports_ssse3(), "");
3991   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
3992   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3993   emit_int8(0x00);
3994   emit_int8((unsigned char)(0xC0 | encode));
3995 }
3996 
3997 void Assembler::vpshufb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
3998   assert(vector_len == AVX_128bit? VM_Version::supports_avx() :
3999          vector_len == AVX_256bit? VM_Version::supports_avx2() :
4000          0, "");
4001   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
4002   int encode = simd_prefix_and_encode(dst, nds, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4003   emit_int8(0x00);
4004   emit_int8((unsigned char)(0xC0 | encode));
4005 }
4006 
4007 void Assembler::pshufb(XMMRegister dst, Address src) {
4008   assert(VM_Version::supports_ssse3(), "");
4009   InstructionMark im(this);
4010   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
4011   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
4012   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4013   emit_int8(0x00);
4014   emit_operand(dst, src);
4015 }
4016 
4017 void Assembler::pshufd(XMMRegister dst, XMMRegister src, int mode) {
4018   assert(isByte(mode), "invalid value");
4019   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4020   int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_128bit;
4021   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4022   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4023   emit_int8(0x70);
4024   emit_int8((unsigned char)(0xC0 | encode));
4025   emit_int8(mode & 0xFF);
4026 }
4027 
4028 void Assembler::vpshufd(XMMRegister dst, XMMRegister src, int mode, int vector_len) {
4029   assert(vector_len == AVX_128bit? VM_Version::supports_avx() :
4030          vector_len == AVX_256bit? VM_Version::supports_avx2() :
4031          0, "");
4032   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4033   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4034   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4035   emit_int8(0x70);
4036   emit_int8((unsigned char)(0xC0 | encode));
4037   emit_int8(mode & 0xFF);
4038 }
4039 
4040 void Assembler::pshufd(XMMRegister dst, Address src, int mode) {
4041   assert(isByte(mode), "invalid value");
4042   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4043   assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
4044   InstructionMark im(this);
4045   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4046   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
4047   simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4048   emit_int8(0x70);
4049   emit_operand(dst, src);
4050   emit_int8(mode & 0xFF);
4051 }
4052 
4053 void Assembler::pshuflw(XMMRegister dst, XMMRegister src, int mode) {
4054   assert(isByte(mode), "invalid value");
4055   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4056   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
4057   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
4058   emit_int8(0x70);
4059   emit_int8((unsigned char)(0xC0 | encode));
4060   emit_int8(mode & 0xFF);
4061 }
4062 
4063 void Assembler::pshuflw(XMMRegister dst, Address src, int mode) {
4064   assert(isByte(mode), "invalid value");
4065   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4066   assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
4067   InstructionMark im(this);
4068   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
4069   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
4070   simd_prefix(dst, xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
4071   emit_int8(0x70);
4072   emit_operand(dst, src);
4073   emit_int8(mode & 0xFF);
4074 }
4075 
4076 void Assembler::psrldq(XMMRegister dst, int shift) {
4077   // Shift left 128 bit value in dst XMMRegister by shift number of bytes.
4078   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4079   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
4080   int encode = simd_prefix_and_encode(xmm3, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4081   emit_int8(0x73);
4082   emit_int8((unsigned char)(0xC0 | encode));
4083   emit_int8(shift);
4084 }
4085 
4086 void Assembler::pslldq(XMMRegister dst, int shift) {
4087   // Shift left 128 bit value in dst XMMRegister by shift number of bytes.
4088   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4089   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
4090   // XMM7 is for /7 encoding: 66 0F 73 /7 ib
4091   int encode = simd_prefix_and_encode(xmm7, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4092   emit_int8(0x73);
4093   emit_int8((unsigned char)(0xC0 | encode));
4094   emit_int8(shift);
4095 }
4096 
4097 void Assembler::ptest(XMMRegister dst, Address src) {
4098   assert(VM_Version::supports_sse4_1(), "");
4099   assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
4100   InstructionMark im(this);
4101   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
4102   simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4103   emit_int8(0x17);
4104   emit_operand(dst, src);
4105 }
4106 
4107 void Assembler::ptest(XMMRegister dst, XMMRegister src) {
4108   assert(VM_Version::supports_sse4_1(), "");
4109   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
4110   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4111   emit_int8(0x17);
4112   emit_int8((unsigned char)(0xC0 | encode));
4113 }
4114 
4115 void Assembler::vptest(XMMRegister dst, Address src) {
4116   assert(VM_Version::supports_avx(), "");
4117   InstructionMark im(this);
4118   InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
4119   assert(dst != xnoreg, "sanity");
4120   // swap src<->dst for encoding
4121   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4122   emit_int8(0x17);
4123   emit_operand(dst, src);
4124 }
4125 
4126 void Assembler::vptest(XMMRegister dst, XMMRegister src) {
4127   assert(VM_Version::supports_avx(), "");
4128   InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
4129   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4130   emit_int8(0x17);
4131   emit_int8((unsigned char)(0xC0 | encode));
4132 }
4133 
4134 void Assembler::punpcklbw(XMMRegister dst, Address src) {
4135   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4136   assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
4137   InstructionMark im(this);
4138   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_vlbw, /* no_mask_reg */ false, /* uses_vl */ true);
4139   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
4140   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4141   emit_int8(0x60);
4142   emit_operand(dst, src);
4143 }
4144 
4145 void Assembler::punpcklbw(XMMRegister dst, XMMRegister src) {
4146   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4147   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_vlbw, /* no_mask_reg */ false, /* uses_vl */ true);
4148   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4149   emit_int8(0x60);
4150   emit_int8((unsigned char)(0xC0 | encode));
4151 }
4152 
4153 void Assembler::punpckldq(XMMRegister dst, Address src) {
4154   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4155   assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
4156   InstructionMark im(this);
4157   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4158   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
4159   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4160   emit_int8(0x62);
4161   emit_operand(dst, src);
4162 }
4163 
4164 void Assembler::punpckldq(XMMRegister dst, XMMRegister src) {
4165   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4166   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4167   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4168   emit_int8(0x62);
4169   emit_int8((unsigned char)(0xC0 | encode));
4170 }
4171 
4172 void Assembler::punpcklqdq(XMMRegister dst, XMMRegister src) {
4173   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4174   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4175   attributes.set_rex_vex_w_reverted();
4176   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4177   emit_int8(0x6C);
4178   emit_int8((unsigned char)(0xC0 | encode));
4179 }
4180 
4181 void Assembler::push(int32_t imm32) {
4182   // in 64bits we push 64bits onto the stack but only
4183   // take a 32bit immediate
4184   emit_int8(0x68);
4185   emit_int32(imm32);
4186 }
4187 
4188 void Assembler::push(Register src) {
4189   int encode = prefix_and_encode(src->encoding());
4190 
4191   emit_int8(0x50 | encode);
4192 }
4193 
4194 void Assembler::pushf() {
4195   emit_int8((unsigned char)0x9C);
4196 }
4197 
4198 #ifndef _LP64 // no 32bit push/pop on amd64
4199 void Assembler::pushl(Address src) {
4200   // Note this will push 64bit on 64bit
4201   InstructionMark im(this);
4202   prefix(src);
4203   emit_int8((unsigned char)0xFF);
4204   emit_operand(rsi, src);
4205 }
4206 #endif
4207 
4208 void Assembler::rcll(Register dst, int imm8) {
4209   assert(isShiftCount(imm8), "illegal shift count");
4210   int encode = prefix_and_encode(dst->encoding());
4211   if (imm8 == 1) {
4212     emit_int8((unsigned char)0xD1);
4213     emit_int8((unsigned char)(0xD0 | encode));
4214   } else {
4215     emit_int8((unsigned char)0xC1);
4216     emit_int8((unsigned char)0xD0 | encode);
4217     emit_int8(imm8);
4218   }
4219 }
4220 
4221 void Assembler::rcpps(XMMRegister dst, XMMRegister src) {
4222   NOT_LP64(assert(VM_Version::supports_sse(), ""));
4223   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
4224   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
4225   emit_int8(0x53);
4226   emit_int8((unsigned char)(0xC0 | encode));
4227 }
4228 
4229 void Assembler::rcpss(XMMRegister dst, XMMRegister src) {
4230   NOT_LP64(assert(VM_Version::supports_sse(), ""));
4231   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
4232   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
4233   emit_int8(0x53);
4234   emit_int8((unsigned char)(0xC0 | encode));
4235 }
4236 
4237 void Assembler::rdtsc() {
4238   emit_int8((unsigned char)0x0F);
4239   emit_int8((unsigned char)0x31);
4240 }
4241 
4242 // copies data from [esi] to [edi] using rcx pointer sized words
4243 // generic
4244 void Assembler::rep_mov() {
4245   emit_int8((unsigned char)0xF3);
4246   // MOVSQ
4247   LP64_ONLY(prefix(REX_W));
4248   emit_int8((unsigned char)0xA5);
4249 }
4250 
4251 // sets rcx bytes with rax, value at [edi]
4252 void Assembler::rep_stosb() {
4253   emit_int8((unsigned char)0xF3); // REP
4254   LP64_ONLY(prefix(REX_W));
4255   emit_int8((unsigned char)0xAA); // STOSB
4256 }
4257 
4258 // sets rcx pointer sized words with rax, value at [edi]
4259 // generic
4260 void Assembler::rep_stos() {
4261   emit_int8((unsigned char)0xF3); // REP
4262   LP64_ONLY(prefix(REX_W));       // LP64:STOSQ, LP32:STOSD
4263   emit_int8((unsigned char)0xAB);
4264 }
4265 
4266 // scans rcx pointer sized words at [edi] for occurance of rax,
4267 // generic
4268 void Assembler::repne_scan() { // repne_scan
4269   emit_int8((unsigned char)0xF2);
4270   // SCASQ
4271   LP64_ONLY(prefix(REX_W));
4272   emit_int8((unsigned char)0xAF);
4273 }
4274 
4275 #ifdef _LP64
4276 // scans rcx 4 byte words at [edi] for occurance of rax,
4277 // generic
4278 void Assembler::repne_scanl() { // repne_scan
4279   emit_int8((unsigned char)0xF2);
4280   // SCASL
4281   emit_int8((unsigned char)0xAF);
4282 }
4283 #endif
4284 
4285 void Assembler::ret(int imm16) {
4286   if (imm16 == 0) {
4287     emit_int8((unsigned char)0xC3);
4288   } else {
4289     emit_int8((unsigned char)0xC2);
4290     emit_int16(imm16);
4291   }
4292 }
4293 
4294 void Assembler::sahf() {
4295 #ifdef _LP64
4296   // Not supported in 64bit mode
4297   ShouldNotReachHere();
4298 #endif
4299   emit_int8((unsigned char)0x9E);
4300 }
4301 
4302 void Assembler::sarl(Register dst, int imm8) {
4303   int encode = prefix_and_encode(dst->encoding());
4304   assert(isShiftCount(imm8), "illegal shift count");
4305   if (imm8 == 1) {
4306     emit_int8((unsigned char)0xD1);
4307     emit_int8((unsigned char)(0xF8 | encode));
4308   } else {
4309     emit_int8((unsigned char)0xC1);
4310     emit_int8((unsigned char)(0xF8 | encode));
4311     emit_int8(imm8);
4312   }
4313 }
4314 
4315 void Assembler::sarl(Register dst) {
4316   int encode = prefix_and_encode(dst->encoding());
4317   emit_int8((unsigned char)0xD3);
4318   emit_int8((unsigned char)(0xF8 | encode));
4319 }
4320 
4321 void Assembler::sbbl(Address dst, int32_t imm32) {
4322   InstructionMark im(this);
4323   prefix(dst);
4324   emit_arith_operand(0x81, rbx, dst, imm32);
4325 }
4326 
4327 void Assembler::sbbl(Register dst, int32_t imm32) {
4328   prefix(dst);
4329   emit_arith(0x81, 0xD8, dst, imm32);
4330 }
4331 
4332 
4333 void Assembler::sbbl(Register dst, Address src) {
4334   InstructionMark im(this);
4335   prefix(src, dst);
4336   emit_int8(0x1B);
4337   emit_operand(dst, src);
4338 }
4339 
4340 void Assembler::sbbl(Register dst, Register src) {
4341   (void) prefix_and_encode(dst->encoding(), src->encoding());
4342   emit_arith(0x1B, 0xC0, dst, src);
4343 }
4344 
4345 void Assembler::setb(Condition cc, Register dst) {
4346   assert(0 <= cc && cc < 16, "illegal cc");
4347   int encode = prefix_and_encode(dst->encoding(), true);
4348   emit_int8(0x0F);
4349   emit_int8((unsigned char)0x90 | cc);
4350   emit_int8((unsigned char)(0xC0 | encode));
4351 }
4352 
4353 void Assembler::palignr(XMMRegister dst, XMMRegister src, int imm8) {
4354   assert(VM_Version::supports_ssse3(), "");
4355   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ false);
4356   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
4357   emit_int8((unsigned char)0x0F);
4358   emit_int8((unsigned char)(0xC0 | encode));
4359   emit_int8(imm8);
4360 }
4361 
4362 void Assembler::vpalignr(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8, int vector_len) {
4363   assert(vector_len == AVX_128bit? VM_Version::supports_avx() :
4364          vector_len == AVX_256bit? VM_Version::supports_avx2() :
4365          0, "");
4366   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
4367   int encode = simd_prefix_and_encode(dst, nds, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
4368   emit_int8((unsigned char)0x0F);
4369   emit_int8((unsigned char)(0xC0 | encode));
4370   emit_int8(imm8);
4371 }
4372 
4373 void Assembler::pblendw(XMMRegister dst, XMMRegister src, int imm8) {
4374   assert(VM_Version::supports_sse4_1(), "");
4375   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
4376   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
4377   emit_int8((unsigned char)0x0E);
4378   emit_int8((unsigned char)(0xC0 | encode));
4379   emit_int8(imm8);
4380 }
4381 
4382 void Assembler::sha1rnds4(XMMRegister dst, XMMRegister src, int imm8) {
4383   assert(VM_Version::supports_sha(), "");
4384   int encode = rex_prefix_and_encode(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3A, /* rex_w */ false);
4385   emit_int8((unsigned char)0xCC);
4386   emit_int8((unsigned char)(0xC0 | encode));
4387   emit_int8((unsigned char)imm8);
4388 }
4389 
4390 void Assembler::sha1nexte(XMMRegister dst, XMMRegister src) {
4391   assert(VM_Version::supports_sha(), "");
4392   int encode = rex_prefix_and_encode(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, /* rex_w */ false);
4393   emit_int8((unsigned char)0xC8);
4394   emit_int8((unsigned char)(0xC0 | encode));
4395 }
4396 
4397 void Assembler::sha1msg1(XMMRegister dst, XMMRegister src) {
4398   assert(VM_Version::supports_sha(), "");
4399   int encode = rex_prefix_and_encode(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, /* rex_w */ false);
4400   emit_int8((unsigned char)0xC9);
4401   emit_int8((unsigned char)(0xC0 | encode));
4402 }
4403 
4404 void Assembler::sha1msg2(XMMRegister dst, XMMRegister src) {
4405   assert(VM_Version::supports_sha(), "");
4406   int encode = rex_prefix_and_encode(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, /* rex_w */ false);
4407   emit_int8((unsigned char)0xCA);
4408   emit_int8((unsigned char)(0xC0 | encode));
4409 }
4410 
4411 // xmm0 is implicit additional source to this instruction.
4412 void Assembler::sha256rnds2(XMMRegister dst, XMMRegister src) {
4413   assert(VM_Version::supports_sha(), "");
4414   int encode = rex_prefix_and_encode(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, /* rex_w */ false);
4415   emit_int8((unsigned char)0xCB);
4416   emit_int8((unsigned char)(0xC0 | encode));
4417 }
4418 
4419 void Assembler::sha256msg1(XMMRegister dst, XMMRegister src) {
4420   assert(VM_Version::supports_sha(), "");
4421   int encode = rex_prefix_and_encode(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, /* rex_w */ false);
4422   emit_int8((unsigned char)0xCC);
4423   emit_int8((unsigned char)(0xC0 | encode));
4424 }
4425 
4426 void Assembler::sha256msg2(XMMRegister dst, XMMRegister src) {
4427   assert(VM_Version::supports_sha(), "");
4428   int encode = rex_prefix_and_encode(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, /* rex_w */ false);
4429   emit_int8((unsigned char)0xCD);
4430   emit_int8((unsigned char)(0xC0 | encode));
4431 }
4432 
4433 
4434 void Assembler::shll(Register dst, int imm8) {
4435   assert(isShiftCount(imm8), "illegal shift count");
4436   int encode = prefix_and_encode(dst->encoding());
4437   if (imm8 == 1 ) {
4438     emit_int8((unsigned char)0xD1);
4439     emit_int8((unsigned char)(0xE0 | encode));
4440   } else {
4441     emit_int8((unsigned char)0xC1);
4442     emit_int8((unsigned char)(0xE0 | encode));
4443     emit_int8(imm8);
4444   }
4445 }
4446 
4447 void Assembler::shll(Register dst) {
4448   int encode = prefix_and_encode(dst->encoding());
4449   emit_int8((unsigned char)0xD3);
4450   emit_int8((unsigned char)(0xE0 | encode));
4451 }
4452 
4453 void Assembler::shrl(Register dst, int imm8) {
4454   assert(isShiftCount(imm8), "illegal shift count");
4455   int encode = prefix_and_encode(dst->encoding());
4456   emit_int8((unsigned char)0xC1);
4457   emit_int8((unsigned char)(0xE8 | encode));
4458   emit_int8(imm8);
4459 }
4460 
4461 void Assembler::shrl(Register dst) {
4462   int encode = prefix_and_encode(dst->encoding());
4463   emit_int8((unsigned char)0xD3);
4464   emit_int8((unsigned char)(0xE8 | encode));
4465 }
4466 
4467 // copies a single word from [esi] to [edi]
4468 void Assembler::smovl() {
4469   emit_int8((unsigned char)0xA5);
4470 }
4471 
4472 void Assembler::sqrtsd(XMMRegister dst, XMMRegister src) {
4473   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4474   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4475   attributes.set_rex_vex_w_reverted();
4476   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
4477   emit_int8(0x51);
4478   emit_int8((unsigned char)(0xC0 | encode));
4479 }
4480 
4481 void Assembler::sqrtsd(XMMRegister dst, Address src) {
4482   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4483   InstructionMark im(this);
4484   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4485   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
4486   attributes.set_rex_vex_w_reverted();
4487   simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
4488   emit_int8(0x51);
4489   emit_operand(dst, src);
4490 }
4491 
4492 void Assembler::sqrtss(XMMRegister dst, XMMRegister src) {
4493   NOT_LP64(assert(VM_Version::supports_sse(), ""));
4494   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4495   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
4496   emit_int8(0x51);
4497   emit_int8((unsigned char)(0xC0 | encode));
4498 }
4499 
4500 void Assembler::std() {
4501   emit_int8((unsigned char)0xFD);
4502 }
4503 
4504 void Assembler::sqrtss(XMMRegister dst, Address src) {
4505   NOT_LP64(assert(VM_Version::supports_sse(), ""));
4506   InstructionMark im(this);
4507   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4508   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
4509   simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
4510   emit_int8(0x51);
4511   emit_operand(dst, src);
4512 }
4513 
4514 void Assembler::stmxcsr( Address dst) {
4515   if (UseAVX > 0 ) {
4516     assert(VM_Version::supports_avx(), "");
4517     InstructionMark im(this);
4518     InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
4519     vex_prefix(dst, 0, 0, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
4520     emit_int8((unsigned char)0xAE);
4521     emit_operand(as_Register(3), dst);
4522   } else {
4523     NOT_LP64(assert(VM_Version::supports_sse(), ""));
4524     InstructionMark im(this);
4525     prefix(dst);
4526     emit_int8(0x0F);
4527     emit_int8((unsigned char)0xAE);
4528     emit_operand(as_Register(3), dst);
4529   }
4530 }
4531 
4532 void Assembler::subl(Address dst, int32_t imm32) {
4533   InstructionMark im(this);
4534   prefix(dst);
4535   emit_arith_operand(0x81, rbp, dst, imm32);
4536 }
4537 
4538 void Assembler::subl(Address dst, Register src) {
4539   InstructionMark im(this);
4540   prefix(dst, src);
4541   emit_int8(0x29);
4542   emit_operand(src, dst);
4543 }
4544 
4545 void Assembler::subl(Register dst, int32_t imm32) {
4546   prefix(dst);
4547   emit_arith(0x81, 0xE8, dst, imm32);
4548 }
4549 
4550 // Force generation of a 4 byte immediate value even if it fits into 8bit
4551 void Assembler::subl_imm32(Register dst, int32_t imm32) {
4552   prefix(dst);
4553   emit_arith_imm32(0x81, 0xE8, dst, imm32);
4554 }
4555 
4556 void Assembler::subl(Register dst, Address src) {
4557   InstructionMark im(this);
4558   prefix(src, dst);
4559   emit_int8(0x2B);
4560   emit_operand(dst, src);
4561 }
4562 
4563 void Assembler::subl(Register dst, Register src) {
4564   (void) prefix_and_encode(dst->encoding(), src->encoding());
4565   emit_arith(0x2B, 0xC0, dst, src);
4566 }
4567 
4568 void Assembler::subsd(XMMRegister dst, XMMRegister src) {
4569   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4570   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4571   attributes.set_rex_vex_w_reverted();
4572   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
4573   emit_int8(0x5C);
4574   emit_int8((unsigned char)(0xC0 | encode));
4575 }
4576 
4577 void Assembler::subsd(XMMRegister dst, Address src) {
4578   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4579   InstructionMark im(this);
4580   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4581   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
4582   attributes.set_rex_vex_w_reverted();
4583   simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
4584   emit_int8(0x5C);
4585   emit_operand(dst, src);
4586 }
4587 
4588 void Assembler::subss(XMMRegister dst, XMMRegister src) {
4589   NOT_LP64(assert(VM_Version::supports_sse(), ""));
4590   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true , /* uses_vl */ false);
4591   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
4592   emit_int8(0x5C);
4593   emit_int8((unsigned char)(0xC0 | encode));
4594 }
4595 
4596 void Assembler::subss(XMMRegister dst, Address src) {
4597   NOT_LP64(assert(VM_Version::supports_sse(), ""));
4598   InstructionMark im(this);
4599   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4600   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
4601   simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
4602   emit_int8(0x5C);
4603   emit_operand(dst, src);
4604 }
4605 
4606 void Assembler::testb(Register dst, int imm8) {
4607   NOT_LP64(assert(dst->has_byte_register(), "must have byte register"));
4608   (void) prefix_and_encode(dst->encoding(), true);
4609   emit_arith_b(0xF6, 0xC0, dst, imm8);
4610 }
4611 
4612 void Assembler::testb(Address dst, int imm8) {
4613   InstructionMark im(this);
4614   prefix(dst);
4615   emit_int8((unsigned char)0xF6);
4616   emit_operand(rax, dst, 1);
4617   emit_int8(imm8);
4618 }
4619 
4620 void Assembler::testl(Register dst, int32_t imm32) {
4621   // not using emit_arith because test
4622   // doesn't support sign-extension of
4623   // 8bit operands
4624   int encode = dst->encoding();
4625   if (encode == 0) {
4626     emit_int8((unsigned char)0xA9);
4627   } else {
4628     encode = prefix_and_encode(encode);
4629     emit_int8((unsigned char)0xF7);
4630     emit_int8((unsigned char)(0xC0 | encode));
4631   }
4632   emit_int32(imm32);
4633 }
4634 
4635 void Assembler::testl(Register dst, Register src) {
4636   (void) prefix_and_encode(dst->encoding(), src->encoding());
4637   emit_arith(0x85, 0xC0, dst, src);
4638 }
4639 
4640 void Assembler::testl(Register dst, Address src) {
4641   InstructionMark im(this);
4642   prefix(src, dst);
4643   emit_int8((unsigned char)0x85);
4644   emit_operand(dst, src);
4645 }
4646 
4647 void Assembler::tzcntl(Register dst, Register src) {
4648   assert(VM_Version::supports_bmi1(), "tzcnt instruction not supported");
4649   emit_int8((unsigned char)0xF3);
4650   int encode = prefix_and_encode(dst->encoding(), src->encoding());
4651   emit_int8(0x0F);
4652   emit_int8((unsigned char)0xBC);
4653   emit_int8((unsigned char)0xC0 | encode);
4654 }
4655 
4656 void Assembler::tzcntq(Register dst, Register src) {
4657   assert(VM_Version::supports_bmi1(), "tzcnt instruction not supported");
4658   emit_int8((unsigned char)0xF3);
4659   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
4660   emit_int8(0x0F);
4661   emit_int8((unsigned char)0xBC);
4662   emit_int8((unsigned char)(0xC0 | encode));
4663 }
4664 
4665 void Assembler::ucomisd(XMMRegister dst, Address src) {
4666   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4667   InstructionMark im(this);
4668   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4669   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
4670   attributes.set_rex_vex_w_reverted();
4671   simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4672   emit_int8(0x2E);
4673   emit_operand(dst, src);
4674 }
4675 
4676 void Assembler::ucomisd(XMMRegister dst, XMMRegister src) {
4677   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4678   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4679   attributes.set_rex_vex_w_reverted();
4680   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4681   emit_int8(0x2E);
4682   emit_int8((unsigned char)(0xC0 | encode));
4683 }
4684 
4685 void Assembler::ucomiss(XMMRegister dst, Address src) {
4686   NOT_LP64(assert(VM_Version::supports_sse(), ""));
4687   InstructionMark im(this);
4688   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4689   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
4690   simd_prefix(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
4691   emit_int8(0x2E);
4692   emit_operand(dst, src);
4693 }
4694 
4695 void Assembler::ucomiss(XMMRegister dst, XMMRegister src) {
4696   NOT_LP64(assert(VM_Version::supports_sse(), ""));
4697   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4698   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
4699   emit_int8(0x2E);
4700   emit_int8((unsigned char)(0xC0 | encode));
4701 }
4702 
4703 void Assembler::xabort(int8_t imm8) {
4704   emit_int8((unsigned char)0xC6);
4705   emit_int8((unsigned char)0xF8);
4706   emit_int8((unsigned char)(imm8 & 0xFF));
4707 }
4708 
4709 void Assembler::xaddb(Address dst, Register src) {
4710   InstructionMark im(this);
4711   prefix(dst, src, true);
4712   emit_int8(0x0F);
4713   emit_int8((unsigned char)0xC0);
4714   emit_operand(src, dst);
4715 }
4716 
4717 void Assembler::xaddw(Address dst, Register src) {
4718   InstructionMark im(this);
4719   emit_int8(0x66);
4720   prefix(dst, src);
4721   emit_int8(0x0F);
4722   emit_int8((unsigned char)0xC1);
4723   emit_operand(src, dst);
4724 }
4725 
4726 void Assembler::xaddl(Address dst, Register src) {
4727   InstructionMark im(this);
4728   prefix(dst, src);
4729   emit_int8(0x0F);
4730   emit_int8((unsigned char)0xC1);
4731   emit_operand(src, dst);
4732 }
4733 
4734 void Assembler::xbegin(Label& abort, relocInfo::relocType rtype) {
4735   InstructionMark im(this);
4736   relocate(rtype);
4737   if (abort.is_bound()) {
4738     address entry = target(abort);
4739     assert(entry != NULL, "abort entry NULL");
4740     intptr_t offset = entry - pc();
4741     emit_int8((unsigned char)0xC7);
4742     emit_int8((unsigned char)0xF8);
4743     emit_int32(offset - 6); // 2 opcode + 4 address
4744   } else {
4745     abort.add_patch_at(code(), locator());
4746     emit_int8((unsigned char)0xC7);
4747     emit_int8((unsigned char)0xF8);
4748     emit_int32(0);
4749   }
4750 }
4751 
4752 void Assembler::xchgb(Register dst, Address src) { // xchg
4753   InstructionMark im(this);
4754   prefix(src, dst, true);
4755   emit_int8((unsigned char)0x86);
4756   emit_operand(dst, src);
4757 }
4758 
4759 void Assembler::xchgw(Register dst, Address src) { // xchg
4760   InstructionMark im(this);
4761   emit_int8(0x66);
4762   prefix(src, dst);
4763   emit_int8((unsigned char)0x87);
4764   emit_operand(dst, src);
4765 }
4766 
4767 void Assembler::xchgl(Register dst, Address src) { // xchg
4768   InstructionMark im(this);
4769   prefix(src, dst);
4770   emit_int8((unsigned char)0x87);
4771   emit_operand(dst, src);
4772 }
4773 
4774 void Assembler::xchgl(Register dst, Register src) {
4775   int encode = prefix_and_encode(dst->encoding(), src->encoding());
4776   emit_int8((unsigned char)0x87);
4777   emit_int8((unsigned char)(0xC0 | encode));
4778 }
4779 
4780 void Assembler::xend() {
4781   emit_int8((unsigned char)0x0F);
4782   emit_int8((unsigned char)0x01);
4783   emit_int8((unsigned char)0xD5);
4784 }
4785 
4786 void Assembler::xgetbv() {
4787   emit_int8(0x0F);
4788   emit_int8(0x01);
4789   emit_int8((unsigned char)0xD0);
4790 }
4791 
4792 void Assembler::xorl(Register dst, int32_t imm32) {
4793   prefix(dst);
4794   emit_arith(0x81, 0xF0, dst, imm32);
4795 }
4796 
4797 void Assembler::xorl(Register dst, Address src) {
4798   InstructionMark im(this);
4799   prefix(src, dst);
4800   emit_int8(0x33);
4801   emit_operand(dst, src);
4802 }
4803 
4804 void Assembler::xorl(Register dst, Register src) {
4805   (void) prefix_and_encode(dst->encoding(), src->encoding());
4806   emit_arith(0x33, 0xC0, dst, src);
4807 }
4808 
4809 void Assembler::xorb(Register dst, Address src) {
4810   InstructionMark im(this);
4811   prefix(src, dst);
4812   emit_int8(0x32);
4813   emit_operand(dst, src);
4814 }
4815 
4816 // AVX 3-operands scalar float-point arithmetic instructions
4817 
4818 void Assembler::vaddsd(XMMRegister dst, XMMRegister nds, Address src) {
4819   assert(VM_Version::supports_avx(), "");
4820   InstructionMark im(this);
4821   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4822   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
4823   attributes.set_rex_vex_w_reverted();
4824   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
4825   emit_int8(0x58);
4826   emit_operand(dst, src);
4827 }
4828 
4829 void Assembler::vaddsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
4830   assert(VM_Version::supports_avx(), "");
4831   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4832   attributes.set_rex_vex_w_reverted();
4833   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
4834   emit_int8(0x58);
4835   emit_int8((unsigned char)(0xC0 | encode));
4836 }
4837 
4838 void Assembler::vaddss(XMMRegister dst, XMMRegister nds, Address src) {
4839   assert(VM_Version::supports_avx(), "");
4840   InstructionMark im(this);
4841   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4842   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
4843   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
4844   emit_int8(0x58);
4845   emit_operand(dst, src);
4846 }
4847 
4848 void Assembler::vaddss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
4849   assert(VM_Version::supports_avx(), "");
4850   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4851   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
4852   emit_int8(0x58);
4853   emit_int8((unsigned char)(0xC0 | encode));
4854 }
4855 
4856 void Assembler::vdivsd(XMMRegister dst, XMMRegister nds, Address src) {
4857   assert(VM_Version::supports_avx(), "");
4858   InstructionMark im(this);
4859   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4860   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
4861   attributes.set_rex_vex_w_reverted();
4862   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
4863   emit_int8(0x5E);
4864   emit_operand(dst, src);
4865 }
4866 
4867 void Assembler::vdivsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
4868   assert(VM_Version::supports_avx(), "");
4869   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4870   attributes.set_rex_vex_w_reverted();
4871   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
4872   emit_int8(0x5E);
4873   emit_int8((unsigned char)(0xC0 | encode));
4874 }
4875 
4876 void Assembler::vdivss(XMMRegister dst, XMMRegister nds, Address src) {
4877   assert(VM_Version::supports_avx(), "");
4878   InstructionMark im(this);
4879   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4880   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
4881   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
4882   emit_int8(0x5E);
4883   emit_operand(dst, src);
4884 }
4885 
4886 void Assembler::vdivss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
4887   assert(VM_Version::supports_avx(), "");
4888   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4889   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
4890   emit_int8(0x5E);
4891   emit_int8((unsigned char)(0xC0 | encode));
4892 }
4893 
4894 void Assembler::vfmadd231sd(XMMRegister dst, XMMRegister src1, XMMRegister src2) {
4895   assert(VM_Version::supports_fma(), "");
4896   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4897   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4898   emit_int8((unsigned char)0xB9);
4899   emit_int8((unsigned char)(0xC0 | encode));
4900 }
4901 
4902 void Assembler::vfmadd231ss(XMMRegister dst, XMMRegister src1, XMMRegister src2) {
4903   assert(VM_Version::supports_fma(), "");
4904   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4905   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4906   emit_int8((unsigned char)0xB9);
4907   emit_int8((unsigned char)(0xC0 | encode));
4908 }
4909 
4910 void Assembler::vmulsd(XMMRegister dst, XMMRegister nds, Address src) {
4911   assert(VM_Version::supports_avx(), "");
4912   InstructionMark im(this);
4913   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4914   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
4915   attributes.set_rex_vex_w_reverted();
4916   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
4917   emit_int8(0x59);
4918   emit_operand(dst, src);
4919 }
4920 
4921 void Assembler::vmulsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
4922   assert(VM_Version::supports_avx(), "");
4923   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4924   attributes.set_rex_vex_w_reverted();
4925   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
4926   emit_int8(0x59);
4927   emit_int8((unsigned char)(0xC0 | encode));
4928 }
4929 
4930 void Assembler::vmulss(XMMRegister dst, XMMRegister nds, Address src) {
4931   assert(VM_Version::supports_avx(), "");
4932   InstructionMark im(this);
4933   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4934   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
4935   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
4936   emit_int8(0x59);
4937   emit_operand(dst, src);
4938 }
4939 
4940 void Assembler::vmulss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
4941   assert(VM_Version::supports_avx(), "");
4942   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4943   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
4944   emit_int8(0x59);
4945   emit_int8((unsigned char)(0xC0 | encode));
4946 }
4947 
4948 void Assembler::vsubsd(XMMRegister dst, XMMRegister nds, Address src) {
4949   assert(VM_Version::supports_avx(), "");
4950   InstructionMark im(this);
4951   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4952   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
4953   attributes.set_rex_vex_w_reverted();
4954   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
4955   emit_int8(0x5C);
4956   emit_operand(dst, src);
4957 }
4958 
4959 void Assembler::vsubsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
4960   assert(VM_Version::supports_avx(), "");
4961   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4962   attributes.set_rex_vex_w_reverted();
4963   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
4964   emit_int8(0x5C);
4965   emit_int8((unsigned char)(0xC0 | encode));
4966 }
4967 
4968 void Assembler::vsubss(XMMRegister dst, XMMRegister nds, Address src) {
4969   assert(VM_Version::supports_avx(), "");
4970   InstructionMark im(this);
4971   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4972   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
4973   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
4974   emit_int8(0x5C);
4975   emit_operand(dst, src);
4976 }
4977 
4978 void Assembler::vsubss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
4979   assert(VM_Version::supports_avx(), "");
4980   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4981   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
4982   emit_int8(0x5C);
4983   emit_int8((unsigned char)(0xC0 | encode));
4984 }
4985 
4986 //====================VECTOR ARITHMETIC=====================================
4987 
4988 // Float-point vector arithmetic
4989 
4990 void Assembler::addpd(XMMRegister dst, XMMRegister src) {
4991   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4992   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4993   attributes.set_rex_vex_w_reverted();
4994   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4995   emit_int8(0x58);
4996   emit_int8((unsigned char)(0xC0 | encode));
4997 }
4998 
4999 void Assembler::addpd(XMMRegister dst, Address src) {
5000   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5001   InstructionMark im(this);
5002   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5003   attributes.set_rex_vex_w_reverted();
5004   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
5005   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5006   emit_int8(0x58);
5007   emit_operand(dst, src);
5008 }
5009 
5010 
5011 void Assembler::addps(XMMRegister dst, XMMRegister src) {
5012   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5013   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5014   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
5015   emit_int8(0x58);
5016   emit_int8((unsigned char)(0xC0 | encode));
5017 }
5018 
5019 void Assembler::vaddpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5020   assert(VM_Version::supports_avx(), "");
5021   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5022   attributes.set_rex_vex_w_reverted();
5023   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5024   emit_int8(0x58);
5025   emit_int8((unsigned char)(0xC0 | encode));
5026 }
5027 
5028 void Assembler::vaddps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5029   assert(VM_Version::supports_avx(), "");
5030   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5031   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
5032   emit_int8(0x58);
5033   emit_int8((unsigned char)(0xC0 | encode));
5034 }
5035 
5036 void Assembler::vaddpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5037   assert(VM_Version::supports_avx(), "");
5038   InstructionMark im(this);
5039   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5040   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
5041   attributes.set_rex_vex_w_reverted();
5042   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5043   emit_int8(0x58);
5044   emit_operand(dst, src);
5045 }
5046 
5047 void Assembler::vaddps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5048   assert(VM_Version::supports_avx(), "");
5049   InstructionMark im(this);
5050   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5051   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
5052   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
5053   emit_int8(0x58);
5054   emit_operand(dst, src);
5055 }
5056 
5057 void Assembler::subpd(XMMRegister dst, XMMRegister src) {
5058   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5059   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5060   attributes.set_rex_vex_w_reverted();
5061   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5062   emit_int8(0x5C);
5063   emit_int8((unsigned char)(0xC0 | encode));
5064 }
5065 
5066 void Assembler::subps(XMMRegister dst, XMMRegister src) {
5067   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5068   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5069   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
5070   emit_int8(0x5C);
5071   emit_int8((unsigned char)(0xC0 | encode));
5072 }
5073 
5074 void Assembler::vsubpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5075   assert(VM_Version::supports_avx(), "");
5076   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5077   attributes.set_rex_vex_w_reverted();
5078   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5079   emit_int8(0x5C);
5080   emit_int8((unsigned char)(0xC0 | encode));
5081 }
5082 
5083 void Assembler::vsubps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5084   assert(VM_Version::supports_avx(), "");
5085   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5086   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
5087   emit_int8(0x5C);
5088   emit_int8((unsigned char)(0xC0 | encode));
5089 }
5090 
5091 void Assembler::vsubpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5092   assert(VM_Version::supports_avx(), "");
5093   InstructionMark im(this);
5094   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5095   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
5096   attributes.set_rex_vex_w_reverted();
5097   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5098   emit_int8(0x5C);
5099   emit_operand(dst, src);
5100 }
5101 
5102 void Assembler::vsubps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5103   assert(VM_Version::supports_avx(), "");
5104   InstructionMark im(this);
5105   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5106   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
5107   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
5108   emit_int8(0x5C);
5109   emit_operand(dst, src);
5110 }
5111 
5112 void Assembler::mulpd(XMMRegister dst, XMMRegister src) {
5113   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5114   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5115   attributes.set_rex_vex_w_reverted();
5116   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5117   emit_int8(0x59);
5118   emit_int8((unsigned char)(0xC0 | encode));
5119 }
5120 
5121 void Assembler::mulpd(XMMRegister dst, Address src) {
5122   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5123   InstructionMark im(this);
5124   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5125   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
5126   attributes.set_rex_vex_w_reverted();
5127   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5128   emit_int8(0x59);
5129   emit_operand(dst, src);
5130 }
5131 
5132 void Assembler::mulps(XMMRegister dst, XMMRegister src) {
5133   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5134   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5135   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
5136   emit_int8(0x59);
5137   emit_int8((unsigned char)(0xC0 | encode));
5138 }
5139 
5140 void Assembler::vmulpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5141   assert(VM_Version::supports_avx(), "");
5142   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5143   attributes.set_rex_vex_w_reverted();
5144   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5145   emit_int8(0x59);
5146   emit_int8((unsigned char)(0xC0 | encode));
5147 }
5148 
5149 void Assembler::vmulps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5150   assert(VM_Version::supports_avx(), "");
5151   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5152   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
5153   emit_int8(0x59);
5154   emit_int8((unsigned char)(0xC0 | encode));
5155 }
5156 
5157 void Assembler::vmulpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5158   assert(VM_Version::supports_avx(), "");
5159   InstructionMark im(this);
5160   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5161   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
5162   attributes.set_rex_vex_w_reverted();
5163   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5164   emit_int8(0x59);
5165   emit_operand(dst, src);
5166 }
5167 
5168 void Assembler::vmulps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5169   assert(VM_Version::supports_avx(), "");
5170   InstructionMark im(this);
5171   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5172   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
5173   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
5174   emit_int8(0x59);
5175   emit_operand(dst, src);
5176 }
5177 
5178 void Assembler::vfmadd231pd(XMMRegister dst, XMMRegister src1, XMMRegister src2, int vector_len) {
5179   assert(VM_Version::supports_fma(), "");
5180   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5181   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5182   emit_int8((unsigned char)0xB8);
5183   emit_int8((unsigned char)(0xC0 | encode));
5184 }
5185 
5186 void Assembler::vfmadd231ps(XMMRegister dst, XMMRegister src1, XMMRegister src2, int vector_len) {
5187   assert(VM_Version::supports_fma(), "");
5188   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5189   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5190   emit_int8((unsigned char)0xB8);
5191   emit_int8((unsigned char)(0xC0 | encode));
5192 }
5193 
5194 void Assembler::vfmadd231pd(XMMRegister dst, XMMRegister src1, Address src2, int vector_len) {
5195   assert(VM_Version::supports_fma(), "");
5196   InstructionMark im(this);
5197   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5198   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
5199   vex_prefix(src2, src1->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5200   emit_int8((unsigned char)0xB8);
5201   emit_operand(dst, src2);
5202 }
5203 
5204 void Assembler::vfmadd231ps(XMMRegister dst, XMMRegister src1, Address src2, int vector_len) {
5205   assert(VM_Version::supports_fma(), "");
5206   InstructionMark im(this);
5207   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5208   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
5209   vex_prefix(src2, src1->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5210   emit_int8((unsigned char)0xB8);
5211   emit_operand(dst, src2);
5212 }
5213 
5214 void Assembler::divpd(XMMRegister dst, XMMRegister src) {
5215   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5216   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5217   attributes.set_rex_vex_w_reverted();
5218   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5219   emit_int8(0x5E);
5220   emit_int8((unsigned char)(0xC0 | encode));
5221 }
5222 
5223 void Assembler::divps(XMMRegister dst, XMMRegister src) {
5224   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5225   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5226   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
5227   emit_int8(0x5E);
5228   emit_int8((unsigned char)(0xC0 | encode));
5229 }
5230 
5231 void Assembler::vdivpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5232   assert(VM_Version::supports_avx(), "");
5233   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5234   attributes.set_rex_vex_w_reverted();
5235   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5236   emit_int8(0x5E);
5237   emit_int8((unsigned char)(0xC0 | encode));
5238 }
5239 
5240 void Assembler::vdivps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5241   assert(VM_Version::supports_avx(), "");
5242   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5243   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
5244   emit_int8(0x5E);
5245   emit_int8((unsigned char)(0xC0 | encode));
5246 }
5247 
5248 void Assembler::vdivpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5249   assert(VM_Version::supports_avx(), "");
5250   InstructionMark im(this);
5251   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5252   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
5253   attributes.set_rex_vex_w_reverted();
5254   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5255   emit_int8(0x5E);
5256   emit_operand(dst, src);
5257 }
5258 
5259 void Assembler::vdivps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5260   assert(VM_Version::supports_avx(), "");
5261   InstructionMark im(this);
5262   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5263   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
5264   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
5265   emit_int8(0x5E);
5266   emit_operand(dst, src);
5267 }
5268 
5269 void Assembler::vsqrtpd(XMMRegister dst, XMMRegister src, int vector_len) {
5270   assert(VM_Version::supports_avx(), "");
5271   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5272   attributes.set_rex_vex_w_reverted();
5273   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5274   emit_int8(0x51);
5275   emit_int8((unsigned char)(0xC0 | encode));
5276 }
5277 
5278 void Assembler::vsqrtpd(XMMRegister dst, Address src, int vector_len) {
5279   assert(VM_Version::supports_avx(), "");
5280   InstructionMark im(this);
5281   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5282   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
5283   attributes.set_rex_vex_w_reverted();
5284   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5285   emit_int8(0x51);
5286   emit_operand(dst, src);
5287 }
5288 
5289 void Assembler::vsqrtps(XMMRegister dst, XMMRegister src, int vector_len) {
5290   assert(VM_Version::supports_avx(), "");
5291   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5292   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
5293   emit_int8(0x51);
5294   emit_int8((unsigned char)(0xC0 | encode));
5295 }
5296 
5297 void Assembler::vsqrtps(XMMRegister dst, Address src, int vector_len) {
5298   assert(VM_Version::supports_avx(), "");
5299   InstructionMark im(this);
5300   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5301   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
5302   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
5303   emit_int8(0x51);
5304   emit_operand(dst, src);
5305 }
5306 
5307 void Assembler::andpd(XMMRegister dst, XMMRegister src) {
5308   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5309   InstructionAttr attributes(AVX_128bit, /* rex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
5310   attributes.set_rex_vex_w_reverted();
5311   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5312   emit_int8(0x54);
5313   emit_int8((unsigned char)(0xC0 | encode));
5314 }
5315 
5316 void Assembler::andps(XMMRegister dst, XMMRegister src) {
5317   NOT_LP64(assert(VM_Version::supports_sse(), ""));
5318   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
5319   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
5320   emit_int8(0x54);
5321   emit_int8((unsigned char)(0xC0 | encode));
5322 }
5323 
5324 void Assembler::andps(XMMRegister dst, Address src) {
5325   NOT_LP64(assert(VM_Version::supports_sse(), ""));
5326   InstructionMark im(this);
5327   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
5328   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
5329   simd_prefix(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
5330   emit_int8(0x54);
5331   emit_operand(dst, src);
5332 }
5333 
5334 void Assembler::andpd(XMMRegister dst, Address src) {
5335   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5336   InstructionMark im(this);
5337   InstructionAttr attributes(AVX_128bit, /* rex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
5338   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
5339   attributes.set_rex_vex_w_reverted();
5340   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5341   emit_int8(0x54);
5342   emit_operand(dst, src);
5343 }
5344 
5345 void Assembler::vandpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5346   assert(VM_Version::supports_avx(), "");
5347   InstructionAttr attributes(vector_len, /* vex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
5348   attributes.set_rex_vex_w_reverted();
5349   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5350   emit_int8(0x54);
5351   emit_int8((unsigned char)(0xC0 | encode));
5352 }
5353 
5354 void Assembler::vandps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5355   assert(VM_Version::supports_avx(), "");
5356   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
5357   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
5358   emit_int8(0x54);
5359   emit_int8((unsigned char)(0xC0 | encode));
5360 }
5361 
5362 void Assembler::vandpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5363   assert(VM_Version::supports_avx(), "");
5364   InstructionMark im(this);
5365   InstructionAttr attributes(vector_len, /* vex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
5366   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
5367   attributes.set_rex_vex_w_reverted();
5368   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5369   emit_int8(0x54);
5370   emit_operand(dst, src);
5371 }
5372 
5373 void Assembler::vandps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5374   assert(VM_Version::supports_avx(), "");
5375   InstructionMark im(this);
5376   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
5377   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
5378   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
5379   emit_int8(0x54);
5380   emit_operand(dst, src);
5381 }
5382 
5383 void Assembler::unpckhpd(XMMRegister dst, XMMRegister src) {
5384   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5385   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5386   attributes.set_rex_vex_w_reverted();
5387   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5388   emit_int8(0x15);
5389   emit_int8((unsigned char)(0xC0 | encode));
5390 }
5391 
5392 void Assembler::unpcklpd(XMMRegister dst, XMMRegister src) {
5393   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5394   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5395   attributes.set_rex_vex_w_reverted();
5396   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5397   emit_int8(0x14);
5398   emit_int8((unsigned char)(0xC0 | encode));
5399 }
5400 
5401 void Assembler::xorpd(XMMRegister dst, XMMRegister src) {
5402   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5403   InstructionAttr attributes(AVX_128bit, /* rex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
5404   attributes.set_rex_vex_w_reverted();
5405   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5406   emit_int8(0x57);
5407   emit_int8((unsigned char)(0xC0 | encode));
5408 }
5409 
5410 void Assembler::xorps(XMMRegister dst, XMMRegister src) {
5411   NOT_LP64(assert(VM_Version::supports_sse(), ""));
5412   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
5413   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
5414   emit_int8(0x57);
5415   emit_int8((unsigned char)(0xC0 | encode));
5416 }
5417 
5418 void Assembler::xorpd(XMMRegister dst, Address src) {
5419   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5420   InstructionMark im(this);
5421   InstructionAttr attributes(AVX_128bit, /* rex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
5422   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
5423   attributes.set_rex_vex_w_reverted();
5424   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5425   emit_int8(0x57);
5426   emit_operand(dst, src);
5427 }
5428 
5429 void Assembler::xorps(XMMRegister dst, Address src) {
5430   NOT_LP64(assert(VM_Version::supports_sse(), ""));
5431   InstructionMark im(this);
5432   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
5433   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
5434   simd_prefix(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
5435   emit_int8(0x57);
5436   emit_operand(dst, src);
5437 }
5438 
5439 void Assembler::vxorpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5440   assert(VM_Version::supports_avx(), "");
5441   InstructionAttr attributes(vector_len, /* vex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
5442   attributes.set_rex_vex_w_reverted();
5443   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5444   emit_int8(0x57);
5445   emit_int8((unsigned char)(0xC0 | encode));
5446 }
5447 
5448 void Assembler::vxorps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5449   assert(VM_Version::supports_avx(), "");
5450   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
5451   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
5452   emit_int8(0x57);
5453   emit_int8((unsigned char)(0xC0 | encode));
5454 }
5455 
5456 void Assembler::vxorpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5457   assert(VM_Version::supports_avx(), "");
5458   InstructionMark im(this);
5459   InstructionAttr attributes(vector_len, /* vex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
5460   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
5461   attributes.set_rex_vex_w_reverted();
5462   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5463   emit_int8(0x57);
5464   emit_operand(dst, src);
5465 }
5466 
5467 void Assembler::vxorps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5468   assert(VM_Version::supports_avx(), "");
5469   InstructionMark im(this);
5470   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
5471   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
5472   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
5473   emit_int8(0x57);
5474   emit_operand(dst, src);
5475 }
5476 
5477 // Integer vector arithmetic
5478 void Assembler::vphaddw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5479   assert(VM_Version::supports_avx() && (vector_len == 0) ||
5480          VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
5481   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
5482   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5483   emit_int8(0x01);
5484   emit_int8((unsigned char)(0xC0 | encode));
5485 }
5486 
5487 void Assembler::vphaddd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5488   assert(VM_Version::supports_avx() && (vector_len == 0) ||
5489          VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
5490   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
5491   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5492   emit_int8(0x02);
5493   emit_int8((unsigned char)(0xC0 | encode));
5494 }
5495 
5496 void Assembler::paddb(XMMRegister dst, XMMRegister src) {
5497   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5498   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5499   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5500   emit_int8((unsigned char)0xFC);
5501   emit_int8((unsigned char)(0xC0 | encode));
5502 }
5503 
5504 void Assembler::paddw(XMMRegister dst, XMMRegister src) {
5505   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5506   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5507   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5508   emit_int8((unsigned char)0xFD);
5509   emit_int8((unsigned char)(0xC0 | encode));
5510 }
5511 
5512 void Assembler::paddd(XMMRegister dst, XMMRegister src) {
5513   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5514   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5515   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5516   emit_int8((unsigned char)0xFE);
5517   emit_int8((unsigned char)(0xC0 | encode));
5518 }
5519 
5520 void Assembler::paddd(XMMRegister dst, Address src) {
5521   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5522   InstructionMark im(this);
5523   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5524   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5525   emit_int8((unsigned char)0xFE);
5526   emit_operand(dst, src);
5527 }
5528 
5529 void Assembler::paddq(XMMRegister dst, XMMRegister src) {
5530   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5531   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5532   attributes.set_rex_vex_w_reverted();
5533   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5534   emit_int8((unsigned char)0xD4);
5535   emit_int8((unsigned char)(0xC0 | encode));
5536 }
5537 
5538 void Assembler::phaddw(XMMRegister dst, XMMRegister src) {
5539   assert(VM_Version::supports_sse3(), "");
5540   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
5541   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5542   emit_int8(0x01);
5543   emit_int8((unsigned char)(0xC0 | encode));
5544 }
5545 
5546 void Assembler::phaddd(XMMRegister dst, XMMRegister src) {
5547   assert(VM_Version::supports_sse3(), "");
5548   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
5549   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5550   emit_int8(0x02);
5551   emit_int8((unsigned char)(0xC0 | encode));
5552 }
5553 
5554 void Assembler::vpaddb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5555   assert(UseAVX > 0, "requires some form of AVX");
5556   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5557   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5558   emit_int8((unsigned char)0xFC);
5559   emit_int8((unsigned char)(0xC0 | encode));
5560 }
5561 
5562 void Assembler::vpaddw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5563   assert(UseAVX > 0, "requires some form of AVX");
5564   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5565   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5566   emit_int8((unsigned char)0xFD);
5567   emit_int8((unsigned char)(0xC0 | encode));
5568 }
5569 
5570 void Assembler::vpaddd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5571   assert(UseAVX > 0, "requires some form of AVX");
5572   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5573   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5574   emit_int8((unsigned char)0xFE);
5575   emit_int8((unsigned char)(0xC0 | encode));
5576 }
5577 
5578 void Assembler::vpaddq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5579   assert(UseAVX > 0, "requires some form of AVX");
5580   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5581   attributes.set_rex_vex_w_reverted();
5582   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5583   emit_int8((unsigned char)0xD4);
5584   emit_int8((unsigned char)(0xC0 | encode));
5585 }
5586 
5587 void Assembler::vpaddb(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5588   assert(UseAVX > 0, "requires some form of AVX");
5589   InstructionMark im(this);
5590   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5591   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
5592   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5593   emit_int8((unsigned char)0xFC);
5594   emit_operand(dst, src);
5595 }
5596 
5597 void Assembler::vpaddw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5598   assert(UseAVX > 0, "requires some form of AVX");
5599   InstructionMark im(this);
5600   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5601   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
5602   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5603   emit_int8((unsigned char)0xFD);
5604   emit_operand(dst, src);
5605 }
5606 
5607 void Assembler::vpaddd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5608   assert(UseAVX > 0, "requires some form of AVX");
5609   InstructionMark im(this);
5610   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5611   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
5612   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5613   emit_int8((unsigned char)0xFE);
5614   emit_operand(dst, src);
5615 }
5616 
5617 void Assembler::vpaddq(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5618   assert(UseAVX > 0, "requires some form of AVX");
5619   InstructionMark im(this);
5620   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5621   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
5622   attributes.set_rex_vex_w_reverted();
5623   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5624   emit_int8((unsigned char)0xD4);
5625   emit_operand(dst, src);
5626 }
5627 
5628 void Assembler::psubb(XMMRegister dst, XMMRegister src) {
5629   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5630   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5631   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5632   emit_int8((unsigned char)0xF8);
5633   emit_int8((unsigned char)(0xC0 | encode));
5634 }
5635 
5636 void Assembler::psubw(XMMRegister dst, XMMRegister src) {
5637   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5638   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5639   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5640   emit_int8((unsigned char)0xF9);
5641   emit_int8((unsigned char)(0xC0 | encode));
5642 }
5643 
5644 void Assembler::psubd(XMMRegister dst, XMMRegister src) {
5645   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5646   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5647   emit_int8((unsigned char)0xFA);
5648   emit_int8((unsigned char)(0xC0 | encode));
5649 }
5650 
5651 void Assembler::psubq(XMMRegister dst, XMMRegister src) {
5652   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5653   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5654   attributes.set_rex_vex_w_reverted();
5655   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5656   emit_int8((unsigned char)0xFB);
5657   emit_int8((unsigned char)(0xC0 | encode));
5658 }
5659 
5660 void Assembler::vpsubb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5661   assert(UseAVX > 0, "requires some form of AVX");
5662   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5663   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5664   emit_int8((unsigned char)0xF8);
5665   emit_int8((unsigned char)(0xC0 | encode));
5666 }
5667 
5668 void Assembler::vpsubw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5669   assert(UseAVX > 0, "requires some form of AVX");
5670   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5671   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5672   emit_int8((unsigned char)0xF9);
5673   emit_int8((unsigned char)(0xC0 | encode));
5674 }
5675 
5676 void Assembler::vpsubd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5677   assert(UseAVX > 0, "requires some form of AVX");
5678   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5679   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5680   emit_int8((unsigned char)0xFA);
5681   emit_int8((unsigned char)(0xC0 | encode));
5682 }
5683 
5684 void Assembler::vpsubq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5685   assert(UseAVX > 0, "requires some form of AVX");
5686   InstructionAttr attributes(vector_len, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5687   attributes.set_rex_vex_w_reverted();
5688   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5689   emit_int8((unsigned char)0xFB);
5690   emit_int8((unsigned char)(0xC0 | encode));
5691 }
5692 
5693 void Assembler::vpsubb(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5694   assert(UseAVX > 0, "requires some form of AVX");
5695   InstructionMark im(this);
5696   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5697   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
5698   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5699   emit_int8((unsigned char)0xF8);
5700   emit_operand(dst, src);
5701 }
5702 
5703 void Assembler::vpsubw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5704   assert(UseAVX > 0, "requires some form of AVX");
5705   InstructionMark im(this);
5706   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5707   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
5708   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5709   emit_int8((unsigned char)0xF9);
5710   emit_operand(dst, src);
5711 }
5712 
5713 void Assembler::vpsubd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5714   assert(UseAVX > 0, "requires some form of AVX");
5715   InstructionMark im(this);
5716   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5717   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
5718   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5719   emit_int8((unsigned char)0xFA);
5720   emit_operand(dst, src);
5721 }
5722 
5723 void Assembler::vpsubq(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5724   assert(UseAVX > 0, "requires some form of AVX");
5725   InstructionMark im(this);
5726   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5727   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
5728   attributes.set_rex_vex_w_reverted();
5729   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5730   emit_int8((unsigned char)0xFB);
5731   emit_operand(dst, src);
5732 }
5733 
5734 void Assembler::pmullw(XMMRegister dst, XMMRegister src) {
5735   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5736   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5737   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5738   emit_int8((unsigned char)0xD5);
5739   emit_int8((unsigned char)(0xC0 | encode));
5740 }
5741 
5742 void Assembler::pmulld(XMMRegister dst, XMMRegister src) {
5743   assert(VM_Version::supports_sse4_1(), "");
5744   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5745   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5746   emit_int8(0x40);
5747   emit_int8((unsigned char)(0xC0 | encode));
5748 }
5749 
5750 void Assembler::vpmullw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5751   assert(UseAVX > 0, "requires some form of AVX");
5752   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5753   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5754   emit_int8((unsigned char)0xD5);
5755   emit_int8((unsigned char)(0xC0 | encode));
5756 }
5757 
5758 void Assembler::vpmulld(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5759   assert(UseAVX > 0, "requires some form of AVX");
5760   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5761   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5762   emit_int8(0x40);
5763   emit_int8((unsigned char)(0xC0 | encode));
5764 }
5765 
5766 void Assembler::vpmullq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5767   assert(UseAVX > 2, "requires some form of EVEX");
5768   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
5769   attributes.set_is_evex_instruction();
5770   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5771   emit_int8(0x40);
5772   emit_int8((unsigned char)(0xC0 | encode));
5773 }
5774 
5775 void Assembler::vpmullw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5776   assert(UseAVX > 0, "requires some form of AVX");
5777   InstructionMark im(this);
5778   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5779   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
5780   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5781   emit_int8((unsigned char)0xD5);
5782   emit_operand(dst, src);
5783 }
5784 
5785 void Assembler::vpmulld(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5786   assert(UseAVX > 0, "requires some form of AVX");
5787   InstructionMark im(this);
5788   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5789   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
5790   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5791   emit_int8(0x40);
5792   emit_operand(dst, src);
5793 }
5794 
5795 void Assembler::vpmullq(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5796   assert(UseAVX > 2, "requires some form of EVEX");
5797   InstructionMark im(this);
5798   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
5799   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
5800   attributes.set_is_evex_instruction();
5801   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5802   emit_int8(0x40);
5803   emit_operand(dst, src);
5804 }
5805 
5806 // Shift packed integers left by specified number of bits.
5807 void Assembler::psllw(XMMRegister dst, int shift) {
5808   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5809   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5810   // XMM6 is for /6 encoding: 66 0F 71 /6 ib
5811   int encode = simd_prefix_and_encode(xmm6, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5812   emit_int8(0x71);
5813   emit_int8((unsigned char)(0xC0 | encode));
5814   emit_int8(shift & 0xFF);
5815 }
5816 
5817 void Assembler::pslld(XMMRegister dst, int shift) {
5818   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5819   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5820   // XMM6 is for /6 encoding: 66 0F 72 /6 ib
5821   int encode = simd_prefix_and_encode(xmm6, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5822   emit_int8(0x72);
5823   emit_int8((unsigned char)(0xC0 | encode));
5824   emit_int8(shift & 0xFF);
5825 }
5826 
5827 void Assembler::psllq(XMMRegister dst, int shift) {
5828   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5829   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5830   // XMM6 is for /6 encoding: 66 0F 73 /6 ib
5831   int encode = simd_prefix_and_encode(xmm6, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5832   emit_int8(0x73);
5833   emit_int8((unsigned char)(0xC0 | encode));
5834   emit_int8(shift & 0xFF);
5835 }
5836 
5837 void Assembler::psllw(XMMRegister dst, XMMRegister shift) {
5838   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5839   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5840   int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5841   emit_int8((unsigned char)0xF1);
5842   emit_int8((unsigned char)(0xC0 | encode));
5843 }
5844 
5845 void Assembler::pslld(XMMRegister dst, XMMRegister shift) {
5846   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5847   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5848   int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5849   emit_int8((unsigned char)0xF2);
5850   emit_int8((unsigned char)(0xC0 | encode));
5851 }
5852 
5853 void Assembler::psllq(XMMRegister dst, XMMRegister shift) {
5854   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5855   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5856   attributes.set_rex_vex_w_reverted();
5857   int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5858   emit_int8((unsigned char)0xF3);
5859   emit_int8((unsigned char)(0xC0 | encode));
5860 }
5861 
5862 void Assembler::vpsllw(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
5863   assert(UseAVX > 0, "requires some form of AVX");
5864   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5865   // XMM6 is for /6 encoding: 66 0F 71 /6 ib
5866   int encode = vex_prefix_and_encode(xmm6->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5867   emit_int8(0x71);
5868   emit_int8((unsigned char)(0xC0 | encode));
5869   emit_int8(shift & 0xFF);
5870 }
5871 
5872 void Assembler::vpslld(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
5873   assert(UseAVX > 0, "requires some form of AVX");
5874   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5875   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5876   // XMM6 is for /6 encoding: 66 0F 72 /6 ib
5877   int encode = vex_prefix_and_encode(xmm6->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5878   emit_int8(0x72);
5879   emit_int8((unsigned char)(0xC0 | encode));
5880   emit_int8(shift & 0xFF);
5881 }
5882 
5883 void Assembler::vpsllq(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
5884   assert(UseAVX > 0, "requires some form of AVX");
5885   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5886   attributes.set_rex_vex_w_reverted();
5887   // XMM6 is for /6 encoding: 66 0F 73 /6 ib
5888   int encode = vex_prefix_and_encode(xmm6->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5889   emit_int8(0x73);
5890   emit_int8((unsigned char)(0xC0 | encode));
5891   emit_int8(shift & 0xFF);
5892 }
5893 
5894 void Assembler::vpsllw(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
5895   assert(UseAVX > 0, "requires some form of AVX");
5896   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5897   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5898   emit_int8((unsigned char)0xF1);
5899   emit_int8((unsigned char)(0xC0 | encode));
5900 }
5901 
5902 void Assembler::vpslld(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
5903   assert(UseAVX > 0, "requires some form of AVX");
5904   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5905   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5906   emit_int8((unsigned char)0xF2);
5907   emit_int8((unsigned char)(0xC0 | encode));
5908 }
5909 
5910 void Assembler::vpsllq(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
5911   assert(UseAVX > 0, "requires some form of AVX");
5912   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5913   attributes.set_rex_vex_w_reverted();
5914   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5915   emit_int8((unsigned char)0xF3);
5916   emit_int8((unsigned char)(0xC0 | encode));
5917 }
5918 
5919 // Shift packed integers logically right by specified number of bits.
5920 void Assembler::psrlw(XMMRegister dst, int shift) {
5921   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5922   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5923   // XMM2 is for /2 encoding: 66 0F 71 /2 ib
5924   int encode = simd_prefix_and_encode(xmm2, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5925   emit_int8(0x71);
5926   emit_int8((unsigned char)(0xC0 | encode));
5927   emit_int8(shift & 0xFF);
5928 }
5929 
5930 void Assembler::psrld(XMMRegister dst, int shift) {
5931   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5932   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5933   // XMM2 is for /2 encoding: 66 0F 72 /2 ib
5934   int encode = simd_prefix_and_encode(xmm2, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5935   emit_int8(0x72);
5936   emit_int8((unsigned char)(0xC0 | encode));
5937   emit_int8(shift & 0xFF);
5938 }
5939 
5940 void Assembler::psrlq(XMMRegister dst, int shift) {
5941   // Do not confuse it with psrldq SSE2 instruction which
5942   // shifts 128 bit value in xmm register by number of bytes.
5943   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5944   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5945   attributes.set_rex_vex_w_reverted();
5946   // XMM2 is for /2 encoding: 66 0F 73 /2 ib
5947   int encode = simd_prefix_and_encode(xmm2, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5948   emit_int8(0x73);
5949   emit_int8((unsigned char)(0xC0 | encode));
5950   emit_int8(shift & 0xFF);
5951 }
5952 
5953 void Assembler::psrlw(XMMRegister dst, XMMRegister shift) {
5954   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5955   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5956   int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5957   emit_int8((unsigned char)0xD1);
5958   emit_int8((unsigned char)(0xC0 | encode));
5959 }
5960 
5961 void Assembler::psrld(XMMRegister dst, XMMRegister shift) {
5962   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5963   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5964   int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5965   emit_int8((unsigned char)0xD2);
5966   emit_int8((unsigned char)(0xC0 | encode));
5967 }
5968 
5969 void Assembler::psrlq(XMMRegister dst, XMMRegister shift) {
5970   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5971   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5972   attributes.set_rex_vex_w_reverted();
5973   int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5974   emit_int8((unsigned char)0xD3);
5975   emit_int8((unsigned char)(0xC0 | encode));
5976 }
5977 
5978 void Assembler::vpsrlw(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
5979   assert(UseAVX > 0, "requires some form of AVX");
5980   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5981   // XMM2 is for /2 encoding: 66 0F 71 /2 ib
5982   int encode = vex_prefix_and_encode(xmm2->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5983   emit_int8(0x71);
5984   emit_int8((unsigned char)(0xC0 | encode));
5985   emit_int8(shift & 0xFF);
5986 }
5987 
5988 void Assembler::vpsrld(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
5989   assert(UseAVX > 0, "requires some form of AVX");
5990   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5991   // XMM2 is for /2 encoding: 66 0F 72 /2 ib
5992   int encode = vex_prefix_and_encode(xmm2->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5993   emit_int8(0x72);
5994   emit_int8((unsigned char)(0xC0 | encode));
5995   emit_int8(shift & 0xFF);
5996 }
5997 
5998 void Assembler::vpsrlq(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
5999   assert(UseAVX > 0, "requires some form of AVX");
6000   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6001   attributes.set_rex_vex_w_reverted();
6002   // XMM2 is for /2 encoding: 66 0F 73 /2 ib
6003   int encode = vex_prefix_and_encode(xmm2->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6004   emit_int8(0x73);
6005   emit_int8((unsigned char)(0xC0 | encode));
6006   emit_int8(shift & 0xFF);
6007 }
6008 
6009 void Assembler::vpsrlw(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
6010   assert(UseAVX > 0, "requires some form of AVX");
6011   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
6012   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6013   emit_int8((unsigned char)0xD1);
6014   emit_int8((unsigned char)(0xC0 | encode));
6015 }
6016 
6017 void Assembler::vpsrld(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
6018   assert(UseAVX > 0, "requires some form of AVX");
6019   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6020   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6021   emit_int8((unsigned char)0xD2);
6022   emit_int8((unsigned char)(0xC0 | encode));
6023 }
6024 
6025 void Assembler::vpsrlq(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
6026   assert(UseAVX > 0, "requires some form of AVX");
6027   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6028   attributes.set_rex_vex_w_reverted();
6029   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6030   emit_int8((unsigned char)0xD3);
6031   emit_int8((unsigned char)(0xC0 | encode));
6032 }
6033 
6034 // Shift packed integers arithmetically right by specified number of bits.
6035 void Assembler::psraw(XMMRegister dst, int shift) {
6036   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
6037   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
6038   // XMM4 is for /4 encoding: 66 0F 71 /4 ib
6039   int encode = simd_prefix_and_encode(xmm4, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6040   emit_int8(0x71);
6041   emit_int8((unsigned char)(0xC0 | encode));
6042   emit_int8(shift & 0xFF);
6043 }
6044 
6045 void Assembler::psrad(XMMRegister dst, int shift) {
6046   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
6047   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6048   // XMM4 is for /4 encoding: 66 0F 72 /4 ib
6049   int encode = simd_prefix_and_encode(xmm4, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6050   emit_int8(0x72);
6051   emit_int8((unsigned char)(0xC0 | encode));
6052   emit_int8(shift & 0xFF);
6053 }
6054 
6055 void Assembler::psraw(XMMRegister dst, XMMRegister shift) {
6056   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
6057   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
6058   int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6059   emit_int8((unsigned char)0xE1);
6060   emit_int8((unsigned char)(0xC0 | encode));
6061 }
6062 
6063 void Assembler::psrad(XMMRegister dst, XMMRegister shift) {
6064   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
6065   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6066   int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6067   emit_int8((unsigned char)0xE2);
6068   emit_int8((unsigned char)(0xC0 | encode));
6069 }
6070 
6071 void Assembler::vpsraw(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
6072   assert(UseAVX > 0, "requires some form of AVX");
6073   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
6074   // XMM4 is for /4 encoding: 66 0F 71 /4 ib
6075   int encode = vex_prefix_and_encode(xmm4->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6076   emit_int8(0x71);
6077   emit_int8((unsigned char)(0xC0 | encode));
6078   emit_int8(shift & 0xFF);
6079 }
6080 
6081 void Assembler::vpsrad(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
6082   assert(UseAVX > 0, "requires some form of AVX");
6083   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6084   // XMM4 is for /4 encoding: 66 0F 71 /4 ib
6085   int encode = vex_prefix_and_encode(xmm4->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6086   emit_int8(0x72);
6087   emit_int8((unsigned char)(0xC0 | encode));
6088   emit_int8(shift & 0xFF);
6089 }
6090 
6091 void Assembler::vpsraw(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
6092   assert(UseAVX > 0, "requires some form of AVX");
6093   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
6094   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6095   emit_int8((unsigned char)0xE1);
6096   emit_int8((unsigned char)(0xC0 | encode));
6097 }
6098 
6099 void Assembler::vpsrad(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
6100   assert(UseAVX > 0, "requires some form of AVX");
6101   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6102   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6103   emit_int8((unsigned char)0xE2);
6104   emit_int8((unsigned char)(0xC0 | encode));
6105 }
6106 
6107 
6108 // logical operations packed integers
6109 void Assembler::pand(XMMRegister dst, XMMRegister src) {
6110   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
6111   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6112   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6113   emit_int8((unsigned char)0xDB);
6114   emit_int8((unsigned char)(0xC0 | encode));
6115 }
6116 
6117 void Assembler::vpand(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
6118   assert(UseAVX > 0, "requires some form of AVX");
6119   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6120   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6121   emit_int8((unsigned char)0xDB);
6122   emit_int8((unsigned char)(0xC0 | encode));
6123 }
6124 
6125 void Assembler::vpand(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
6126   assert(UseAVX > 0, "requires some form of AVX");
6127   InstructionMark im(this);
6128   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6129   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
6130   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6131   emit_int8((unsigned char)0xDB);
6132   emit_operand(dst, src);
6133 }
6134 
6135 void Assembler::pandn(XMMRegister dst, XMMRegister src) {
6136   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
6137   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6138   attributes.set_rex_vex_w_reverted();
6139   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6140   emit_int8((unsigned char)0xDF);
6141   emit_int8((unsigned char)(0xC0 | encode));
6142 }
6143 
6144 void Assembler::por(XMMRegister dst, XMMRegister src) {
6145   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
6146   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6147   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6148   emit_int8((unsigned char)0xEB);
6149   emit_int8((unsigned char)(0xC0 | encode));
6150 }
6151 
6152 void Assembler::vpor(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
6153   assert(UseAVX > 0, "requires some form of AVX");
6154   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6155   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6156   emit_int8((unsigned char)0xEB);
6157   emit_int8((unsigned char)(0xC0 | encode));
6158 }
6159 
6160 void Assembler::vpor(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
6161   assert(UseAVX > 0, "requires some form of AVX");
6162   InstructionMark im(this);
6163   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6164   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
6165   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6166   emit_int8((unsigned char)0xEB);
6167   emit_operand(dst, src);
6168 }
6169 
6170 void Assembler::pxor(XMMRegister dst, XMMRegister src) {
6171   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
6172   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6173   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6174   emit_int8((unsigned char)0xEF);
6175   emit_int8((unsigned char)(0xC0 | encode));
6176 }
6177 
6178 void Assembler::vpxor(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
6179   assert(UseAVX > 0, "requires some form of AVX");
6180   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6181   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6182   emit_int8((unsigned char)0xEF);
6183   emit_int8((unsigned char)(0xC0 | encode));
6184 }
6185 
6186 void Assembler::vpxor(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
6187   assert(UseAVX > 0, "requires some form of AVX");
6188   InstructionMark im(this);
6189   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6190   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
6191   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6192   emit_int8((unsigned char)0xEF);
6193   emit_operand(dst, src);
6194 }
6195 
6196 
6197 // vinserti forms
6198 
6199 void Assembler::vinserti128(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) {
6200   assert(VM_Version::supports_avx2(), "");
6201   assert(imm8 <= 0x01, "imm8: %u", imm8);
6202   int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_256bit;
6203   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6204   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6205   emit_int8(0x38);
6206   emit_int8((unsigned char)(0xC0 | encode));
6207   // 0x00 - insert into lower 128 bits
6208   // 0x01 - insert into upper 128 bits
6209   emit_int8(imm8 & 0x01);
6210 }
6211 
6212 void Assembler::vinserti128(XMMRegister dst, XMMRegister nds, Address src, uint8_t imm8) {
6213   assert(VM_Version::supports_avx2(), "");
6214   assert(dst != xnoreg, "sanity");
6215   assert(imm8 <= 0x01, "imm8: %u", imm8);
6216   int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_256bit;
6217   InstructionMark im(this);
6218   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6219   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
6220   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6221   emit_int8(0x38);
6222   emit_operand(dst, src);
6223   // 0x00 - insert into lower 128 bits
6224   // 0x01 - insert into upper 128 bits
6225   emit_int8(imm8 & 0x01);
6226 }
6227 
6228 void Assembler::vinserti32x4(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) {
6229   assert(VM_Version::supports_evex(), "");
6230   assert(imm8 <= 0x03, "imm8: %u", imm8);
6231   InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6232   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6233   emit_int8(0x38);
6234   emit_int8((unsigned char)(0xC0 | encode));
6235   // 0x00 - insert into q0 128 bits (0..127)
6236   // 0x01 - insert into q1 128 bits (128..255)
6237   // 0x02 - insert into q2 128 bits (256..383)
6238   // 0x03 - insert into q3 128 bits (384..511)
6239   emit_int8(imm8 & 0x03);
6240 }
6241 
6242 void Assembler::vinserti32x4(XMMRegister dst, XMMRegister nds, Address src, uint8_t imm8) {
6243   assert(VM_Version::supports_avx(), "");
6244   assert(dst != xnoreg, "sanity");
6245   assert(imm8 <= 0x03, "imm8: %u", imm8);
6246   int vector_len = VM_Version::supports_evex() ? AVX_512bit : AVX_256bit;
6247   InstructionMark im(this);
6248   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6249   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
6250   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6251   emit_int8(0x18);
6252   emit_operand(dst, src);
6253   // 0x00 - insert into q0 128 bits (0..127)
6254   // 0x01 - insert into q1 128 bits (128..255)
6255   // 0x02 - insert into q2 128 bits (256..383)
6256   // 0x03 - insert into q3 128 bits (384..511)
6257   emit_int8(imm8 & 0x03);
6258 }
6259 
6260 void Assembler::vinserti64x4(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) {
6261   assert(VM_Version::supports_evex(), "");
6262   assert(imm8 <= 0x01, "imm8: %u", imm8);
6263   InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6264   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6265   emit_int8(0x38);
6266   emit_int8((unsigned char)(0xC0 | encode));
6267   // 0x00 - insert into lower 256 bits
6268   // 0x01 - insert into upper 256 bits
6269   emit_int8(imm8 & 0x01);
6270 }
6271 
6272 
6273 // vinsertf forms
6274 
6275 void Assembler::vinsertf128(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) {
6276   assert(VM_Version::supports_avx(), "");
6277   assert(imm8 <= 0x01, "imm8: %u", imm8);
6278   int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_256bit;
6279   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6280   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6281   emit_int8(0x18);
6282   emit_int8((unsigned char)(0xC0 | encode));
6283   // 0x00 - insert into lower 128 bits
6284   // 0x01 - insert into upper 128 bits
6285   emit_int8(imm8 & 0x01);
6286 }
6287 
6288 void Assembler::vinsertf128(XMMRegister dst, XMMRegister nds, Address src, uint8_t imm8) {
6289   assert(VM_Version::supports_avx(), "");
6290   assert(dst != xnoreg, "sanity");
6291   assert(imm8 <= 0x01, "imm8: %u", imm8);
6292   int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_256bit;
6293   InstructionMark im(this);
6294   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6295   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
6296   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6297   emit_int8(0x18);
6298   emit_operand(dst, src);
6299   // 0x00 - insert into lower 128 bits
6300   // 0x01 - insert into upper 128 bits
6301   emit_int8(imm8 & 0x01);
6302 }
6303 
6304 void Assembler::vinsertf32x4(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) {
6305   assert(VM_Version::supports_evex(), "");
6306   assert(imm8 <= 0x03, "imm8: %u", imm8);
6307   InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6308   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6309   emit_int8(0x18);
6310   emit_int8((unsigned char)(0xC0 | encode));
6311   // 0x00 - insert into q0 128 bits (0..127)
6312   // 0x01 - insert into q1 128 bits (128..255)
6313   // 0x02 - insert into q2 128 bits (256..383)
6314   // 0x03 - insert into q3 128 bits (384..511)
6315   emit_int8(imm8 & 0x03);
6316 }
6317 
6318 void Assembler::vinsertf32x4(XMMRegister dst, XMMRegister nds, Address src, uint8_t imm8) {
6319   assert(VM_Version::supports_avx(), "");
6320   assert(dst != xnoreg, "sanity");
6321   assert(imm8 <= 0x03, "imm8: %u", imm8);
6322   int vector_len = VM_Version::supports_evex() ? AVX_512bit : AVX_256bit;
6323   InstructionMark im(this);
6324   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6325   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
6326   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6327   emit_int8(0x18);
6328   emit_operand(dst, src);
6329   // 0x00 - insert into q0 128 bits (0..127)
6330   // 0x01 - insert into q1 128 bits (128..255)
6331   // 0x02 - insert into q2 128 bits (256..383)
6332   // 0x03 - insert into q3 128 bits (384..511)
6333   emit_int8(imm8 & 0x03);
6334 }
6335 
6336 void Assembler::vinsertf64x4(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) {
6337   assert(VM_Version::supports_evex(), "");
6338   assert(imm8 <= 0x01, "imm8: %u", imm8);
6339   InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6340   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6341   emit_int8(0x1A);
6342   emit_int8((unsigned char)(0xC0 | encode));
6343   // 0x00 - insert into lower 256 bits
6344   // 0x01 - insert into upper 256 bits
6345   emit_int8(imm8 & 0x01);
6346 }
6347 
6348 void Assembler::vinsertf64x4(XMMRegister dst, XMMRegister nds, Address src, uint8_t imm8) {
6349   assert(VM_Version::supports_evex(), "");
6350   assert(dst != xnoreg, "sanity");
6351   assert(imm8 <= 0x01, "imm8: %u", imm8);
6352   InstructionMark im(this);
6353   InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6354   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_64bit);
6355   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6356   emit_int8(0x1A);
6357   emit_operand(dst, src);
6358   // 0x00 - insert into lower 256 bits
6359   // 0x01 - insert into upper 256 bits
6360   emit_int8(imm8 & 0x01);
6361 }
6362 
6363 
6364 // vextracti forms
6365 
6366 void Assembler::vextracti128(XMMRegister dst, XMMRegister src, uint8_t imm8) {
6367   assert(VM_Version::supports_avx(), "");
6368   assert(imm8 <= 0x01, "imm8: %u", imm8);
6369   int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_256bit;
6370   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6371   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6372   emit_int8(0x39);
6373   emit_int8((unsigned char)(0xC0 | encode));
6374   // 0x00 - extract from lower 128 bits
6375   // 0x01 - extract from upper 128 bits
6376   emit_int8(imm8 & 0x01);
6377 }
6378 
6379 void Assembler::vextracti128(Address dst, XMMRegister src, uint8_t imm8) {
6380   assert(VM_Version::supports_avx2(), "");
6381   assert(src != xnoreg, "sanity");
6382   assert(imm8 <= 0x01, "imm8: %u", imm8);
6383   int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_256bit;
6384   InstructionMark im(this);
6385   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6386   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
6387   attributes.reset_is_clear_context();
6388   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6389   emit_int8(0x39);
6390   emit_operand(src, dst);
6391   // 0x00 - extract from lower 128 bits
6392   // 0x01 - extract from upper 128 bits
6393   emit_int8(imm8 & 0x01);
6394 }
6395 
6396 void Assembler::vextracti32x4(XMMRegister dst, XMMRegister src, uint8_t imm8) {
6397   assert(VM_Version::supports_avx(), "");
6398   assert(imm8 <= 0x03, "imm8: %u", imm8);
6399   int vector_len = VM_Version::supports_evex() ? AVX_512bit : AVX_256bit;
6400   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6401   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6402   emit_int8(0x39);
6403   emit_int8((unsigned char)(0xC0 | encode));
6404   // 0x00 - extract from bits 127:0
6405   // 0x01 - extract from bits 255:128
6406   // 0x02 - extract from bits 383:256
6407   // 0x03 - extract from bits 511:384
6408   emit_int8(imm8 & 0x03);
6409 }
6410 
6411 void Assembler::vextracti32x4(Address dst, XMMRegister src, uint8_t imm8) {
6412   assert(VM_Version::supports_evex(), "");
6413   assert(src != xnoreg, "sanity");
6414   assert(imm8 <= 0x03, "imm8: %u", imm8);
6415   InstructionMark im(this);
6416   InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6417   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
6418   attributes.reset_is_clear_context();
6419   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6420   emit_int8(0x39);
6421   emit_operand(src, dst);
6422   // 0x00 - extract from bits 127:0
6423   // 0x01 - extract from bits 255:128
6424   // 0x02 - extract from bits 383:256
6425   // 0x03 - extract from bits 511:384
6426   emit_int8(imm8 & 0x03);
6427 }
6428 
6429 void Assembler::vextracti64x2(XMMRegister dst, XMMRegister src, uint8_t imm8) {
6430   assert(VM_Version::supports_avx512dq(), "");
6431   assert(imm8 <= 0x03, "imm8: %u", imm8);
6432   InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6433   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6434   emit_int8(0x39);
6435   emit_int8((unsigned char)(0xC0 | encode));
6436   // 0x00 - extract from bits 127:0
6437   // 0x01 - extract from bits 255:128
6438   // 0x02 - extract from bits 383:256
6439   // 0x03 - extract from bits 511:384
6440   emit_int8(imm8 & 0x03);
6441 }
6442 
6443 void Assembler::vextracti64x4(XMMRegister dst, XMMRegister src, uint8_t imm8) {
6444   assert(VM_Version::supports_evex(), "");
6445   assert(imm8 <= 0x01, "imm8: %u", imm8);
6446   InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6447   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6448   emit_int8(0x3B);
6449   emit_int8((unsigned char)(0xC0 | encode));
6450   // 0x00 - extract from lower 256 bits
6451   // 0x01 - extract from upper 256 bits
6452   emit_int8(imm8 & 0x01);
6453 }
6454 
6455 
6456 // vextractf forms
6457 
6458 void Assembler::vextractf128(XMMRegister dst, XMMRegister src, uint8_t imm8) {
6459   assert(VM_Version::supports_avx(), "");
6460   assert(imm8 <= 0x01, "imm8: %u", imm8);
6461   int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_256bit;
6462   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6463   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6464   emit_int8(0x19);
6465   emit_int8((unsigned char)(0xC0 | encode));
6466   // 0x00 - extract from lower 128 bits
6467   // 0x01 - extract from upper 128 bits
6468   emit_int8(imm8 & 0x01);
6469 }
6470 
6471 void Assembler::vextractf128(Address dst, XMMRegister src, uint8_t imm8) {
6472   assert(VM_Version::supports_avx(), "");
6473   assert(src != xnoreg, "sanity");
6474   assert(imm8 <= 0x01, "imm8: %u", imm8);
6475   int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_256bit;
6476   InstructionMark im(this);
6477   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6478   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
6479   attributes.reset_is_clear_context();
6480   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6481   emit_int8(0x19);
6482   emit_operand(src, dst);
6483   // 0x00 - extract from lower 128 bits
6484   // 0x01 - extract from upper 128 bits
6485   emit_int8(imm8 & 0x01);
6486 }
6487 
6488 void Assembler::vextractf32x4(XMMRegister dst, XMMRegister src, uint8_t imm8) {
6489   assert(VM_Version::supports_avx(), "");
6490   assert(imm8 <= 0x03, "imm8: %u", imm8);
6491   int vector_len = VM_Version::supports_evex() ? AVX_512bit : AVX_256bit;
6492   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6493   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6494   emit_int8(0x19);
6495   emit_int8((unsigned char)(0xC0 | encode));
6496   // 0x00 - extract from bits 127:0
6497   // 0x01 - extract from bits 255:128
6498   // 0x02 - extract from bits 383:256
6499   // 0x03 - extract from bits 511:384
6500   emit_int8(imm8 & 0x03);
6501 }
6502 
6503 void Assembler::vextractf32x4(Address dst, XMMRegister src, uint8_t imm8) {
6504   assert(VM_Version::supports_evex(), "");
6505   assert(src != xnoreg, "sanity");
6506   assert(imm8 <= 0x03, "imm8: %u", imm8);
6507   InstructionMark im(this);
6508   InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6509   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
6510   attributes.reset_is_clear_context();
6511   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6512   emit_int8(0x19);
6513   emit_operand(src, dst);
6514   // 0x00 - extract from bits 127:0
6515   // 0x01 - extract from bits 255:128
6516   // 0x02 - extract from bits 383:256
6517   // 0x03 - extract from bits 511:384
6518   emit_int8(imm8 & 0x03);
6519 }
6520 
6521 void Assembler::vextractf64x2(XMMRegister dst, XMMRegister src, uint8_t imm8) {
6522   assert(VM_Version::supports_avx512dq(), "");
6523   assert(imm8 <= 0x03, "imm8: %u", imm8);
6524   InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6525   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6526   emit_int8(0x19);
6527   emit_int8((unsigned char)(0xC0 | encode));
6528   // 0x00 - extract from bits 127:0
6529   // 0x01 - extract from bits 255:128
6530   // 0x02 - extract from bits 383:256
6531   // 0x03 - extract from bits 511:384
6532   emit_int8(imm8 & 0x03);
6533 }
6534 
6535 void Assembler::vextractf64x4(XMMRegister dst, XMMRegister src, uint8_t imm8) {
6536   assert(VM_Version::supports_evex(), "");
6537   assert(imm8 <= 0x01, "imm8: %u", imm8);
6538   InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6539   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6540   emit_int8(0x1B);
6541   emit_int8((unsigned char)(0xC0 | encode));
6542   // 0x00 - extract from lower 256 bits
6543   // 0x01 - extract from upper 256 bits
6544   emit_int8(imm8 & 0x01);
6545 }
6546 
6547 void Assembler::vextractf64x4(Address dst, XMMRegister src, uint8_t imm8) {
6548   assert(VM_Version::supports_evex(), "");
6549   assert(src != xnoreg, "sanity");
6550   assert(imm8 <= 0x01, "imm8: %u", imm8);
6551   InstructionMark im(this);
6552   InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6553   attributes.set_address_attributes(/* tuple_type */ EVEX_T4,/* input_size_in_bits */  EVEX_64bit);
6554   attributes.reset_is_clear_context();
6555   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6556   emit_int8(0x1B);
6557   emit_operand(src, dst);
6558   // 0x00 - extract from lower 256 bits
6559   // 0x01 - extract from upper 256 bits
6560   emit_int8(imm8 & 0x01);
6561 }
6562 
6563 
6564 // legacy word/dword replicate
6565 void Assembler::vpbroadcastw(XMMRegister dst, XMMRegister src) {
6566   assert(VM_Version::supports_avx2(), "");
6567   InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
6568   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6569   emit_int8(0x79);
6570   emit_int8((unsigned char)(0xC0 | encode));
6571 }
6572 
6573 void Assembler::vpbroadcastd(XMMRegister dst, XMMRegister src) {
6574   assert(VM_Version::supports_avx2(), "");
6575   InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6576   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6577   emit_int8(0x58);
6578   emit_int8((unsigned char)(0xC0 | encode));
6579 }
6580 
6581 
6582 // xmm/mem sourced byte/word/dword/qword replicate
6583 
6584 // duplicate 1-byte integer data from src into programmed locations in dest : requires AVX512BW and AVX512VL
6585 void Assembler::evpbroadcastb(XMMRegister dst, XMMRegister src, int vector_len) {
6586   assert(VM_Version::supports_evex(), "");
6587   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
6588   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6589   emit_int8(0x78);
6590   emit_int8((unsigned char)(0xC0 | encode));
6591 }
6592 
6593 void Assembler::evpbroadcastb(XMMRegister dst, Address src, int vector_len) {
6594   assert(VM_Version::supports_evex(), "");
6595   assert(dst != xnoreg, "sanity");
6596   InstructionMark im(this);
6597   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
6598   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_8bit);
6599   // swap src<->dst for encoding
6600   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6601   emit_int8(0x78);
6602   emit_operand(dst, src);
6603 }
6604 
6605 // duplicate 2-byte integer data from src into programmed locations in dest : requires AVX512BW and AVX512VL
6606 void Assembler::evpbroadcastw(XMMRegister dst, XMMRegister src, int vector_len) {
6607   assert(VM_Version::supports_evex(), "");
6608   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
6609   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6610   emit_int8(0x79);
6611   emit_int8((unsigned char)(0xC0 | encode));
6612 }
6613 
6614 void Assembler::evpbroadcastw(XMMRegister dst, Address src, int vector_len) {
6615   assert(VM_Version::supports_evex(), "");
6616   assert(dst != xnoreg, "sanity");
6617   InstructionMark im(this);
6618   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
6619   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_16bit);
6620   // swap src<->dst for encoding
6621   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6622   emit_int8(0x79);
6623   emit_operand(dst, src);
6624 }
6625 
6626 // duplicate 4-byte integer data from src into programmed locations in dest : requires AVX512VL
6627 void Assembler::evpbroadcastd(XMMRegister dst, XMMRegister src, int vector_len) {
6628   assert(VM_Version::supports_evex(), "");
6629   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6630   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6631   emit_int8(0x58);
6632   emit_int8((unsigned char)(0xC0 | encode));
6633 }
6634 
6635 void Assembler::evpbroadcastd(XMMRegister dst, Address src, int vector_len) {
6636   assert(VM_Version::supports_evex(), "");
6637   assert(dst != xnoreg, "sanity");
6638   InstructionMark im(this);
6639   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6640   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
6641   // swap src<->dst for encoding
6642   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6643   emit_int8(0x58);
6644   emit_operand(dst, src);
6645 }
6646 
6647 // duplicate 8-byte integer data from src into programmed locations in dest : requires AVX512VL
6648 void Assembler::evpbroadcastq(XMMRegister dst, XMMRegister src, int vector_len) {
6649   assert(VM_Version::supports_evex(), "");
6650   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6651   attributes.set_rex_vex_w_reverted();
6652   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6653   emit_int8(0x59);
6654   emit_int8((unsigned char)(0xC0 | encode));
6655 }
6656 
6657 void Assembler::evpbroadcastq(XMMRegister dst, Address src, int vector_len) {
6658   assert(VM_Version::supports_evex(), "");
6659   assert(dst != xnoreg, "sanity");
6660   InstructionMark im(this);
6661   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6662   attributes.set_rex_vex_w_reverted();
6663   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
6664   // swap src<->dst for encoding
6665   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6666   emit_int8(0x59);
6667   emit_operand(dst, src);
6668 }
6669 
6670 
6671 // scalar single/double precision replicate
6672 
6673 // duplicate single precision data from src into programmed locations in dest : requires AVX512VL
6674 void Assembler::evpbroadcastss(XMMRegister dst, XMMRegister src, int vector_len) {
6675   assert(VM_Version::supports_evex(), "");
6676   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6677   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6678   emit_int8(0x18);
6679   emit_int8((unsigned char)(0xC0 | encode));
6680 }
6681 
6682 void Assembler::evpbroadcastss(XMMRegister dst, Address src, int vector_len) {
6683   assert(VM_Version::supports_evex(), "");
6684   assert(dst != xnoreg, "sanity");
6685   InstructionMark im(this);
6686   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6687   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
6688   // swap src<->dst for encoding
6689   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6690   emit_int8(0x18);
6691   emit_operand(dst, src);
6692 }
6693 
6694 // duplicate double precision data from src into programmed locations in dest : requires AVX512VL
6695 void Assembler::evpbroadcastsd(XMMRegister dst, XMMRegister src, int vector_len) {
6696   assert(VM_Version::supports_evex(), "");
6697   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6698   attributes.set_rex_vex_w_reverted();
6699   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6700   emit_int8(0x19);
6701   emit_int8((unsigned char)(0xC0 | encode));
6702 }
6703 
6704 void Assembler::evpbroadcastsd(XMMRegister dst, Address src, int vector_len) {
6705   assert(VM_Version::supports_evex(), "");
6706   assert(dst != xnoreg, "sanity");
6707   InstructionMark im(this);
6708   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6709   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
6710   attributes.set_rex_vex_w_reverted();
6711   // swap src<->dst for encoding
6712   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6713   emit_int8(0x19);
6714   emit_operand(dst, src);
6715 }
6716 
6717 
6718 // gpr source broadcast forms
6719 
6720 // duplicate 1-byte integer data from src into programmed locations in dest : requires AVX512BW and AVX512VL
6721 void Assembler::evpbroadcastb(XMMRegister dst, Register src, int vector_len) {
6722   assert(VM_Version::supports_evex(), "");
6723   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
6724   attributes.set_is_evex_instruction();
6725   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6726   emit_int8(0x7A);
6727   emit_int8((unsigned char)(0xC0 | encode));
6728 }
6729 
6730 // duplicate 2-byte integer data from src into programmed locations in dest : requires AVX512BW and AVX512VL
6731 void Assembler::evpbroadcastw(XMMRegister dst, Register src, int vector_len) {
6732   assert(VM_Version::supports_evex(), "");
6733   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
6734   attributes.set_is_evex_instruction();
6735   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6736   emit_int8(0x7B);
6737   emit_int8((unsigned char)(0xC0 | encode));
6738 }
6739 
6740 // duplicate 4-byte integer data from src into programmed locations in dest : requires AVX512VL
6741 void Assembler::evpbroadcastd(XMMRegister dst, Register src, int vector_len) {
6742   assert(VM_Version::supports_evex(), "");
6743   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6744   attributes.set_is_evex_instruction();
6745   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6746   emit_int8(0x7C);
6747   emit_int8((unsigned char)(0xC0 | encode));
6748 }
6749 
6750 // duplicate 8-byte integer data from src into programmed locations in dest : requires AVX512VL
6751 void Assembler::evpbroadcastq(XMMRegister dst, Register src, int vector_len) {
6752   assert(VM_Version::supports_evex(), "");
6753   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6754   attributes.set_is_evex_instruction();
6755   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6756   emit_int8(0x7C);
6757   emit_int8((unsigned char)(0xC0 | encode));
6758 }
6759 
6760 
6761 // Carry-Less Multiplication Quadword
6762 void Assembler::pclmulqdq(XMMRegister dst, XMMRegister src, int mask) {
6763   assert(VM_Version::supports_clmul(), "");
6764   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
6765   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6766   emit_int8(0x44);
6767   emit_int8((unsigned char)(0xC0 | encode));
6768   emit_int8((unsigned char)mask);
6769 }
6770 
6771 // Carry-Less Multiplication Quadword
6772 void Assembler::vpclmulqdq(XMMRegister dst, XMMRegister nds, XMMRegister src, int mask) {
6773   assert(VM_Version::supports_avx() && VM_Version::supports_clmul(), "");
6774   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
6775   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6776   emit_int8(0x44);
6777   emit_int8((unsigned char)(0xC0 | encode));
6778   emit_int8((unsigned char)mask);
6779 }
6780 
6781 void Assembler::vzeroupper() {
6782   if (VM_Version::supports_vzeroupper()) {
6783     InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
6784     (void)vex_prefix_and_encode(0, 0, 0, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
6785     emit_int8(0x77);
6786   }
6787 }
6788 
6789 #ifndef _LP64
6790 // 32bit only pieces of the assembler
6791 
6792 void Assembler::cmp_literal32(Register src1, int32_t imm32, RelocationHolder const& rspec) {
6793   // NO PREFIX AS NEVER 64BIT
6794   InstructionMark im(this);
6795   emit_int8((unsigned char)0x81);
6796   emit_int8((unsigned char)(0xF8 | src1->encoding()));
6797   emit_data(imm32, rspec, 0);
6798 }
6799 
6800 void Assembler::cmp_literal32(Address src1, int32_t imm32, RelocationHolder const& rspec) {
6801   // NO PREFIX AS NEVER 64BIT (not even 32bit versions of 64bit regs
6802   InstructionMark im(this);
6803   emit_int8((unsigned char)0x81);
6804   emit_operand(rdi, src1);
6805   emit_data(imm32, rspec, 0);
6806 }
6807 
6808 // The 64-bit (32bit platform) cmpxchg compares the value at adr with the contents of rdx:rax,
6809 // and stores rcx:rbx into adr if so; otherwise, the value at adr is loaded
6810 // into rdx:rax.  The ZF is set if the compared values were equal, and cleared otherwise.
6811 void Assembler::cmpxchg8(Address adr) {
6812   InstructionMark im(this);
6813   emit_int8(0x0F);
6814   emit_int8((unsigned char)0xC7);
6815   emit_operand(rcx, adr);
6816 }
6817 
6818 void Assembler::decl(Register dst) {
6819   // Don't use it directly. Use MacroAssembler::decrementl() instead.
6820  emit_int8(0x48 | dst->encoding());
6821 }
6822 
6823 #endif // _LP64
6824 
6825 // 64bit typically doesn't use the x87 but needs to for the trig funcs
6826 
6827 void Assembler::fabs() {
6828   emit_int8((unsigned char)0xD9);
6829   emit_int8((unsigned char)0xE1);
6830 }
6831 
6832 void Assembler::fadd(int i) {
6833   emit_farith(0xD8, 0xC0, i);
6834 }
6835 
6836 void Assembler::fadd_d(Address src) {
6837   InstructionMark im(this);
6838   emit_int8((unsigned char)0xDC);
6839   emit_operand32(rax, src);
6840 }
6841 
6842 void Assembler::fadd_s(Address src) {
6843   InstructionMark im(this);
6844   emit_int8((unsigned char)0xD8);
6845   emit_operand32(rax, src);
6846 }
6847 
6848 void Assembler::fadda(int i) {
6849   emit_farith(0xDC, 0xC0, i);
6850 }
6851 
6852 void Assembler::faddp(int i) {
6853   emit_farith(0xDE, 0xC0, i);
6854 }
6855 
6856 void Assembler::fchs() {
6857   emit_int8((unsigned char)0xD9);
6858   emit_int8((unsigned char)0xE0);
6859 }
6860 
6861 void Assembler::fcom(int i) {
6862   emit_farith(0xD8, 0xD0, i);
6863 }
6864 
6865 void Assembler::fcomp(int i) {
6866   emit_farith(0xD8, 0xD8, i);
6867 }
6868 
6869 void Assembler::fcomp_d(Address src) {
6870   InstructionMark im(this);
6871   emit_int8((unsigned char)0xDC);
6872   emit_operand32(rbx, src);
6873 }
6874 
6875 void Assembler::fcomp_s(Address src) {
6876   InstructionMark im(this);
6877   emit_int8((unsigned char)0xD8);
6878   emit_operand32(rbx, src);
6879 }
6880 
6881 void Assembler::fcompp() {
6882   emit_int8((unsigned char)0xDE);
6883   emit_int8((unsigned char)0xD9);
6884 }
6885 
6886 void Assembler::fcos() {
6887   emit_int8((unsigned char)0xD9);
6888   emit_int8((unsigned char)0xFF);
6889 }
6890 
6891 void Assembler::fdecstp() {
6892   emit_int8((unsigned char)0xD9);
6893   emit_int8((unsigned char)0xF6);
6894 }
6895 
6896 void Assembler::fdiv(int i) {
6897   emit_farith(0xD8, 0xF0, i);
6898 }
6899 
6900 void Assembler::fdiv_d(Address src) {
6901   InstructionMark im(this);
6902   emit_int8((unsigned char)0xDC);
6903   emit_operand32(rsi, src);
6904 }
6905 
6906 void Assembler::fdiv_s(Address src) {
6907   InstructionMark im(this);
6908   emit_int8((unsigned char)0xD8);
6909   emit_operand32(rsi, src);
6910 }
6911 
6912 void Assembler::fdiva(int i) {
6913   emit_farith(0xDC, 0xF8, i);
6914 }
6915 
6916 // Note: The Intel manual (Pentium Processor User's Manual, Vol.3, 1994)
6917 //       is erroneous for some of the floating-point instructions below.
6918 
6919 void Assembler::fdivp(int i) {
6920   emit_farith(0xDE, 0xF8, i);                    // ST(0) <- ST(0) / ST(1) and pop (Intel manual wrong)
6921 }
6922 
6923 void Assembler::fdivr(int i) {
6924   emit_farith(0xD8, 0xF8, i);
6925 }
6926 
6927 void Assembler::fdivr_d(Address src) {
6928   InstructionMark im(this);
6929   emit_int8((unsigned char)0xDC);
6930   emit_operand32(rdi, src);
6931 }
6932 
6933 void Assembler::fdivr_s(Address src) {
6934   InstructionMark im(this);
6935   emit_int8((unsigned char)0xD8);
6936   emit_operand32(rdi, src);
6937 }
6938 
6939 void Assembler::fdivra(int i) {
6940   emit_farith(0xDC, 0xF0, i);
6941 }
6942 
6943 void Assembler::fdivrp(int i) {
6944   emit_farith(0xDE, 0xF0, i);                    // ST(0) <- ST(1) / ST(0) and pop (Intel manual wrong)
6945 }
6946 
6947 void Assembler::ffree(int i) {
6948   emit_farith(0xDD, 0xC0, i);
6949 }
6950 
6951 void Assembler::fild_d(Address adr) {
6952   InstructionMark im(this);
6953   emit_int8((unsigned char)0xDF);
6954   emit_operand32(rbp, adr);
6955 }
6956 
6957 void Assembler::fild_s(Address adr) {
6958   InstructionMark im(this);
6959   emit_int8((unsigned char)0xDB);
6960   emit_operand32(rax, adr);
6961 }
6962 
6963 void Assembler::fincstp() {
6964   emit_int8((unsigned char)0xD9);
6965   emit_int8((unsigned char)0xF7);
6966 }
6967 
6968 void Assembler::finit() {
6969   emit_int8((unsigned char)0x9B);
6970   emit_int8((unsigned char)0xDB);
6971   emit_int8((unsigned char)0xE3);
6972 }
6973 
6974 void Assembler::fist_s(Address adr) {
6975   InstructionMark im(this);
6976   emit_int8((unsigned char)0xDB);
6977   emit_operand32(rdx, adr);
6978 }
6979 
6980 void Assembler::fistp_d(Address adr) {
6981   InstructionMark im(this);
6982   emit_int8((unsigned char)0xDF);
6983   emit_operand32(rdi, adr);
6984 }
6985 
6986 void Assembler::fistp_s(Address adr) {
6987   InstructionMark im(this);
6988   emit_int8((unsigned char)0xDB);
6989   emit_operand32(rbx, adr);
6990 }
6991 
6992 void Assembler::fld1() {
6993   emit_int8((unsigned char)0xD9);
6994   emit_int8((unsigned char)0xE8);
6995 }
6996 
6997 void Assembler::fld_d(Address adr) {
6998   InstructionMark im(this);
6999   emit_int8((unsigned char)0xDD);
7000   emit_operand32(rax, adr);
7001 }
7002 
7003 void Assembler::fld_s(Address adr) {
7004   InstructionMark im(this);
7005   emit_int8((unsigned char)0xD9);
7006   emit_operand32(rax, adr);
7007 }
7008 
7009 
7010 void Assembler::fld_s(int index) {
7011   emit_farith(0xD9, 0xC0, index);
7012 }
7013 
7014 void Assembler::fld_x(Address adr) {
7015   InstructionMark im(this);
7016   emit_int8((unsigned char)0xDB);
7017   emit_operand32(rbp, adr);
7018 }
7019 
7020 void Assembler::fldcw(Address src) {
7021   InstructionMark im(this);
7022   emit_int8((unsigned char)0xD9);
7023   emit_operand32(rbp, src);
7024 }
7025 
7026 void Assembler::fldenv(Address src) {
7027   InstructionMark im(this);
7028   emit_int8((unsigned char)0xD9);
7029   emit_operand32(rsp, src);
7030 }
7031 
7032 void Assembler::fldlg2() {
7033   emit_int8((unsigned char)0xD9);
7034   emit_int8((unsigned char)0xEC);
7035 }
7036 
7037 void Assembler::fldln2() {
7038   emit_int8((unsigned char)0xD9);
7039   emit_int8((unsigned char)0xED);
7040 }
7041 
7042 void Assembler::fldz() {
7043   emit_int8((unsigned char)0xD9);
7044   emit_int8((unsigned char)0xEE);
7045 }
7046 
7047 void Assembler::flog() {
7048   fldln2();
7049   fxch();
7050   fyl2x();
7051 }
7052 
7053 void Assembler::flog10() {
7054   fldlg2();
7055   fxch();
7056   fyl2x();
7057 }
7058 
7059 void Assembler::fmul(int i) {
7060   emit_farith(0xD8, 0xC8, i);
7061 }
7062 
7063 void Assembler::fmul_d(Address src) {
7064   InstructionMark im(this);
7065   emit_int8((unsigned char)0xDC);
7066   emit_operand32(rcx, src);
7067 }
7068 
7069 void Assembler::fmul_s(Address src) {
7070   InstructionMark im(this);
7071   emit_int8((unsigned char)0xD8);
7072   emit_operand32(rcx, src);
7073 }
7074 
7075 void Assembler::fmula(int i) {
7076   emit_farith(0xDC, 0xC8, i);
7077 }
7078 
7079 void Assembler::fmulp(int i) {
7080   emit_farith(0xDE, 0xC8, i);
7081 }
7082 
7083 void Assembler::fnsave(Address dst) {
7084   InstructionMark im(this);
7085   emit_int8((unsigned char)0xDD);
7086   emit_operand32(rsi, dst);
7087 }
7088 
7089 void Assembler::fnstcw(Address src) {
7090   InstructionMark im(this);
7091   emit_int8((unsigned char)0x9B);
7092   emit_int8((unsigned char)0xD9);
7093   emit_operand32(rdi, src);
7094 }
7095 
7096 void Assembler::fnstsw_ax() {
7097   emit_int8((unsigned char)0xDF);
7098   emit_int8((unsigned char)0xE0);
7099 }
7100 
7101 void Assembler::fprem() {
7102   emit_int8((unsigned char)0xD9);
7103   emit_int8((unsigned char)0xF8);
7104 }
7105 
7106 void Assembler::fprem1() {
7107   emit_int8((unsigned char)0xD9);
7108   emit_int8((unsigned char)0xF5);
7109 }
7110 
7111 void Assembler::frstor(Address src) {
7112   InstructionMark im(this);
7113   emit_int8((unsigned char)0xDD);
7114   emit_operand32(rsp, src);
7115 }
7116 
7117 void Assembler::fsin() {
7118   emit_int8((unsigned char)0xD9);
7119   emit_int8((unsigned char)0xFE);
7120 }
7121 
7122 void Assembler::fsqrt() {
7123   emit_int8((unsigned char)0xD9);
7124   emit_int8((unsigned char)0xFA);
7125 }
7126 
7127 void Assembler::fst_d(Address adr) {
7128   InstructionMark im(this);
7129   emit_int8((unsigned char)0xDD);
7130   emit_operand32(rdx, adr);
7131 }
7132 
7133 void Assembler::fst_s(Address adr) {
7134   InstructionMark im(this);
7135   emit_int8((unsigned char)0xD9);
7136   emit_operand32(rdx, adr);
7137 }
7138 
7139 void Assembler::fstp_d(Address adr) {
7140   InstructionMark im(this);
7141   emit_int8((unsigned char)0xDD);
7142   emit_operand32(rbx, adr);
7143 }
7144 
7145 void Assembler::fstp_d(int index) {
7146   emit_farith(0xDD, 0xD8, index);
7147 }
7148 
7149 void Assembler::fstp_s(Address adr) {
7150   InstructionMark im(this);
7151   emit_int8((unsigned char)0xD9);
7152   emit_operand32(rbx, adr);
7153 }
7154 
7155 void Assembler::fstp_x(Address adr) {
7156   InstructionMark im(this);
7157   emit_int8((unsigned char)0xDB);
7158   emit_operand32(rdi, adr);
7159 }
7160 
7161 void Assembler::fsub(int i) {
7162   emit_farith(0xD8, 0xE0, i);
7163 }
7164 
7165 void Assembler::fsub_d(Address src) {
7166   InstructionMark im(this);
7167   emit_int8((unsigned char)0xDC);
7168   emit_operand32(rsp, src);
7169 }
7170 
7171 void Assembler::fsub_s(Address src) {
7172   InstructionMark im(this);
7173   emit_int8((unsigned char)0xD8);
7174   emit_operand32(rsp, src);
7175 }
7176 
7177 void Assembler::fsuba(int i) {
7178   emit_farith(0xDC, 0xE8, i);
7179 }
7180 
7181 void Assembler::fsubp(int i) {
7182   emit_farith(0xDE, 0xE8, i);                    // ST(0) <- ST(0) - ST(1) and pop (Intel manual wrong)
7183 }
7184 
7185 void Assembler::fsubr(int i) {
7186   emit_farith(0xD8, 0xE8, i);
7187 }
7188 
7189 void Assembler::fsubr_d(Address src) {
7190   InstructionMark im(this);
7191   emit_int8((unsigned char)0xDC);
7192   emit_operand32(rbp, src);
7193 }
7194 
7195 void Assembler::fsubr_s(Address src) {
7196   InstructionMark im(this);
7197   emit_int8((unsigned char)0xD8);
7198   emit_operand32(rbp, src);
7199 }
7200 
7201 void Assembler::fsubra(int i) {
7202   emit_farith(0xDC, 0xE0, i);
7203 }
7204 
7205 void Assembler::fsubrp(int i) {
7206   emit_farith(0xDE, 0xE0, i);                    // ST(0) <- ST(1) - ST(0) and pop (Intel manual wrong)
7207 }
7208 
7209 void Assembler::ftan() {
7210   emit_int8((unsigned char)0xD9);
7211   emit_int8((unsigned char)0xF2);
7212   emit_int8((unsigned char)0xDD);
7213   emit_int8((unsigned char)0xD8);
7214 }
7215 
7216 void Assembler::ftst() {
7217   emit_int8((unsigned char)0xD9);
7218   emit_int8((unsigned char)0xE4);
7219 }
7220 
7221 void Assembler::fucomi(int i) {
7222   // make sure the instruction is supported (introduced for P6, together with cmov)
7223   guarantee(VM_Version::supports_cmov(), "illegal instruction");
7224   emit_farith(0xDB, 0xE8, i);
7225 }
7226 
7227 void Assembler::fucomip(int i) {
7228   // make sure the instruction is supported (introduced for P6, together with cmov)
7229   guarantee(VM_Version::supports_cmov(), "illegal instruction");
7230   emit_farith(0xDF, 0xE8, i);
7231 }
7232 
7233 void Assembler::fwait() {
7234   emit_int8((unsigned char)0x9B);
7235 }
7236 
7237 void Assembler::fxch(int i) {
7238   emit_farith(0xD9, 0xC8, i);
7239 }
7240 
7241 void Assembler::fyl2x() {
7242   emit_int8((unsigned char)0xD9);
7243   emit_int8((unsigned char)0xF1);
7244 }
7245 
7246 void Assembler::frndint() {
7247   emit_int8((unsigned char)0xD9);
7248   emit_int8((unsigned char)0xFC);
7249 }
7250 
7251 void Assembler::f2xm1() {
7252   emit_int8((unsigned char)0xD9);
7253   emit_int8((unsigned char)0xF0);
7254 }
7255 
7256 void Assembler::fldl2e() {
7257   emit_int8((unsigned char)0xD9);
7258   emit_int8((unsigned char)0xEA);
7259 }
7260 
7261 // SSE SIMD prefix byte values corresponding to VexSimdPrefix encoding.
7262 static int simd_pre[4] = { 0, 0x66, 0xF3, 0xF2 };
7263 // SSE opcode second byte values (first is 0x0F) corresponding to VexOpcode encoding.
7264 static int simd_opc[4] = { 0,    0, 0x38, 0x3A };
7265 
7266 // Generate SSE legacy REX prefix and SIMD opcode based on VEX encoding.
7267 void Assembler::rex_prefix(Address adr, XMMRegister xreg, VexSimdPrefix pre, VexOpcode opc, bool rex_w) {
7268   if (pre > 0) {
7269     emit_int8(simd_pre[pre]);
7270   }
7271   if (rex_w) {
7272     prefixq(adr, xreg);
7273   } else {
7274     prefix(adr, xreg);
7275   }
7276   if (opc > 0) {
7277     emit_int8(0x0F);
7278     int opc2 = simd_opc[opc];
7279     if (opc2 > 0) {
7280       emit_int8(opc2);
7281     }
7282   }
7283 }
7284 
7285 int Assembler::rex_prefix_and_encode(int dst_enc, int src_enc, VexSimdPrefix pre, VexOpcode opc, bool rex_w) {
7286   if (pre > 0) {
7287     emit_int8(simd_pre[pre]);
7288   }
7289   int encode = (rex_w) ? prefixq_and_encode(dst_enc, src_enc) : prefix_and_encode(dst_enc, src_enc);
7290   if (opc > 0) {
7291     emit_int8(0x0F);
7292     int opc2 = simd_opc[opc];
7293     if (opc2 > 0) {
7294       emit_int8(opc2);
7295     }
7296   }
7297   return encode;
7298 }
7299 
7300 
7301 void Assembler::vex_prefix(bool vex_r, bool vex_b, bool vex_x, int nds_enc, VexSimdPrefix pre, VexOpcode opc) {
7302   int vector_len = _attributes->get_vector_len();
7303   bool vex_w = _attributes->is_rex_vex_w();
7304   if (vex_b || vex_x || vex_w || (opc == VEX_OPCODE_0F_38) || (opc == VEX_OPCODE_0F_3A)) {
7305     prefix(VEX_3bytes);
7306 
7307     int byte1 = (vex_r ? VEX_R : 0) | (vex_x ? VEX_X : 0) | (vex_b ? VEX_B : 0);
7308     byte1 = (~byte1) & 0xE0;
7309     byte1 |= opc;
7310     emit_int8(byte1);
7311 
7312     int byte2 = ((~nds_enc) & 0xf) << 3;
7313     byte2 |= (vex_w ? VEX_W : 0) | ((vector_len > 0) ? 4 : 0) | pre;
7314     emit_int8(byte2);
7315   } else {
7316     prefix(VEX_2bytes);
7317 
7318     int byte1 = vex_r ? VEX_R : 0;
7319     byte1 = (~byte1) & 0x80;
7320     byte1 |= ((~nds_enc) & 0xf) << 3;
7321     byte1 |= ((vector_len > 0 ) ? 4 : 0) | pre;
7322     emit_int8(byte1);
7323   }
7324 }
7325 
7326 // This is a 4 byte encoding
7327 void Assembler::evex_prefix(bool vex_r, bool vex_b, bool vex_x, bool evex_r, bool evex_v, int nds_enc, VexSimdPrefix pre, VexOpcode opc){
7328   // EVEX 0x62 prefix
7329   prefix(EVEX_4bytes);
7330   bool vex_w = _attributes->is_rex_vex_w();
7331   int evex_encoding = (vex_w ? VEX_W : 0);
7332   // EVEX.b is not currently used for broadcast of single element or data rounding modes
7333   _attributes->set_evex_encoding(evex_encoding);
7334 
7335   // P0: byte 2, initialized to RXBR`00mm
7336   // instead of not'd
7337   int byte2 = (vex_r ? VEX_R : 0) | (vex_x ? VEX_X : 0) | (vex_b ? VEX_B : 0) | (evex_r ? EVEX_Rb : 0);
7338   byte2 = (~byte2) & 0xF0;
7339   // confine opc opcode extensions in mm bits to lower two bits
7340   // of form {0F, 0F_38, 0F_3A}
7341   byte2 |= opc;
7342   emit_int8(byte2);
7343 
7344   // P1: byte 3 as Wvvvv1pp
7345   int byte3 = ((~nds_enc) & 0xf) << 3;
7346   // p[10] is always 1
7347   byte3 |= EVEX_F;
7348   byte3 |= (vex_w & 1) << 7;
7349   // confine pre opcode extensions in pp bits to lower two bits
7350   // of form {66, F3, F2}
7351   byte3 |= pre;
7352   emit_int8(byte3);
7353 
7354   // P2: byte 4 as zL'Lbv'aaa
7355   // kregs are implemented in the low 3 bits as aaa (hard code k1, it will be initialized for now)
7356   int byte4 = (_attributes->is_no_reg_mask()) ?
7357               0 :
7358               _attributes->get_embedded_opmask_register_specifier();
7359   // EVEX.v` for extending EVEX.vvvv or VIDX
7360   byte4 |= (evex_v ? 0: EVEX_V);
7361   // third EXEC.b for broadcast actions
7362   byte4 |= (_attributes->is_extended_context() ? EVEX_Rb : 0);
7363   // fourth EVEX.L'L for vector length : 0 is 128, 1 is 256, 2 is 512, currently we do not support 1024
7364   byte4 |= ((_attributes->get_vector_len())& 0x3) << 5;
7365   // last is EVEX.z for zero/merge actions
7366   if (_attributes->is_no_reg_mask() == false) {
7367     byte4 |= (_attributes->is_clear_context() ? EVEX_Z : 0);
7368   }
7369   emit_int8(byte4);
7370 }
7371 
7372 void Assembler::vex_prefix(Address adr, int nds_enc, int xreg_enc, VexSimdPrefix pre, VexOpcode opc, InstructionAttr *attributes) {
7373   bool vex_r = ((xreg_enc & 8) == 8) ? 1 : 0;
7374   bool vex_b = adr.base_needs_rex();
7375   bool vex_x = adr.index_needs_rex();
7376   set_attributes(attributes);
7377   attributes->set_current_assembler(this);
7378 
7379   // if vector length is turned off, revert to AVX for vectors smaller than 512-bit
7380   if (UseAVX > 2 && _legacy_mode_vl && attributes->uses_vl()) {
7381     switch (attributes->get_vector_len()) {
7382     case AVX_128bit:
7383     case AVX_256bit:
7384       attributes->set_is_legacy_mode();
7385       break;
7386     }
7387   }
7388 
7389   // For pure EVEX check and see if this instruction
7390   // is allowed in legacy mode and has resources which will
7391   // fit in it.  Pure EVEX instructions will use set_is_evex_instruction in their definition,
7392   // else that field is set when we encode to EVEX
7393   if (UseAVX > 2 && !attributes->is_legacy_mode() &&
7394       !_is_managed && !attributes->is_evex_instruction()) {
7395     if (!_legacy_mode_vl && attributes->get_vector_len() != AVX_512bit) {
7396       bool check_register_bank = NOT_IA32(true) IA32_ONLY(false);
7397       if (check_register_bank) {
7398         // check nds_enc and xreg_enc for upper bank usage
7399         if (nds_enc < 16 && xreg_enc < 16) {
7400           attributes->set_is_legacy_mode();
7401         }
7402       } else {
7403         attributes->set_is_legacy_mode();
7404       }
7405     }
7406   }
7407 
7408   _is_managed = false;
7409   if (UseAVX > 2 && !attributes->is_legacy_mode())
7410   {
7411     bool evex_r = (xreg_enc >= 16);
7412     bool evex_v = (nds_enc >= 16);
7413     attributes->set_is_evex_instruction();
7414     evex_prefix(vex_r, vex_b, vex_x, evex_r, evex_v, nds_enc, pre, opc);
7415   } else {
7416     if (UseAVX > 2 && attributes->is_rex_vex_w_reverted()) {
7417       attributes->set_rex_vex_w(false);
7418     }
7419     vex_prefix(vex_r, vex_b, vex_x, nds_enc, pre, opc);
7420   }
7421 }
7422 
7423 int Assembler::vex_prefix_and_encode(int dst_enc, int nds_enc, int src_enc, VexSimdPrefix pre, VexOpcode opc, InstructionAttr *attributes) {
7424   bool vex_r = ((dst_enc & 8) == 8) ? 1 : 0;
7425   bool vex_b = ((src_enc & 8) == 8) ? 1 : 0;
7426   bool vex_x = false;
7427   set_attributes(attributes);
7428   attributes->set_current_assembler(this);
7429   bool check_register_bank = NOT_IA32(true) IA32_ONLY(false);
7430 
7431   // if vector length is turned off, revert to AVX for vectors smaller than 512-bit
7432   if (UseAVX > 2 && _legacy_mode_vl && attributes->uses_vl()) {
7433     switch (attributes->get_vector_len()) {
7434     case AVX_128bit:
7435     case AVX_256bit:
7436       if (check_register_bank) {
7437         if (dst_enc >= 16 || nds_enc >= 16 || src_enc >= 16) {
7438           // up propagate arithmetic instructions to meet RA requirements
7439           attributes->set_vector_len(AVX_512bit);
7440         } else {
7441           attributes->set_is_legacy_mode();
7442         }
7443       } else {
7444         attributes->set_is_legacy_mode();
7445       }
7446       break;
7447     }
7448   }
7449 
7450   // For pure EVEX check and see if this instruction
7451   // is allowed in legacy mode and has resources which will
7452   // fit in it.  Pure EVEX instructions will use set_is_evex_instruction in their definition,
7453   // else that field is set when we encode to EVEX
7454   if (UseAVX > 2 && !attributes->is_legacy_mode() &&
7455       !_is_managed && !attributes->is_evex_instruction()) {
7456     if (!_legacy_mode_vl && attributes->get_vector_len() != AVX_512bit) {
7457       if (check_register_bank) {
7458         // check dst_enc, nds_enc and src_enc for upper bank usage
7459         if (dst_enc < 16 && nds_enc < 16 && src_enc < 16) {
7460           attributes->set_is_legacy_mode();
7461         }
7462       } else {
7463         attributes->set_is_legacy_mode();
7464       }
7465     }
7466   }
7467 
7468   _is_managed = false;
7469   if (UseAVX > 2 && !attributes->is_legacy_mode())
7470   {
7471     bool evex_r = (dst_enc >= 16);
7472     bool evex_v = (nds_enc >= 16);
7473     // can use vex_x as bank extender on rm encoding
7474     vex_x = (src_enc >= 16);
7475     attributes->set_is_evex_instruction();
7476     evex_prefix(vex_r, vex_b, vex_x, evex_r, evex_v, nds_enc, pre, opc);
7477   } else {
7478     if (UseAVX > 2 && attributes->is_rex_vex_w_reverted()) {
7479       attributes->set_rex_vex_w(false);
7480     }
7481     vex_prefix(vex_r, vex_b, vex_x, nds_enc, pre, opc);
7482   }
7483 
7484   // return modrm byte components for operands
7485   return (((dst_enc & 7) << 3) | (src_enc & 7));
7486 }
7487 
7488 
7489 void Assembler::simd_prefix(XMMRegister xreg, XMMRegister nds, Address adr, VexSimdPrefix pre,
7490                             VexOpcode opc, InstructionAttr *attributes) {
7491   if (UseAVX > 0) {
7492     int xreg_enc = xreg->encoding();
7493     int nds_enc = nds->is_valid() ? nds->encoding() : 0;
7494     vex_prefix(adr, nds_enc, xreg_enc, pre, opc, attributes);
7495   } else {
7496     assert((nds == xreg) || (nds == xnoreg), "wrong sse encoding");
7497     rex_prefix(adr, xreg, pre, opc, attributes->is_rex_vex_w());
7498   }
7499 }
7500 
7501 int Assembler::simd_prefix_and_encode(XMMRegister dst, XMMRegister nds, XMMRegister src, VexSimdPrefix pre,
7502                                       VexOpcode opc, InstructionAttr *attributes) {
7503   int dst_enc = dst->encoding();
7504   int src_enc = src->encoding();
7505   if (UseAVX > 0) {
7506     int nds_enc = nds->is_valid() ? nds->encoding() : 0;
7507     return vex_prefix_and_encode(dst_enc, nds_enc, src_enc, pre, opc, attributes);
7508   } else {
7509     assert((nds == dst) || (nds == src) || (nds == xnoreg), "wrong sse encoding");
7510     return rex_prefix_and_encode(dst_enc, src_enc, pre, opc, attributes->is_rex_vex_w());
7511   }
7512 }
7513 
7514 void Assembler::cmppd(XMMRegister dst, XMMRegister nds, XMMRegister src, int cop, int vector_len) {
7515   assert(VM_Version::supports_avx(), "");
7516   assert(!VM_Version::supports_evex(), "");
7517   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
7518   int encode = simd_prefix_and_encode(dst, nds, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7519   emit_int8((unsigned char)0xC2);
7520   emit_int8((unsigned char)(0xC0 | encode));
7521   emit_int8((unsigned char)(0xF & cop));
7522 }
7523 
7524 void Assembler::blendvpd(XMMRegister dst, XMMRegister nds, XMMRegister src1, XMMRegister src2, int vector_len) {
7525   assert(VM_Version::supports_avx(), "");
7526   assert(!VM_Version::supports_evex(), "");
7527   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
7528   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src1->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
7529   emit_int8((unsigned char)0x4B);
7530   emit_int8((unsigned char)(0xC0 | encode));
7531   int src2_enc = src2->encoding();
7532   emit_int8((unsigned char)(0xF0 & src2_enc<<4));
7533 }
7534 
7535 void Assembler::cmpps(XMMRegister dst, XMMRegister nds, XMMRegister src, int cop, int vector_len) {
7536   assert(VM_Version::supports_avx(), "");
7537   assert(!VM_Version::supports_evex(), "");
7538   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
7539   int encode = simd_prefix_and_encode(dst, nds, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
7540   emit_int8((unsigned char)0xC2);
7541   emit_int8((unsigned char)(0xC0 | encode));
7542   emit_int8((unsigned char)(0xF & cop));
7543 }
7544 
7545 void Assembler::blendvps(XMMRegister dst, XMMRegister nds, XMMRegister src1, XMMRegister src2, int vector_len) {
7546   assert(VM_Version::supports_avx(), "");
7547   assert(!VM_Version::supports_evex(), "");
7548   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
7549   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src1->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
7550   emit_int8((unsigned char)0x4A);
7551   emit_int8((unsigned char)(0xC0 | encode));
7552   int src2_enc = src2->encoding();
7553   emit_int8((unsigned char)(0xF0 & src2_enc<<4));
7554 }
7555 
7556 void Assembler::vpblendd(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8, int vector_len) {
7557   assert(VM_Version::supports_avx2(), "");
7558   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
7559   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
7560   emit_int8((unsigned char)0x02);
7561   emit_int8((unsigned char)(0xC0 | encode));
7562   emit_int8((unsigned char)imm8);
7563 }
7564 
7565 void Assembler::shlxl(Register dst, Register src1, Register src2) {
7566   assert(VM_Version::supports_bmi2(), "");
7567   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
7568   int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src1->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
7569   emit_int8((unsigned char)0xF7);
7570   emit_int8((unsigned char)(0xC0 | encode));
7571 }
7572 
7573 void Assembler::shlxq(Register dst, Register src1, Register src2) {
7574   assert(VM_Version::supports_bmi2(), "");
7575   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
7576   int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src1->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
7577   emit_int8((unsigned char)0xF7);
7578   emit_int8((unsigned char)(0xC0 | encode));
7579 }
7580 
7581 #ifndef _LP64
7582 
7583 void Assembler::incl(Register dst) {
7584   // Don't use it directly. Use MacroAssembler::incrementl() instead.
7585   emit_int8(0x40 | dst->encoding());
7586 }
7587 
7588 void Assembler::lea(Register dst, Address src) {
7589   leal(dst, src);
7590 }
7591 
7592 void Assembler::mov_literal32(Address dst, int32_t imm32, RelocationHolder const& rspec) {
7593   InstructionMark im(this);
7594   emit_int8((unsigned char)0xC7);
7595   emit_operand(rax, dst);
7596   emit_data((int)imm32, rspec, 0);
7597 }
7598 
7599 void Assembler::mov_literal32(Register dst, int32_t imm32, RelocationHolder const& rspec) {
7600   InstructionMark im(this);
7601   int encode = prefix_and_encode(dst->encoding());
7602   emit_int8((unsigned char)(0xB8 | encode));
7603   emit_data((int)imm32, rspec, 0);
7604 }
7605 
7606 void Assembler::popa() { // 32bit
7607   emit_int8(0x61);
7608 }
7609 
7610 void Assembler::push_literal32(int32_t imm32, RelocationHolder const& rspec) {
7611   InstructionMark im(this);
7612   emit_int8(0x68);
7613   emit_data(imm32, rspec, 0);
7614 }
7615 
7616 void Assembler::pusha() { // 32bit
7617   emit_int8(0x60);
7618 }
7619 
7620 void Assembler::set_byte_if_not_zero(Register dst) {
7621   emit_int8(0x0F);
7622   emit_int8((unsigned char)0x95);
7623   emit_int8((unsigned char)(0xE0 | dst->encoding()));
7624 }
7625 
7626 void Assembler::shldl(Register dst, Register src) {
7627   emit_int8(0x0F);
7628   emit_int8((unsigned char)0xA5);
7629   emit_int8((unsigned char)(0xC0 | src->encoding() << 3 | dst->encoding()));
7630 }
7631 
7632 // 0F A4 / r ib
7633 void Assembler::shldl(Register dst, Register src, int8_t imm8) {
7634   emit_int8(0x0F);
7635   emit_int8((unsigned char)0xA4);
7636   emit_int8((unsigned char)(0xC0 | src->encoding() << 3 | dst->encoding()));
7637   emit_int8(imm8);
7638 }
7639 
7640 void Assembler::shrdl(Register dst, Register src) {
7641   emit_int8(0x0F);
7642   emit_int8((unsigned char)0xAD);
7643   emit_int8((unsigned char)(0xC0 | src->encoding() << 3 | dst->encoding()));
7644 }
7645 
7646 #else // LP64
7647 
7648 void Assembler::set_byte_if_not_zero(Register dst) {
7649   int enc = prefix_and_encode(dst->encoding(), true);
7650   emit_int8(0x0F);
7651   emit_int8((unsigned char)0x95);
7652   emit_int8((unsigned char)(0xE0 | enc));
7653 }
7654 
7655 // 64bit only pieces of the assembler
7656 // This should only be used by 64bit instructions that can use rip-relative
7657 // it cannot be used by instructions that want an immediate value.
7658 
7659 bool Assembler::reachable(AddressLiteral adr) {
7660   int64_t disp;
7661   // None will force a 64bit literal to the code stream. Likely a placeholder
7662   // for something that will be patched later and we need to certain it will
7663   // always be reachable.
7664   if (adr.reloc() == relocInfo::none) {
7665     return false;
7666   }
7667   if (adr.reloc() == relocInfo::internal_word_type) {
7668     // This should be rip relative and easily reachable.
7669     return true;
7670   }
7671   if (adr.reloc() == relocInfo::virtual_call_type ||
7672       adr.reloc() == relocInfo::opt_virtual_call_type ||
7673       adr.reloc() == relocInfo::static_call_type ||
7674       adr.reloc() == relocInfo::static_stub_type ) {
7675     // This should be rip relative within the code cache and easily
7676     // reachable until we get huge code caches. (At which point
7677     // ic code is going to have issues).
7678     return true;
7679   }
7680   if (adr.reloc() != relocInfo::external_word_type &&
7681       adr.reloc() != relocInfo::poll_return_type &&  // these are really external_word but need special
7682       adr.reloc() != relocInfo::poll_type &&         // relocs to identify them
7683       adr.reloc() != relocInfo::runtime_call_type ) {
7684     return false;
7685   }
7686 
7687   // Stress the correction code
7688   if (ForceUnreachable) {
7689     // Must be runtimecall reloc, see if it is in the codecache
7690     // Flipping stuff in the codecache to be unreachable causes issues
7691     // with things like inline caches where the additional instructions
7692     // are not handled.
7693     if (CodeCache::find_blob(adr._target) == NULL) {
7694       return false;
7695     }
7696   }
7697   // For external_word_type/runtime_call_type if it is reachable from where we
7698   // are now (possibly a temp buffer) and where we might end up
7699   // anywhere in the codeCache then we are always reachable.
7700   // This would have to change if we ever save/restore shared code
7701   // to be more pessimistic.
7702   disp = (int64_t)adr._target - ((int64_t)CodeCache::low_bound() + sizeof(int));
7703   if (!is_simm32(disp)) return false;
7704   disp = (int64_t)adr._target - ((int64_t)CodeCache::high_bound() + sizeof(int));
7705   if (!is_simm32(disp)) return false;
7706 
7707   disp = (int64_t)adr._target - ((int64_t)pc() + sizeof(int));
7708 
7709   // Because rip relative is a disp + address_of_next_instruction and we
7710   // don't know the value of address_of_next_instruction we apply a fudge factor
7711   // to make sure we will be ok no matter the size of the instruction we get placed into.
7712   // We don't have to fudge the checks above here because they are already worst case.
7713 
7714   // 12 == override/rex byte, opcode byte, rm byte, sib byte, a 4-byte disp , 4-byte literal
7715   // + 4 because better safe than sorry.
7716   const int fudge = 12 + 4;
7717   if (disp < 0) {
7718     disp -= fudge;
7719   } else {
7720     disp += fudge;
7721   }
7722   return is_simm32(disp);
7723 }
7724 
7725 // Check if the polling page is not reachable from the code cache using rip-relative
7726 // addressing.
7727 bool Assembler::is_polling_page_far() {
7728   intptr_t addr = (intptr_t)os::get_polling_page();
7729   return ForceUnreachable ||
7730          !is_simm32(addr - (intptr_t)CodeCache::low_bound()) ||
7731          !is_simm32(addr - (intptr_t)CodeCache::high_bound());
7732 }
7733 
7734 void Assembler::emit_data64(jlong data,
7735                             relocInfo::relocType rtype,
7736                             int format) {
7737   if (rtype == relocInfo::none) {
7738     emit_int64(data);
7739   } else {
7740     emit_data64(data, Relocation::spec_simple(rtype), format);
7741   }
7742 }
7743 
7744 void Assembler::emit_data64(jlong data,
7745                             RelocationHolder const& rspec,
7746                             int format) {
7747   assert(imm_operand == 0, "default format must be immediate in this file");
7748   assert(imm_operand == format, "must be immediate");
7749   assert(inst_mark() != NULL, "must be inside InstructionMark");
7750   // Do not use AbstractAssembler::relocate, which is not intended for
7751   // embedded words.  Instead, relocate to the enclosing instruction.
7752   code_section()->relocate(inst_mark(), rspec, format);
7753 #ifdef ASSERT
7754   check_relocation(rspec, format);
7755 #endif
7756   emit_int64(data);
7757 }
7758 
7759 int Assembler::prefix_and_encode(int reg_enc, bool byteinst) {
7760   if (reg_enc >= 8) {
7761     prefix(REX_B);
7762     reg_enc -= 8;
7763   } else if (byteinst && reg_enc >= 4) {
7764     prefix(REX);
7765   }
7766   return reg_enc;
7767 }
7768 
7769 int Assembler::prefixq_and_encode(int reg_enc) {
7770   if (reg_enc < 8) {
7771     prefix(REX_W);
7772   } else {
7773     prefix(REX_WB);
7774     reg_enc -= 8;
7775   }
7776   return reg_enc;
7777 }
7778 
7779 int Assembler::prefix_and_encode(int dst_enc, bool dst_is_byte, int src_enc, bool src_is_byte) {
7780   if (dst_enc < 8) {
7781     if (src_enc >= 8) {
7782       prefix(REX_B);
7783       src_enc -= 8;
7784     } else if ((src_is_byte && src_enc >= 4) || (dst_is_byte && dst_enc >= 4)) {
7785       prefix(REX);
7786     }
7787   } else {
7788     if (src_enc < 8) {
7789       prefix(REX_R);
7790     } else {
7791       prefix(REX_RB);
7792       src_enc -= 8;
7793     }
7794     dst_enc -= 8;
7795   }
7796   return dst_enc << 3 | src_enc;
7797 }
7798 
7799 int Assembler::prefixq_and_encode(int dst_enc, int src_enc) {
7800   if (dst_enc < 8) {
7801     if (src_enc < 8) {
7802       prefix(REX_W);
7803     } else {
7804       prefix(REX_WB);
7805       src_enc -= 8;
7806     }
7807   } else {
7808     if (src_enc < 8) {
7809       prefix(REX_WR);
7810     } else {
7811       prefix(REX_WRB);
7812       src_enc -= 8;
7813     }
7814     dst_enc -= 8;
7815   }
7816   return dst_enc << 3 | src_enc;
7817 }
7818 
7819 void Assembler::prefix(Register reg) {
7820   if (reg->encoding() >= 8) {
7821     prefix(REX_B);
7822   }
7823 }
7824 
7825 void Assembler::prefix(Register dst, Register src, Prefix p) {
7826   if (src->encoding() >= 8) {
7827     p = (Prefix)(p | REX_B);
7828   }
7829   if (dst->encoding() >= 8) {
7830     p = (Prefix)( p | REX_R);
7831   }
7832   if (p != Prefix_EMPTY) {
7833     // do not generate an empty prefix
7834     prefix(p);
7835   }
7836 }
7837 
7838 void Assembler::prefix(Register dst, Address adr, Prefix p) {
7839   if (adr.base_needs_rex()) {
7840     if (adr.index_needs_rex()) {
7841       assert(false, "prefix(Register dst, Address adr, Prefix p) does not support handling of an X");
7842     } else {
7843       prefix(REX_B);
7844     }
7845   } else {
7846     if (adr.index_needs_rex()) {
7847       assert(false, "prefix(Register dst, Address adr, Prefix p) does not support handling of an X");
7848     }
7849   }
7850   if (dst->encoding() >= 8) {
7851     p = (Prefix)(p | REX_R);
7852   }
7853   if (p != Prefix_EMPTY) {
7854     // do not generate an empty prefix
7855     prefix(p);
7856   }
7857 }
7858 
7859 void Assembler::prefix(Address adr) {
7860   if (adr.base_needs_rex()) {
7861     if (adr.index_needs_rex()) {
7862       prefix(REX_XB);
7863     } else {
7864       prefix(REX_B);
7865     }
7866   } else {
7867     if (adr.index_needs_rex()) {
7868       prefix(REX_X);
7869     }
7870   }
7871 }
7872 
7873 void Assembler::prefixq(Address adr) {
7874   if (adr.base_needs_rex()) {
7875     if (adr.index_needs_rex()) {
7876       prefix(REX_WXB);
7877     } else {
7878       prefix(REX_WB);
7879     }
7880   } else {
7881     if (adr.index_needs_rex()) {
7882       prefix(REX_WX);
7883     } else {
7884       prefix(REX_W);
7885     }
7886   }
7887 }
7888 
7889 
7890 void Assembler::prefix(Address adr, Register reg, bool byteinst) {
7891   if (reg->encoding() < 8) {
7892     if (adr.base_needs_rex()) {
7893       if (adr.index_needs_rex()) {
7894         prefix(REX_XB);
7895       } else {
7896         prefix(REX_B);
7897       }
7898     } else {
7899       if (adr.index_needs_rex()) {
7900         prefix(REX_X);
7901       } else if (byteinst && reg->encoding() >= 4 ) {
7902         prefix(REX);
7903       }
7904     }
7905   } else {
7906     if (adr.base_needs_rex()) {
7907       if (adr.index_needs_rex()) {
7908         prefix(REX_RXB);
7909       } else {
7910         prefix(REX_RB);
7911       }
7912     } else {
7913       if (adr.index_needs_rex()) {
7914         prefix(REX_RX);
7915       } else {
7916         prefix(REX_R);
7917       }
7918     }
7919   }
7920 }
7921 
7922 void Assembler::prefixq(Address adr, Register src) {
7923   if (src->encoding() < 8) {
7924     if (adr.base_needs_rex()) {
7925       if (adr.index_needs_rex()) {
7926         prefix(REX_WXB);
7927       } else {
7928         prefix(REX_WB);
7929       }
7930     } else {
7931       if (adr.index_needs_rex()) {
7932         prefix(REX_WX);
7933       } else {
7934         prefix(REX_W);
7935       }
7936     }
7937   } else {
7938     if (adr.base_needs_rex()) {
7939       if (adr.index_needs_rex()) {
7940         prefix(REX_WRXB);
7941       } else {
7942         prefix(REX_WRB);
7943       }
7944     } else {
7945       if (adr.index_needs_rex()) {
7946         prefix(REX_WRX);
7947       } else {
7948         prefix(REX_WR);
7949       }
7950     }
7951   }
7952 }
7953 
7954 void Assembler::prefix(Address adr, XMMRegister reg) {
7955   if (reg->encoding() < 8) {
7956     if (adr.base_needs_rex()) {
7957       if (adr.index_needs_rex()) {
7958         prefix(REX_XB);
7959       } else {
7960         prefix(REX_B);
7961       }
7962     } else {
7963       if (adr.index_needs_rex()) {
7964         prefix(REX_X);
7965       }
7966     }
7967   } else {
7968     if (adr.base_needs_rex()) {
7969       if (adr.index_needs_rex()) {
7970         prefix(REX_RXB);
7971       } else {
7972         prefix(REX_RB);
7973       }
7974     } else {
7975       if (adr.index_needs_rex()) {
7976         prefix(REX_RX);
7977       } else {
7978         prefix(REX_R);
7979       }
7980     }
7981   }
7982 }
7983 
7984 void Assembler::prefixq(Address adr, XMMRegister src) {
7985   if (src->encoding() < 8) {
7986     if (adr.base_needs_rex()) {
7987       if (adr.index_needs_rex()) {
7988         prefix(REX_WXB);
7989       } else {
7990         prefix(REX_WB);
7991       }
7992     } else {
7993       if (adr.index_needs_rex()) {
7994         prefix(REX_WX);
7995       } else {
7996         prefix(REX_W);
7997       }
7998     }
7999   } else {
8000     if (adr.base_needs_rex()) {
8001       if (adr.index_needs_rex()) {
8002         prefix(REX_WRXB);
8003       } else {
8004         prefix(REX_WRB);
8005       }
8006     } else {
8007       if (adr.index_needs_rex()) {
8008         prefix(REX_WRX);
8009       } else {
8010         prefix(REX_WR);
8011       }
8012     }
8013   }
8014 }
8015 
8016 void Assembler::adcq(Register dst, int32_t imm32) {
8017   (void) prefixq_and_encode(dst->encoding());
8018   emit_arith(0x81, 0xD0, dst, imm32);
8019 }
8020 
8021 void Assembler::adcq(Register dst, Address src) {
8022   InstructionMark im(this);
8023   prefixq(src, dst);
8024   emit_int8(0x13);
8025   emit_operand(dst, src);
8026 }
8027 
8028 void Assembler::adcq(Register dst, Register src) {
8029   (void) prefixq_and_encode(dst->encoding(), src->encoding());
8030   emit_arith(0x13, 0xC0, dst, src);
8031 }
8032 
8033 void Assembler::addq(Address dst, int32_t imm32) {
8034   InstructionMark im(this);
8035   prefixq(dst);
8036   emit_arith_operand(0x81, rax, dst,imm32);
8037 }
8038 
8039 void Assembler::addq(Address dst, Register src) {
8040   InstructionMark im(this);
8041   prefixq(dst, src);
8042   emit_int8(0x01);
8043   emit_operand(src, dst);
8044 }
8045 
8046 void Assembler::addq(Register dst, int32_t imm32) {
8047   (void) prefixq_and_encode(dst->encoding());
8048   emit_arith(0x81, 0xC0, dst, imm32);
8049 }
8050 
8051 void Assembler::addq(Register dst, Address src) {
8052   InstructionMark im(this);
8053   prefixq(src, dst);
8054   emit_int8(0x03);
8055   emit_operand(dst, src);
8056 }
8057 
8058 void Assembler::addq(Register dst, Register src) {
8059   (void) prefixq_and_encode(dst->encoding(), src->encoding());
8060   emit_arith(0x03, 0xC0, dst, src);
8061 }
8062 
8063 void Assembler::adcxq(Register dst, Register src) {
8064   //assert(VM_Version::supports_adx(), "adx instructions not supported");
8065   emit_int8((unsigned char)0x66);
8066   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
8067   emit_int8(0x0F);
8068   emit_int8(0x38);
8069   emit_int8((unsigned char)0xF6);
8070   emit_int8((unsigned char)(0xC0 | encode));
8071 }
8072 
8073 void Assembler::adoxq(Register dst, Register src) {
8074   //assert(VM_Version::supports_adx(), "adx instructions not supported");
8075   emit_int8((unsigned char)0xF3);
8076   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
8077   emit_int8(0x0F);
8078   emit_int8(0x38);
8079   emit_int8((unsigned char)0xF6);
8080   emit_int8((unsigned char)(0xC0 | encode));
8081 }
8082 
8083 void Assembler::andq(Address dst, int32_t imm32) {
8084   InstructionMark im(this);
8085   prefixq(dst);
8086   emit_int8((unsigned char)0x81);
8087   emit_operand(rsp, dst, 4);
8088   emit_int32(imm32);
8089 }
8090 
8091 void Assembler::andq(Register dst, int32_t imm32) {
8092   (void) prefixq_and_encode(dst->encoding());
8093   emit_arith(0x81, 0xE0, dst, imm32);
8094 }
8095 
8096 void Assembler::andq(Register dst, Address src) {
8097   InstructionMark im(this);
8098   prefixq(src, dst);
8099   emit_int8(0x23);
8100   emit_operand(dst, src);
8101 }
8102 
8103 void Assembler::andq(Register dst, Register src) {
8104   (void) prefixq_and_encode(dst->encoding(), src->encoding());
8105   emit_arith(0x23, 0xC0, dst, src);
8106 }
8107 
8108 void Assembler::andnq(Register dst, Register src1, Register src2) {
8109   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
8110   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
8111   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
8112   emit_int8((unsigned char)0xF2);
8113   emit_int8((unsigned char)(0xC0 | encode));
8114 }
8115 
8116 void Assembler::andnq(Register dst, Register src1, Address src2) {
8117   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
8118   InstructionMark im(this);
8119   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
8120   vex_prefix(src2, src1->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
8121   emit_int8((unsigned char)0xF2);
8122   emit_operand(dst, src2);
8123 }
8124 
8125 void Assembler::bsfq(Register dst, Register src) {
8126   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
8127   emit_int8(0x0F);
8128   emit_int8((unsigned char)0xBC);
8129   emit_int8((unsigned char)(0xC0 | encode));
8130 }
8131 
8132 void Assembler::bsrq(Register dst, Register src) {
8133   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
8134   emit_int8(0x0F);
8135   emit_int8((unsigned char)0xBD);
8136   emit_int8((unsigned char)(0xC0 | encode));
8137 }
8138 
8139 void Assembler::bswapq(Register reg) {
8140   int encode = prefixq_and_encode(reg->encoding());
8141   emit_int8(0x0F);
8142   emit_int8((unsigned char)(0xC8 | encode));
8143 }
8144 
8145 void Assembler::blsiq(Register dst, Register src) {
8146   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
8147   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
8148   int encode = vex_prefix_and_encode(rbx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
8149   emit_int8((unsigned char)0xF3);
8150   emit_int8((unsigned char)(0xC0 | encode));
8151 }
8152 
8153 void Assembler::blsiq(Register dst, Address src) {
8154   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
8155   InstructionMark im(this);
8156   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
8157   vex_prefix(src, dst->encoding(), rbx->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
8158   emit_int8((unsigned char)0xF3);
8159   emit_operand(rbx, src);
8160 }
8161 
8162 void Assembler::blsmskq(Register dst, Register src) {
8163   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
8164   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
8165   int encode = vex_prefix_and_encode(rdx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
8166   emit_int8((unsigned char)0xF3);
8167   emit_int8((unsigned char)(0xC0 | encode));
8168 }
8169 
8170 void Assembler::blsmskq(Register dst, Address src) {
8171   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
8172   InstructionMark im(this);
8173   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
8174   vex_prefix(src, dst->encoding(), rdx->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
8175   emit_int8((unsigned char)0xF3);
8176   emit_operand(rdx, src);
8177 }
8178 
8179 void Assembler::blsrq(Register dst, Register src) {
8180   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
8181   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
8182   int encode = vex_prefix_and_encode(rcx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
8183   emit_int8((unsigned char)0xF3);
8184   emit_int8((unsigned char)(0xC0 | encode));
8185 }
8186 
8187 void Assembler::blsrq(Register dst, Address src) {
8188   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
8189   InstructionMark im(this);
8190   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
8191   vex_prefix(src, dst->encoding(), rcx->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
8192   emit_int8((unsigned char)0xF3);
8193   emit_operand(rcx, src);
8194 }
8195 
8196 void Assembler::cdqq() {
8197   prefix(REX_W);
8198   emit_int8((unsigned char)0x99);
8199 }
8200 
8201 void Assembler::clflush(Address adr) {
8202   prefix(adr);
8203   emit_int8(0x0F);
8204   emit_int8((unsigned char)0xAE);
8205   emit_operand(rdi, adr);
8206 }
8207 
8208 void Assembler::cmovq(Condition cc, Register dst, Register src) {
8209   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
8210   emit_int8(0x0F);
8211   emit_int8(0x40 | cc);
8212   emit_int8((unsigned char)(0xC0 | encode));
8213 }
8214 
8215 void Assembler::cmovq(Condition cc, Register dst, Address src) {
8216   InstructionMark im(this);
8217   prefixq(src, dst);
8218   emit_int8(0x0F);
8219   emit_int8(0x40 | cc);
8220   emit_operand(dst, src);
8221 }
8222 
8223 void Assembler::cmpq(Address dst, int32_t imm32) {
8224   InstructionMark im(this);
8225   prefixq(dst);
8226   emit_int8((unsigned char)0x81);
8227   emit_operand(rdi, dst, 4);
8228   emit_int32(imm32);
8229 }
8230 
8231 void Assembler::cmpq(Register dst, int32_t imm32) {
8232   (void) prefixq_and_encode(dst->encoding());
8233   emit_arith(0x81, 0xF8, dst, imm32);
8234 }
8235 
8236 void Assembler::cmpq(Address dst, Register src) {
8237   InstructionMark im(this);
8238   prefixq(dst, src);
8239   emit_int8(0x3B);
8240   emit_operand(src, dst);
8241 }
8242 
8243 void Assembler::cmpq(Register dst, Register src) {
8244   (void) prefixq_and_encode(dst->encoding(), src->encoding());
8245   emit_arith(0x3B, 0xC0, dst, src);
8246 }
8247 
8248 void Assembler::cmpq(Register dst, Address  src) {
8249   InstructionMark im(this);
8250   prefixq(src, dst);
8251   emit_int8(0x3B);
8252   emit_operand(dst, src);
8253 }
8254 
8255 void Assembler::cmpxchgq(Register reg, Address adr) {
8256   InstructionMark im(this);
8257   prefixq(adr, reg);
8258   emit_int8(0x0F);
8259   emit_int8((unsigned char)0xB1);
8260   emit_operand(reg, adr);
8261 }
8262 
8263 void Assembler::cvtsi2sdq(XMMRegister dst, Register src) {
8264   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
8265   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
8266   int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
8267   emit_int8(0x2A);
8268   emit_int8((unsigned char)(0xC0 | encode));
8269 }
8270 
8271 void Assembler::cvtsi2sdq(XMMRegister dst, Address src) {
8272   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
8273   InstructionMark im(this);
8274   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
8275   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
8276   simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
8277   emit_int8(0x2A);
8278   emit_operand(dst, src);
8279 }
8280 
8281 void Assembler::cvtsi2ssq(XMMRegister dst, Address src) {
8282   NOT_LP64(assert(VM_Version::supports_sse(), ""));
8283   InstructionMark im(this);
8284   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
8285   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
8286   simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
8287   emit_int8(0x2A);
8288   emit_operand(dst, src);
8289 }
8290 
8291 void Assembler::cvttsd2siq(Register dst, XMMRegister src) {
8292   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
8293   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
8294   int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
8295   emit_int8(0x2C);
8296   emit_int8((unsigned char)(0xC0 | encode));
8297 }
8298 
8299 void Assembler::cvttss2siq(Register dst, XMMRegister src) {
8300   NOT_LP64(assert(VM_Version::supports_sse(), ""));
8301   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
8302   int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
8303   emit_int8(0x2C);
8304   emit_int8((unsigned char)(0xC0 | encode));
8305 }
8306 
8307 void Assembler::decl(Register dst) {
8308   // Don't use it directly. Use MacroAssembler::decrementl() instead.
8309   // Use two-byte form (one-byte form is a REX prefix in 64-bit mode)
8310   int encode = prefix_and_encode(dst->encoding());
8311   emit_int8((unsigned char)0xFF);
8312   emit_int8((unsigned char)(0xC8 | encode));
8313 }
8314 
8315 void Assembler::decq(Register dst) {
8316   // Don't use it directly. Use MacroAssembler::decrementq() instead.
8317   // Use two-byte form (one-byte from is a REX prefix in 64-bit mode)
8318   int encode = prefixq_and_encode(dst->encoding());
8319   emit_int8((unsigned char)0xFF);
8320   emit_int8(0xC8 | encode);
8321 }
8322 
8323 void Assembler::decq(Address dst) {
8324   // Don't use it directly. Use MacroAssembler::decrementq() instead.
8325   InstructionMark im(this);
8326   prefixq(dst);
8327   emit_int8((unsigned char)0xFF);
8328   emit_operand(rcx, dst);
8329 }
8330 
8331 void Assembler::fxrstor(Address src) {
8332   prefixq(src);
8333   emit_int8(0x0F);
8334   emit_int8((unsigned char)0xAE);
8335   emit_operand(as_Register(1), src);
8336 }
8337 
8338 void Assembler::xrstor(Address src) {
8339   prefixq(src);
8340   emit_int8(0x0F);
8341   emit_int8((unsigned char)0xAE);
8342   emit_operand(as_Register(5), src);
8343 }
8344 
8345 void Assembler::fxsave(Address dst) {
8346   prefixq(dst);
8347   emit_int8(0x0F);
8348   emit_int8((unsigned char)0xAE);
8349   emit_operand(as_Register(0), dst);
8350 }
8351 
8352 void Assembler::xsave(Address dst) {
8353   prefixq(dst);
8354   emit_int8(0x0F);
8355   emit_int8((unsigned char)0xAE);
8356   emit_operand(as_Register(4), dst);
8357 }
8358 
8359 void Assembler::idivq(Register src) {
8360   int encode = prefixq_and_encode(src->encoding());
8361   emit_int8((unsigned char)0xF7);
8362   emit_int8((unsigned char)(0xF8 | encode));
8363 }
8364 
8365 void Assembler::imulq(Register dst, Register src) {
8366   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
8367   emit_int8(0x0F);
8368   emit_int8((unsigned char)0xAF);
8369   emit_int8((unsigned char)(0xC0 | encode));
8370 }
8371 
8372 void Assembler::imulq(Register dst, Register src, int value) {
8373   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
8374   if (is8bit(value)) {
8375     emit_int8(0x6B);
8376     emit_int8((unsigned char)(0xC0 | encode));
8377     emit_int8(value & 0xFF);
8378   } else {
8379     emit_int8(0x69);
8380     emit_int8((unsigned char)(0xC0 | encode));
8381     emit_int32(value);
8382   }
8383 }
8384 
8385 void Assembler::imulq(Register dst, Address src) {
8386   InstructionMark im(this);
8387   prefixq(src, dst);
8388   emit_int8(0x0F);
8389   emit_int8((unsigned char) 0xAF);
8390   emit_operand(dst, src);
8391 }
8392 
8393 void Assembler::incl(Register dst) {
8394   // Don't use it directly. Use MacroAssembler::incrementl() instead.
8395   // Use two-byte form (one-byte from is a REX prefix in 64-bit mode)
8396   int encode = prefix_and_encode(dst->encoding());
8397   emit_int8((unsigned char)0xFF);
8398   emit_int8((unsigned char)(0xC0 | encode));
8399 }
8400 
8401 void Assembler::incq(Register dst) {
8402   // Don't use it directly. Use MacroAssembler::incrementq() instead.
8403   // Use two-byte form (one-byte from is a REX prefix in 64-bit mode)
8404   int encode = prefixq_and_encode(dst->encoding());
8405   emit_int8((unsigned char)0xFF);
8406   emit_int8((unsigned char)(0xC0 | encode));
8407 }
8408 
8409 void Assembler::incq(Address dst) {
8410   // Don't use it directly. Use MacroAssembler::incrementq() instead.
8411   InstructionMark im(this);
8412   prefixq(dst);
8413   emit_int8((unsigned char)0xFF);
8414   emit_operand(rax, dst);
8415 }
8416 
8417 void Assembler::lea(Register dst, Address src) {
8418   leaq(dst, src);
8419 }
8420 
8421 void Assembler::leaq(Register dst, Address src) {
8422   InstructionMark im(this);
8423   prefixq(src, dst);
8424   emit_int8((unsigned char)0x8D);
8425   emit_operand(dst, src);
8426 }
8427 
8428 void Assembler::mov64(Register dst, int64_t imm64) {
8429   InstructionMark im(this);
8430   int encode = prefixq_and_encode(dst->encoding());
8431   emit_int8((unsigned char)(0xB8 | encode));
8432   emit_int64(imm64);
8433 }
8434 
8435 void Assembler::mov_literal64(Register dst, intptr_t imm64, RelocationHolder const& rspec) {
8436   InstructionMark im(this);
8437   int encode = prefixq_and_encode(dst->encoding());
8438   emit_int8(0xB8 | encode);
8439   emit_data64(imm64, rspec);
8440 }
8441 
8442 void Assembler::mov_narrow_oop(Register dst, int32_t imm32, RelocationHolder const& rspec) {
8443   InstructionMark im(this);
8444   int encode = prefix_and_encode(dst->encoding());
8445   emit_int8((unsigned char)(0xB8 | encode));
8446   emit_data((int)imm32, rspec, narrow_oop_operand);
8447 }
8448 
8449 void Assembler::mov_narrow_oop(Address dst, int32_t imm32,  RelocationHolder const& rspec) {
8450   InstructionMark im(this);
8451   prefix(dst);
8452   emit_int8((unsigned char)0xC7);
8453   emit_operand(rax, dst, 4);
8454   emit_data((int)imm32, rspec, narrow_oop_operand);
8455 }
8456 
8457 void Assembler::cmp_narrow_oop(Register src1, int32_t imm32, RelocationHolder const& rspec) {
8458   InstructionMark im(this);
8459   int encode = prefix_and_encode(src1->encoding());
8460   emit_int8((unsigned char)0x81);
8461   emit_int8((unsigned char)(0xF8 | encode));
8462   emit_data((int)imm32, rspec, narrow_oop_operand);
8463 }
8464 
8465 void Assembler::cmp_narrow_oop(Address src1, int32_t imm32, RelocationHolder const& rspec) {
8466   InstructionMark im(this);
8467   prefix(src1);
8468   emit_int8((unsigned char)0x81);
8469   emit_operand(rax, src1, 4);
8470   emit_data((int)imm32, rspec, narrow_oop_operand);
8471 }
8472 
8473 void Assembler::lzcntq(Register dst, Register src) {
8474   assert(VM_Version::supports_lzcnt(), "encoding is treated as BSR");
8475   emit_int8((unsigned char)0xF3);
8476   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
8477   emit_int8(0x0F);
8478   emit_int8((unsigned char)0xBD);
8479   emit_int8((unsigned char)(0xC0 | encode));
8480 }
8481 
8482 void Assembler::movdq(XMMRegister dst, Register src) {
8483   // table D-1 says MMX/SSE2
8484   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
8485   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
8486   int encode = simd_prefix_and_encode(dst, xnoreg, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
8487   emit_int8(0x6E);
8488   emit_int8((unsigned char)(0xC0 | encode));
8489 }
8490 
8491 void Assembler::movdq(Register dst, XMMRegister src) {
8492   // table D-1 says MMX/SSE2
8493   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
8494   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
8495   // swap src/dst to get correct prefix
8496   int encode = simd_prefix_and_encode(src, xnoreg, as_XMMRegister(dst->encoding()), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
8497   emit_int8(0x7E);
8498   emit_int8((unsigned char)(0xC0 | encode));
8499 }
8500 
8501 void Assembler::movq(Register dst, Register src) {
8502   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
8503   emit_int8((unsigned char)0x8B);
8504   emit_int8((unsigned char)(0xC0 | encode));
8505 }
8506 
8507 void Assembler::movq(Register dst, Address src) {
8508   InstructionMark im(this);
8509   prefixq(src, dst);
8510   emit_int8((unsigned char)0x8B);
8511   emit_operand(dst, src);
8512 }
8513 
8514 void Assembler::movq(Address dst, Register src) {
8515   InstructionMark im(this);
8516   prefixq(dst, src);
8517   emit_int8((unsigned char)0x89);
8518   emit_operand(src, dst);
8519 }
8520 
8521 void Assembler::movsbq(Register dst, Address src) {
8522   InstructionMark im(this);
8523   prefixq(src, dst);
8524   emit_int8(0x0F);
8525   emit_int8((unsigned char)0xBE);
8526   emit_operand(dst, src);
8527 }
8528 
8529 void Assembler::movsbq(Register dst, Register src) {
8530   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
8531   emit_int8(0x0F);
8532   emit_int8((unsigned char)0xBE);
8533   emit_int8((unsigned char)(0xC0 | encode));
8534 }
8535 
8536 void Assembler::movslq(Register dst, int32_t imm32) {
8537   // dbx shows movslq(rcx, 3) as movq     $0x0000000049000000,(%rbx)
8538   // and movslq(r8, 3); as movl     $0x0000000048000000,(%rbx)
8539   // as a result we shouldn't use until tested at runtime...
8540   ShouldNotReachHere();
8541   InstructionMark im(this);
8542   int encode = prefixq_and_encode(dst->encoding());
8543   emit_int8((unsigned char)(0xC7 | encode));
8544   emit_int32(imm32);
8545 }
8546 
8547 void Assembler::movslq(Address dst, int32_t imm32) {
8548   assert(is_simm32(imm32), "lost bits");
8549   InstructionMark im(this);
8550   prefixq(dst);
8551   emit_int8((unsigned char)0xC7);
8552   emit_operand(rax, dst, 4);
8553   emit_int32(imm32);
8554 }
8555 
8556 void Assembler::movslq(Register dst, Address src) {
8557   InstructionMark im(this);
8558   prefixq(src, dst);
8559   emit_int8(0x63);
8560   emit_operand(dst, src);
8561 }
8562 
8563 void Assembler::movslq(Register dst, Register src) {
8564   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
8565   emit_int8(0x63);
8566   emit_int8((unsigned char)(0xC0 | encode));
8567 }
8568 
8569 void Assembler::movswq(Register dst, Address src) {
8570   InstructionMark im(this);
8571   prefixq(src, dst);
8572   emit_int8(0x0F);
8573   emit_int8((unsigned char)0xBF);
8574   emit_operand(dst, src);
8575 }
8576 
8577 void Assembler::movswq(Register dst, Register src) {
8578   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
8579   emit_int8((unsigned char)0x0F);
8580   emit_int8((unsigned char)0xBF);
8581   emit_int8((unsigned char)(0xC0 | encode));
8582 }
8583 
8584 void Assembler::movzbq(Register dst, Address src) {
8585   InstructionMark im(this);
8586   prefixq(src, dst);
8587   emit_int8((unsigned char)0x0F);
8588   emit_int8((unsigned char)0xB6);
8589   emit_operand(dst, src);
8590 }
8591 
8592 void Assembler::movzbq(Register dst, Register src) {
8593   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
8594   emit_int8(0x0F);
8595   emit_int8((unsigned char)0xB6);
8596   emit_int8(0xC0 | encode);
8597 }
8598 
8599 void Assembler::movzwq(Register dst, Address src) {
8600   InstructionMark im(this);
8601   prefixq(src, dst);
8602   emit_int8((unsigned char)0x0F);
8603   emit_int8((unsigned char)0xB7);
8604   emit_operand(dst, src);
8605 }
8606 
8607 void Assembler::movzwq(Register dst, Register src) {
8608   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
8609   emit_int8((unsigned char)0x0F);
8610   emit_int8((unsigned char)0xB7);
8611   emit_int8((unsigned char)(0xC0 | encode));
8612 }
8613 
8614 void Assembler::mulq(Address src) {
8615   InstructionMark im(this);
8616   prefixq(src);
8617   emit_int8((unsigned char)0xF7);
8618   emit_operand(rsp, src);
8619 }
8620 
8621 void Assembler::mulq(Register src) {
8622   int encode = prefixq_and_encode(src->encoding());
8623   emit_int8((unsigned char)0xF7);
8624   emit_int8((unsigned char)(0xE0 | encode));
8625 }
8626 
8627 void Assembler::mulxq(Register dst1, Register dst2, Register src) {
8628   assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
8629   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
8630   int encode = vex_prefix_and_encode(dst1->encoding(), dst2->encoding(), src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_38, &attributes);
8631   emit_int8((unsigned char)0xF6);
8632   emit_int8((unsigned char)(0xC0 | encode));
8633 }
8634 
8635 void Assembler::negq(Register dst) {
8636   int encode = prefixq_and_encode(dst->encoding());
8637   emit_int8((unsigned char)0xF7);
8638   emit_int8((unsigned char)(0xD8 | encode));
8639 }
8640 
8641 void Assembler::notq(Register dst) {
8642   int encode = prefixq_and_encode(dst->encoding());
8643   emit_int8((unsigned char)0xF7);
8644   emit_int8((unsigned char)(0xD0 | encode));
8645 }
8646 
8647 void Assembler::orq(Address dst, int32_t imm32) {
8648   InstructionMark im(this);
8649   prefixq(dst);
8650   emit_int8((unsigned char)0x81);
8651   emit_operand(rcx, dst, 4);
8652   emit_int32(imm32);
8653 }
8654 
8655 void Assembler::orq(Register dst, int32_t imm32) {
8656   (void) prefixq_and_encode(dst->encoding());
8657   emit_arith(0x81, 0xC8, dst, imm32);
8658 }
8659 
8660 void Assembler::orq(Register dst, Address src) {
8661   InstructionMark im(this);
8662   prefixq(src, dst);
8663   emit_int8(0x0B);
8664   emit_operand(dst, src);
8665 }
8666 
8667 void Assembler::orq(Register dst, Register src) {
8668   (void) prefixq_and_encode(dst->encoding(), src->encoding());
8669   emit_arith(0x0B, 0xC0, dst, src);
8670 }
8671 
8672 void Assembler::popa() { // 64bit
8673   movq(r15, Address(rsp, 0));
8674   movq(r14, Address(rsp, wordSize));
8675   movq(r13, Address(rsp, 2 * wordSize));
8676   movq(r12, Address(rsp, 3 * wordSize));
8677   movq(r11, Address(rsp, 4 * wordSize));
8678   movq(r10, Address(rsp, 5 * wordSize));
8679   movq(r9,  Address(rsp, 6 * wordSize));
8680   movq(r8,  Address(rsp, 7 * wordSize));
8681   movq(rdi, Address(rsp, 8 * wordSize));
8682   movq(rsi, Address(rsp, 9 * wordSize));
8683   movq(rbp, Address(rsp, 10 * wordSize));
8684   // skip rsp
8685   movq(rbx, Address(rsp, 12 * wordSize));
8686   movq(rdx, Address(rsp, 13 * wordSize));
8687   movq(rcx, Address(rsp, 14 * wordSize));
8688   movq(rax, Address(rsp, 15 * wordSize));
8689 
8690   addq(rsp, 16 * wordSize);
8691 }
8692 
8693 void Assembler::popcntq(Register dst, Address src) {
8694   assert(VM_Version::supports_popcnt(), "must support");
8695   InstructionMark im(this);
8696   emit_int8((unsigned char)0xF3);
8697   prefixq(src, dst);
8698   emit_int8((unsigned char)0x0F);
8699   emit_int8((unsigned char)0xB8);
8700   emit_operand(dst, src);
8701 }
8702 
8703 void Assembler::popcntq(Register dst, Register src) {
8704   assert(VM_Version::supports_popcnt(), "must support");
8705   emit_int8((unsigned char)0xF3);
8706   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
8707   emit_int8((unsigned char)0x0F);
8708   emit_int8((unsigned char)0xB8);
8709   emit_int8((unsigned char)(0xC0 | encode));
8710 }
8711 
8712 void Assembler::vpopcntd(XMMRegister dst, XMMRegister src, int vector_len) {
8713   assert(UseAVX > 2 && VM_Version::support_avx512_vpopcntdq(), "must support vpopcntdq feature");
8714   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
8715   attributes.set_is_evex_instruction();
8716   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
8717   emit_int8(0x55);
8718   emit_int8((unsigned char)(0xC0 | encode));
8719 }
8720 
8721 void Assembler::vpopcntq(XMMRegister dst, XMMRegister src, int vector_len) {
8722   assert(UseAVX > 2 && VM_Version::support_avx512_vpopcntdq(), "must support vpopcntdq feature");
8723   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
8724   attributes.set_is_evex_instruction();
8725   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
8726   emit_int8(0x55);
8727   emit_int8((unsigned char)(0xC0 | encode));
8728 }
8729 
8730 void Assembler::popq(Address dst) {
8731   InstructionMark im(this);
8732   prefixq(dst);
8733   emit_int8((unsigned char)0x8F);
8734   emit_operand(rax, dst);
8735 }
8736 
8737 void Assembler::pusha() { // 64bit
8738   // we have to store original rsp.  ABI says that 128 bytes
8739   // below rsp are local scratch.
8740   movq(Address(rsp, -5 * wordSize), rsp);
8741 
8742   subq(rsp, 16 * wordSize);
8743 
8744   movq(Address(rsp, 15 * wordSize), rax);
8745   movq(Address(rsp, 14 * wordSize), rcx);
8746   movq(Address(rsp, 13 * wordSize), rdx);
8747   movq(Address(rsp, 12 * wordSize), rbx);
8748   // skip rsp
8749   movq(Address(rsp, 10 * wordSize), rbp);
8750   movq(Address(rsp, 9 * wordSize), rsi);
8751   movq(Address(rsp, 8 * wordSize), rdi);
8752   movq(Address(rsp, 7 * wordSize), r8);
8753   movq(Address(rsp, 6 * wordSize), r9);
8754   movq(Address(rsp, 5 * wordSize), r10);
8755   movq(Address(rsp, 4 * wordSize), r11);
8756   movq(Address(rsp, 3 * wordSize), r12);
8757   movq(Address(rsp, 2 * wordSize), r13);
8758   movq(Address(rsp, wordSize), r14);
8759   movq(Address(rsp, 0), r15);
8760 }
8761 
8762 void Assembler::pushq(Address src) {
8763   InstructionMark im(this);
8764   prefixq(src);
8765   emit_int8((unsigned char)0xFF);
8766   emit_operand(rsi, src);
8767 }
8768 
8769 void Assembler::rclq(Register dst, int imm8) {
8770   assert(isShiftCount(imm8 >> 1), "illegal shift count");
8771   int encode = prefixq_and_encode(dst->encoding());
8772   if (imm8 == 1) {
8773     emit_int8((unsigned char)0xD1);
8774     emit_int8((unsigned char)(0xD0 | encode));
8775   } else {
8776     emit_int8((unsigned char)0xC1);
8777     emit_int8((unsigned char)(0xD0 | encode));
8778     emit_int8(imm8);
8779   }
8780 }
8781 
8782 void Assembler::rcrq(Register dst, int imm8) {
8783   assert(isShiftCount(imm8 >> 1), "illegal shift count");
8784   int encode = prefixq_and_encode(dst->encoding());
8785   if (imm8 == 1) {
8786     emit_int8((unsigned char)0xD1);
8787     emit_int8((unsigned char)(0xD8 | encode));
8788   } else {
8789     emit_int8((unsigned char)0xC1);
8790     emit_int8((unsigned char)(0xD8 | encode));
8791     emit_int8(imm8);
8792   }
8793 }
8794 
8795 void Assembler::rorq(Register dst, int imm8) {
8796   assert(isShiftCount(imm8 >> 1), "illegal shift count");
8797   int encode = prefixq_and_encode(dst->encoding());
8798   if (imm8 == 1) {
8799     emit_int8((unsigned char)0xD1);
8800     emit_int8((unsigned char)(0xC8 | encode));
8801   } else {
8802     emit_int8((unsigned char)0xC1);
8803     emit_int8((unsigned char)(0xc8 | encode));
8804     emit_int8(imm8);
8805   }
8806 }
8807 
8808 void Assembler::rorxq(Register dst, Register src, int imm8) {
8809   assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
8810   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
8811   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_3A, &attributes);
8812   emit_int8((unsigned char)0xF0);
8813   emit_int8((unsigned char)(0xC0 | encode));
8814   emit_int8(imm8);
8815 }
8816 
8817 void Assembler::rorxd(Register dst, Register src, int imm8) {
8818   assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
8819   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
8820   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_3A, &attributes);
8821   emit_int8((unsigned char)0xF0);
8822   emit_int8((unsigned char)(0xC0 | encode));
8823   emit_int8(imm8);
8824 }
8825 
8826 void Assembler::sarq(Register dst, int imm8) {
8827   assert(isShiftCount(imm8 >> 1), "illegal shift count");
8828   int encode = prefixq_and_encode(dst->encoding());
8829   if (imm8 == 1) {
8830     emit_int8((unsigned char)0xD1);
8831     emit_int8((unsigned char)(0xF8 | encode));
8832   } else {
8833     emit_int8((unsigned char)0xC1);
8834     emit_int8((unsigned char)(0xF8 | encode));
8835     emit_int8(imm8);
8836   }
8837 }
8838 
8839 void Assembler::sarq(Register dst) {
8840   int encode = prefixq_and_encode(dst->encoding());
8841   emit_int8((unsigned char)0xD3);
8842   emit_int8((unsigned char)(0xF8 | encode));
8843 }
8844 
8845 void Assembler::sbbq(Address dst, int32_t imm32) {
8846   InstructionMark im(this);
8847   prefixq(dst);
8848   emit_arith_operand(0x81, rbx, dst, imm32);
8849 }
8850 
8851 void Assembler::sbbq(Register dst, int32_t imm32) {
8852   (void) prefixq_and_encode(dst->encoding());
8853   emit_arith(0x81, 0xD8, dst, imm32);
8854 }
8855 
8856 void Assembler::sbbq(Register dst, Address src) {
8857   InstructionMark im(this);
8858   prefixq(src, dst);
8859   emit_int8(0x1B);
8860   emit_operand(dst, src);
8861 }
8862 
8863 void Assembler::sbbq(Register dst, Register src) {
8864   (void) prefixq_and_encode(dst->encoding(), src->encoding());
8865   emit_arith(0x1B, 0xC0, dst, src);
8866 }
8867 
8868 void Assembler::shlq(Register dst, int imm8) {
8869   assert(isShiftCount(imm8 >> 1), "illegal shift count");
8870   int encode = prefixq_and_encode(dst->encoding());
8871   if (imm8 == 1) {
8872     emit_int8((unsigned char)0xD1);
8873     emit_int8((unsigned char)(0xE0 | encode));
8874   } else {
8875     emit_int8((unsigned char)0xC1);
8876     emit_int8((unsigned char)(0xE0 | encode));
8877     emit_int8(imm8);
8878   }
8879 }
8880 
8881 void Assembler::shlq(Register dst) {
8882   int encode = prefixq_and_encode(dst->encoding());
8883   emit_int8((unsigned char)0xD3);
8884   emit_int8((unsigned char)(0xE0 | encode));
8885 }
8886 
8887 void Assembler::shrq(Register dst, int imm8) {
8888   assert(isShiftCount(imm8 >> 1), "illegal shift count");
8889   int encode = prefixq_and_encode(dst->encoding());
8890   emit_int8((unsigned char)0xC1);
8891   emit_int8((unsigned char)(0xE8 | encode));
8892   emit_int8(imm8);
8893 }
8894 
8895 void Assembler::shrq(Register dst) {
8896   int encode = prefixq_and_encode(dst->encoding());
8897   emit_int8((unsigned char)0xD3);
8898   emit_int8(0xE8 | encode);
8899 }
8900 
8901 void Assembler::subq(Address dst, int32_t imm32) {
8902   InstructionMark im(this);
8903   prefixq(dst);
8904   emit_arith_operand(0x81, rbp, dst, imm32);
8905 }
8906 
8907 void Assembler::subq(Address dst, Register src) {
8908   InstructionMark im(this);
8909   prefixq(dst, src);
8910   emit_int8(0x29);
8911   emit_operand(src, dst);
8912 }
8913 
8914 void Assembler::subq(Register dst, int32_t imm32) {
8915   (void) prefixq_and_encode(dst->encoding());
8916   emit_arith(0x81, 0xE8, dst, imm32);
8917 }
8918 
8919 // Force generation of a 4 byte immediate value even if it fits into 8bit
8920 void Assembler::subq_imm32(Register dst, int32_t imm32) {
8921   (void) prefixq_and_encode(dst->encoding());
8922   emit_arith_imm32(0x81, 0xE8, dst, imm32);
8923 }
8924 
8925 void Assembler::subq(Register dst, Address src) {
8926   InstructionMark im(this);
8927   prefixq(src, dst);
8928   emit_int8(0x2B);
8929   emit_operand(dst, src);
8930 }
8931 
8932 void Assembler::subq(Register dst, Register src) {
8933   (void) prefixq_and_encode(dst->encoding(), src->encoding());
8934   emit_arith(0x2B, 0xC0, dst, src);
8935 }
8936 
8937 void Assembler::testq(Register dst, int32_t imm32) {
8938   // not using emit_arith because test
8939   // doesn't support sign-extension of
8940   // 8bit operands
8941   int encode = dst->encoding();
8942   if (encode == 0) {
8943     prefix(REX_W);
8944     emit_int8((unsigned char)0xA9);
8945   } else {
8946     encode = prefixq_and_encode(encode);
8947     emit_int8((unsigned char)0xF7);
8948     emit_int8((unsigned char)(0xC0 | encode));
8949   }
8950   emit_int32(imm32);
8951 }
8952 
8953 void Assembler::testq(Register dst, Register src) {
8954   (void) prefixq_and_encode(dst->encoding(), src->encoding());
8955   emit_arith(0x85, 0xC0, dst, src);
8956 }
8957 
8958 void Assembler::xaddq(Address dst, Register src) {
8959   InstructionMark im(this);
8960   prefixq(dst, src);
8961   emit_int8(0x0F);
8962   emit_int8((unsigned char)0xC1);
8963   emit_operand(src, dst);
8964 }
8965 
8966 void Assembler::xchgq(Register dst, Address src) {
8967   InstructionMark im(this);
8968   prefixq(src, dst);
8969   emit_int8((unsigned char)0x87);
8970   emit_operand(dst, src);
8971 }
8972 
8973 void Assembler::xchgq(Register dst, Register src) {
8974   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
8975   emit_int8((unsigned char)0x87);
8976   emit_int8((unsigned char)(0xc0 | encode));
8977 }
8978 
8979 void Assembler::xorq(Register dst, Register src) {
8980   (void) prefixq_and_encode(dst->encoding(), src->encoding());
8981   emit_arith(0x33, 0xC0, dst, src);
8982 }
8983 
8984 void Assembler::xorq(Register dst, Address src) {
8985   InstructionMark im(this);
8986   prefixq(src, dst);
8987   emit_int8(0x33);
8988   emit_operand(dst, src);
8989 }
8990 
8991 #endif // !LP64