1 /*
   2  * Copyright (c) 1997, 2016, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #include "precompiled.hpp"
  26 #include "asm/assembler.hpp"
  27 #include "asm/assembler.inline.hpp"
  28 #include "gc/shared/cardTableModRefBS.hpp"
  29 #include "gc/shared/collectedHeap.inline.hpp"
  30 #include "interpreter/interpreter.hpp"
  31 #include "memory/resourceArea.hpp"
  32 #include "prims/methodHandles.hpp"
  33 #include "runtime/biasedLocking.hpp"
  34 #include "runtime/interfaceSupport.hpp"
  35 #include "runtime/objectMonitor.hpp"
  36 #include "runtime/os.hpp"
  37 #include "runtime/sharedRuntime.hpp"
  38 #include "runtime/stubRoutines.hpp"
  39 #include "utilities/macros.hpp"
  40 #if INCLUDE_ALL_GCS
  41 #include "gc/g1/g1CollectedHeap.inline.hpp"
  42 #include "gc/g1/g1SATBCardTableModRefBS.hpp"
  43 #include "gc/g1/heapRegion.hpp"
  44 #endif // INCLUDE_ALL_GCS
  45 
  46 #ifdef PRODUCT
  47 #define BLOCK_COMMENT(str) /* nothing */
  48 #define STOP(error) stop(error)
  49 #else
  50 #define BLOCK_COMMENT(str) block_comment(str)
  51 #define STOP(error) block_comment(error); stop(error)
  52 #endif
  53 
  54 #define BIND(label) bind(label); BLOCK_COMMENT(#label ":")
  55 // Implementation of AddressLiteral
  56 
  57 // A 2-D table for managing compressed displacement(disp8) on EVEX enabled platforms.
  58 unsigned char tuple_table[Assembler::EVEX_ETUP + 1][Assembler::AVX_512bit + 1] = {
  59   // -----------------Table 4.5 -------------------- //
  60   16, 32, 64,  // EVEX_FV(0)
  61   4,  4,  4,   // EVEX_FV(1) - with Evex.b
  62   16, 32, 64,  // EVEX_FV(2) - with Evex.w
  63   8,  8,  8,   // EVEX_FV(3) - with Evex.w and Evex.b
  64   8,  16, 32,  // EVEX_HV(0)
  65   4,  4,  4,   // EVEX_HV(1) - with Evex.b
  66   // -----------------Table 4.6 -------------------- //
  67   16, 32, 64,  // EVEX_FVM(0)
  68   1,  1,  1,   // EVEX_T1S(0)
  69   2,  2,  2,   // EVEX_T1S(1)
  70   4,  4,  4,   // EVEX_T1S(2)
  71   8,  8,  8,   // EVEX_T1S(3)
  72   4,  4,  4,   // EVEX_T1F(0)
  73   8,  8,  8,   // EVEX_T1F(1)
  74   8,  8,  8,   // EVEX_T2(0)
  75   0,  16, 16,  // EVEX_T2(1)
  76   0,  16, 16,  // EVEX_T4(0)
  77   0,  0,  32,  // EVEX_T4(1)
  78   0,  0,  32,  // EVEX_T8(0)
  79   8,  16, 32,  // EVEX_HVM(0)
  80   4,  8,  16,  // EVEX_QVM(0)
  81   2,  4,  8,   // EVEX_OVM(0)
  82   16, 16, 16,  // EVEX_M128(0)
  83   8,  32, 64,  // EVEX_DUP(0)
  84   0,  0,  0    // EVEX_NTUP
  85 };
  86 
  87 AddressLiteral::AddressLiteral(address target, relocInfo::relocType rtype) {
  88   _is_lval = false;
  89   _target = target;
  90   switch (rtype) {
  91   case relocInfo::oop_type:
  92   case relocInfo::metadata_type:
  93     // Oops are a special case. Normally they would be their own section
  94     // but in cases like icBuffer they are literals in the code stream that
  95     // we don't have a section for. We use none so that we get a literal address
  96     // which is always patchable.
  97     break;
  98   case relocInfo::external_word_type:
  99     _rspec = external_word_Relocation::spec(target);
 100     break;
 101   case relocInfo::internal_word_type:
 102     _rspec = internal_word_Relocation::spec(target);
 103     break;
 104   case relocInfo::opt_virtual_call_type:
 105     _rspec = opt_virtual_call_Relocation::spec();
 106     break;
 107   case relocInfo::static_call_type:
 108     _rspec = static_call_Relocation::spec();
 109     break;
 110   case relocInfo::runtime_call_type:
 111     _rspec = runtime_call_Relocation::spec();
 112     break;
 113   case relocInfo::poll_type:
 114   case relocInfo::poll_return_type:
 115     _rspec = Relocation::spec_simple(rtype);
 116     break;
 117   case relocInfo::none:
 118     break;
 119   default:
 120     ShouldNotReachHere();
 121     break;
 122   }
 123 }
 124 
 125 // Implementation of Address
 126 
 127 #ifdef _LP64
 128 
 129 Address Address::make_array(ArrayAddress adr) {
 130   // Not implementable on 64bit machines
 131   // Should have been handled higher up the call chain.
 132   ShouldNotReachHere();
 133   return Address();
 134 }
 135 
 136 // exceedingly dangerous constructor
 137 Address::Address(int disp, address loc, relocInfo::relocType rtype) {
 138   _base  = noreg;
 139   _index = noreg;
 140   _scale = no_scale;
 141   _disp  = disp;
 142   switch (rtype) {
 143     case relocInfo::external_word_type:
 144       _rspec = external_word_Relocation::spec(loc);
 145       break;
 146     case relocInfo::internal_word_type:
 147       _rspec = internal_word_Relocation::spec(loc);
 148       break;
 149     case relocInfo::runtime_call_type:
 150       // HMM
 151       _rspec = runtime_call_Relocation::spec();
 152       break;
 153     case relocInfo::poll_type:
 154     case relocInfo::poll_return_type:
 155       _rspec = Relocation::spec_simple(rtype);
 156       break;
 157     case relocInfo::none:
 158       break;
 159     default:
 160       ShouldNotReachHere();
 161   }
 162 }
 163 #else // LP64
 164 
 165 Address Address::make_array(ArrayAddress adr) {
 166   AddressLiteral base = adr.base();
 167   Address index = adr.index();
 168   assert(index._disp == 0, "must not have disp"); // maybe it can?
 169   Address array(index._base, index._index, index._scale, (intptr_t) base.target());
 170   array._rspec = base._rspec;
 171   return array;
 172 }
 173 
 174 // exceedingly dangerous constructor
 175 Address::Address(address loc, RelocationHolder spec) {
 176   _base  = noreg;
 177   _index = noreg;
 178   _scale = no_scale;
 179   _disp  = (intptr_t) loc;
 180   _rspec = spec;
 181 }
 182 
 183 #endif // _LP64
 184 
 185 
 186 
 187 // Convert the raw encoding form into the form expected by the constructor for
 188 // Address.  An index of 4 (rsp) corresponds to having no index, so convert
 189 // that to noreg for the Address constructor.
 190 Address Address::make_raw(int base, int index, int scale, int disp, relocInfo::relocType disp_reloc) {
 191   RelocationHolder rspec;
 192   if (disp_reloc != relocInfo::none) {
 193     rspec = Relocation::spec_simple(disp_reloc);
 194   }
 195   bool valid_index = index != rsp->encoding();
 196   if (valid_index) {
 197     Address madr(as_Register(base), as_Register(index), (Address::ScaleFactor)scale, in_ByteSize(disp));
 198     madr._rspec = rspec;
 199     return madr;
 200   } else {
 201     Address madr(as_Register(base), noreg, Address::no_scale, in_ByteSize(disp));
 202     madr._rspec = rspec;
 203     return madr;
 204   }
 205 }
 206 
 207 // Implementation of Assembler
 208 
 209 int AbstractAssembler::code_fill_byte() {
 210   return (u_char)'\xF4'; // hlt
 211 }
 212 
 213 // make this go away someday
 214 void Assembler::emit_data(jint data, relocInfo::relocType rtype, int format) {
 215   if (rtype == relocInfo::none)
 216     emit_int32(data);
 217   else
 218     emit_data(data, Relocation::spec_simple(rtype), format);
 219 }
 220 
 221 void Assembler::emit_data(jint data, RelocationHolder const& rspec, int format) {
 222   assert(imm_operand == 0, "default format must be immediate in this file");
 223   assert(inst_mark() != NULL, "must be inside InstructionMark");
 224   if (rspec.type() !=  relocInfo::none) {
 225     #ifdef ASSERT
 226       check_relocation(rspec, format);
 227     #endif
 228     // Do not use AbstractAssembler::relocate, which is not intended for
 229     // embedded words.  Instead, relocate to the enclosing instruction.
 230 
 231     // hack. call32 is too wide for mask so use disp32
 232     if (format == call32_operand)
 233       code_section()->relocate(inst_mark(), rspec, disp32_operand);
 234     else
 235       code_section()->relocate(inst_mark(), rspec, format);
 236   }
 237   emit_int32(data);
 238 }
 239 
 240 static int encode(Register r) {
 241   int enc = r->encoding();
 242   if (enc >= 8) {
 243     enc -= 8;
 244   }
 245   return enc;
 246 }
 247 
 248 void Assembler::emit_arith_b(int op1, int op2, Register dst, int imm8) {
 249   assert(dst->has_byte_register(), "must have byte register");
 250   assert(isByte(op1) && isByte(op2), "wrong opcode");
 251   assert(isByte(imm8), "not a byte");
 252   assert((op1 & 0x01) == 0, "should be 8bit operation");
 253   emit_int8(op1);
 254   emit_int8(op2 | encode(dst));
 255   emit_int8(imm8);
 256 }
 257 
 258 
 259 void Assembler::emit_arith(int op1, int op2, Register dst, int32_t imm32) {
 260   assert(isByte(op1) && isByte(op2), "wrong opcode");
 261   assert((op1 & 0x01) == 1, "should be 32bit operation");
 262   assert((op1 & 0x02) == 0, "sign-extension bit should not be set");
 263   if (is8bit(imm32)) {
 264     emit_int8(op1 | 0x02); // set sign bit
 265     emit_int8(op2 | encode(dst));
 266     emit_int8(imm32 & 0xFF);
 267   } else {
 268     emit_int8(op1);
 269     emit_int8(op2 | encode(dst));
 270     emit_int32(imm32);
 271   }
 272 }
 273 
 274 // Force generation of a 4 byte immediate value even if it fits into 8bit
 275 void Assembler::emit_arith_imm32(int op1, int op2, Register dst, int32_t imm32) {
 276   assert(isByte(op1) && isByte(op2), "wrong opcode");
 277   assert((op1 & 0x01) == 1, "should be 32bit operation");
 278   assert((op1 & 0x02) == 0, "sign-extension bit should not be set");
 279   emit_int8(op1);
 280   emit_int8(op2 | encode(dst));
 281   emit_int32(imm32);
 282 }
 283 
 284 // immediate-to-memory forms
 285 void Assembler::emit_arith_operand(int op1, Register rm, Address adr, int32_t imm32) {
 286   assert((op1 & 0x01) == 1, "should be 32bit operation");
 287   assert((op1 & 0x02) == 0, "sign-extension bit should not be set");
 288   if (is8bit(imm32)) {
 289     emit_int8(op1 | 0x02); // set sign bit
 290     emit_operand(rm, adr, 1);
 291     emit_int8(imm32 & 0xFF);
 292   } else {
 293     emit_int8(op1);
 294     emit_operand(rm, adr, 4);
 295     emit_int32(imm32);
 296   }
 297 }
 298 
 299 
 300 void Assembler::emit_arith(int op1, int op2, Register dst, Register src) {
 301   assert(isByte(op1) && isByte(op2), "wrong opcode");
 302   emit_int8(op1);
 303   emit_int8(op2 | encode(dst) << 3 | encode(src));
 304 }
 305 
 306 
 307 bool Assembler::query_compressed_disp_byte(int disp, bool is_evex_inst, int vector_len,
 308                                            int cur_tuple_type, int in_size_in_bits, int cur_encoding) {
 309   int mod_idx = 0;
 310   // We will test if the displacement fits the compressed format and if so
 311   // apply the compression to the displacment iff the result is8bit.
 312   if (VM_Version::supports_evex() && is_evex_inst) {
 313     switch (cur_tuple_type) {
 314     case EVEX_FV:
 315       if ((cur_encoding & VEX_W) == VEX_W) {
 316         mod_idx = ((cur_encoding & EVEX_Rb) == EVEX_Rb) ? 3 : 2;
 317       } else {
 318         mod_idx = ((cur_encoding & EVEX_Rb) == EVEX_Rb) ? 1 : 0;
 319       }
 320       break;
 321 
 322     case EVEX_HV:
 323       mod_idx = ((cur_encoding & EVEX_Rb) == EVEX_Rb) ? 1 : 0;
 324       break;
 325 
 326     case EVEX_FVM:
 327       break;
 328 
 329     case EVEX_T1S:
 330       switch (in_size_in_bits) {
 331       case EVEX_8bit:
 332         break;
 333 
 334       case EVEX_16bit:
 335         mod_idx = 1;
 336         break;
 337 
 338       case EVEX_32bit:
 339         mod_idx = 2;
 340         break;
 341 
 342       case EVEX_64bit:
 343         mod_idx = 3;
 344         break;
 345       }
 346       break;
 347 
 348     case EVEX_T1F:
 349     case EVEX_T2:
 350     case EVEX_T4:
 351       mod_idx = (in_size_in_bits == EVEX_64bit) ? 1 : 0;
 352       break;
 353 
 354     case EVEX_T8:
 355       break;
 356 
 357     case EVEX_HVM:
 358       break;
 359 
 360     case EVEX_QVM:
 361       break;
 362 
 363     case EVEX_OVM:
 364       break;
 365 
 366     case EVEX_M128:
 367       break;
 368 
 369     case EVEX_DUP:
 370       break;
 371 
 372     default:
 373       assert(0, "no valid evex tuple_table entry");
 374       break;
 375     }
 376 
 377     if (vector_len >= AVX_128bit && vector_len <= AVX_512bit) {
 378       int disp_factor = tuple_table[cur_tuple_type + mod_idx][vector_len];
 379       if ((disp % disp_factor) == 0) {
 380         int new_disp = disp / disp_factor;
 381         if ((-0x80 <= new_disp && new_disp < 0x80)) {
 382           disp = new_disp;
 383         }
 384       } else {
 385         return false;
 386       }
 387     }
 388   }
 389   return (-0x80 <= disp && disp < 0x80);
 390 }
 391 
 392 
 393 bool Assembler::emit_compressed_disp_byte(int &disp) {
 394   int mod_idx = 0;
 395   // We will test if the displacement fits the compressed format and if so
 396   // apply the compression to the displacment iff the result is8bit.
 397   if (VM_Version::supports_evex() && _attributes && _attributes->is_evex_instruction()) {
 398     int evex_encoding = _attributes->get_evex_encoding();
 399     int tuple_type = _attributes->get_tuple_type();
 400     switch (tuple_type) {
 401     case EVEX_FV:
 402       if ((evex_encoding & VEX_W) == VEX_W) {
 403         mod_idx = ((evex_encoding & EVEX_Rb) == EVEX_Rb) ? 3 : 2;
 404       } else {
 405         mod_idx = ((evex_encoding & EVEX_Rb) == EVEX_Rb) ? 1 : 0;
 406       }
 407       break;
 408 
 409     case EVEX_HV:
 410       mod_idx = ((evex_encoding & EVEX_Rb) == EVEX_Rb) ? 1 : 0;
 411       break;
 412 
 413     case EVEX_FVM:
 414       break;
 415 
 416     case EVEX_T1S:
 417       switch (_attributes->get_input_size()) {
 418       case EVEX_8bit:
 419         break;
 420 
 421       case EVEX_16bit:
 422         mod_idx = 1;
 423         break;
 424 
 425       case EVEX_32bit:
 426         mod_idx = 2;
 427         break;
 428 
 429       case EVEX_64bit:
 430         mod_idx = 3;
 431         break;
 432       }
 433       break;
 434 
 435     case EVEX_T1F:
 436     case EVEX_T2:
 437     case EVEX_T4:
 438       mod_idx = (_attributes->get_input_size() == EVEX_64bit) ? 1 : 0;
 439       break;
 440 
 441     case EVEX_T8:
 442       break;
 443 
 444     case EVEX_HVM:
 445       break;
 446 
 447     case EVEX_QVM:
 448       break;
 449 
 450     case EVEX_OVM:
 451       break;
 452 
 453     case EVEX_M128:
 454       break;
 455 
 456     case EVEX_DUP:
 457       break;
 458 
 459     default:
 460       assert(0, "no valid evex tuple_table entry");
 461       break;
 462     }
 463 
 464     int vector_len = _attributes->get_vector_len();
 465     if (vector_len >= AVX_128bit && vector_len <= AVX_512bit) {
 466       int disp_factor = tuple_table[tuple_type + mod_idx][vector_len];
 467       if ((disp % disp_factor) == 0) {
 468         int new_disp = disp / disp_factor;
 469         if (is8bit(new_disp)) {
 470           disp = new_disp;
 471         }
 472       } else {
 473         return false;
 474       }
 475     }
 476   }
 477   return is8bit(disp);
 478 }
 479 
 480 
 481 void Assembler::emit_operand(Register reg, Register base, Register index,
 482                              Address::ScaleFactor scale, int disp,
 483                              RelocationHolder const& rspec,
 484                              int rip_relative_correction) {
 485   relocInfo::relocType rtype = (relocInfo::relocType) rspec.type();
 486 
 487   // Encode the registers as needed in the fields they are used in
 488 
 489   int regenc = encode(reg) << 3;
 490   int indexenc = index->is_valid() ? encode(index) << 3 : 0;
 491   int baseenc = base->is_valid() ? encode(base) : 0;
 492 
 493   if (base->is_valid()) {
 494     if (index->is_valid()) {
 495       assert(scale != Address::no_scale, "inconsistent address");
 496       // [base + index*scale + disp]
 497       if (disp == 0 && rtype == relocInfo::none  &&
 498           base != rbp LP64_ONLY(&& base != r13)) {
 499         // [base + index*scale]
 500         // [00 reg 100][ss index base]
 501         assert(index != rsp, "illegal addressing mode");
 502         emit_int8(0x04 | regenc);
 503         emit_int8(scale << 6 | indexenc | baseenc);
 504       } else if (emit_compressed_disp_byte(disp) && rtype == relocInfo::none) {
 505         // [base + index*scale + imm8]
 506         // [01 reg 100][ss index base] imm8
 507         assert(index != rsp, "illegal addressing mode");
 508         emit_int8(0x44 | regenc);
 509         emit_int8(scale << 6 | indexenc | baseenc);
 510         emit_int8(disp & 0xFF);
 511       } else {
 512         // [base + index*scale + disp32]
 513         // [10 reg 100][ss index base] disp32
 514         assert(index != rsp, "illegal addressing mode");
 515         emit_int8(0x84 | regenc);
 516         emit_int8(scale << 6 | indexenc | baseenc);
 517         emit_data(disp, rspec, disp32_operand);
 518       }
 519     } else if (base == rsp LP64_ONLY(|| base == r12)) {
 520       // [rsp + disp]
 521       if (disp == 0 && rtype == relocInfo::none) {
 522         // [rsp]
 523         // [00 reg 100][00 100 100]
 524         emit_int8(0x04 | regenc);
 525         emit_int8(0x24);
 526       } else if (emit_compressed_disp_byte(disp) && rtype == relocInfo::none) {
 527         // [rsp + imm8]
 528         // [01 reg 100][00 100 100] disp8
 529         emit_int8(0x44 | regenc);
 530         emit_int8(0x24);
 531         emit_int8(disp & 0xFF);
 532       } else {
 533         // [rsp + imm32]
 534         // [10 reg 100][00 100 100] disp32
 535         emit_int8(0x84 | regenc);
 536         emit_int8(0x24);
 537         emit_data(disp, rspec, disp32_operand);
 538       }
 539     } else {
 540       // [base + disp]
 541       assert(base != rsp LP64_ONLY(&& base != r12), "illegal addressing mode");
 542       if (disp == 0 && rtype == relocInfo::none &&
 543           base != rbp LP64_ONLY(&& base != r13)) {
 544         // [base]
 545         // [00 reg base]
 546         emit_int8(0x00 | regenc | baseenc);
 547       } else if (emit_compressed_disp_byte(disp) && rtype == relocInfo::none) {
 548         // [base + disp8]
 549         // [01 reg base] disp8
 550         emit_int8(0x40 | regenc | baseenc);
 551         emit_int8(disp & 0xFF);
 552       } else {
 553         // [base + disp32]
 554         // [10 reg base] disp32
 555         emit_int8(0x80 | regenc | baseenc);
 556         emit_data(disp, rspec, disp32_operand);
 557       }
 558     }
 559   } else {
 560     if (index->is_valid()) {
 561       assert(scale != Address::no_scale, "inconsistent address");
 562       // [index*scale + disp]
 563       // [00 reg 100][ss index 101] disp32
 564       assert(index != rsp, "illegal addressing mode");
 565       emit_int8(0x04 | regenc);
 566       emit_int8(scale << 6 | indexenc | 0x05);
 567       emit_data(disp, rspec, disp32_operand);
 568     } else if (rtype != relocInfo::none ) {
 569       // [disp] (64bit) RIP-RELATIVE (32bit) abs
 570       // [00 000 101] disp32
 571 
 572       emit_int8(0x05 | regenc);
 573       // Note that the RIP-rel. correction applies to the generated
 574       // disp field, but _not_ to the target address in the rspec.
 575 
 576       // disp was created by converting the target address minus the pc
 577       // at the start of the instruction. That needs more correction here.
 578       // intptr_t disp = target - next_ip;
 579       assert(inst_mark() != NULL, "must be inside InstructionMark");
 580       address next_ip = pc() + sizeof(int32_t) + rip_relative_correction;
 581       int64_t adjusted = disp;
 582       // Do rip-rel adjustment for 64bit
 583       LP64_ONLY(adjusted -=  (next_ip - inst_mark()));
 584       assert(is_simm32(adjusted),
 585              "must be 32bit offset (RIP relative address)");
 586       emit_data((int32_t) adjusted, rspec, disp32_operand);
 587 
 588     } else {
 589       // 32bit never did this, did everything as the rip-rel/disp code above
 590       // [disp] ABSOLUTE
 591       // [00 reg 100][00 100 101] disp32
 592       emit_int8(0x04 | regenc);
 593       emit_int8(0x25);
 594       emit_data(disp, rspec, disp32_operand);
 595     }
 596   }
 597 }
 598 
 599 void Assembler::emit_operand(XMMRegister reg, Register base, Register index,
 600                              Address::ScaleFactor scale, int disp,
 601                              RelocationHolder const& rspec) {
 602   if (UseAVX > 2) {
 603     int xreg_enc = reg->encoding();
 604     if (xreg_enc > 15) {
 605       XMMRegister new_reg = as_XMMRegister(xreg_enc & 0xf);
 606       emit_operand((Register)new_reg, base, index, scale, disp, rspec);
 607       return;
 608     }
 609   }
 610   emit_operand((Register)reg, base, index, scale, disp, rspec);
 611 }
 612 
 613 // Secret local extension to Assembler::WhichOperand:
 614 #define end_pc_operand (_WhichOperand_limit)
 615 
 616 address Assembler::locate_operand(address inst, WhichOperand which) {
 617   // Decode the given instruction, and return the address of
 618   // an embedded 32-bit operand word.
 619 
 620   // If "which" is disp32_operand, selects the displacement portion
 621   // of an effective address specifier.
 622   // If "which" is imm64_operand, selects the trailing immediate constant.
 623   // If "which" is call32_operand, selects the displacement of a call or jump.
 624   // Caller is responsible for ensuring that there is such an operand,
 625   // and that it is 32/64 bits wide.
 626 
 627   // If "which" is end_pc_operand, find the end of the instruction.
 628 
 629   address ip = inst;
 630   bool is_64bit = false;
 631 
 632   debug_only(bool has_disp32 = false);
 633   int tail_size = 0; // other random bytes (#32, #16, etc.) at end of insn
 634 
 635   again_after_prefix:
 636   switch (0xFF & *ip++) {
 637 
 638   // These convenience macros generate groups of "case" labels for the switch.
 639 #define REP4(x) (x)+0: case (x)+1: case (x)+2: case (x)+3
 640 #define REP8(x) (x)+0: case (x)+1: case (x)+2: case (x)+3: \
 641              case (x)+4: case (x)+5: case (x)+6: case (x)+7
 642 #define REP16(x) REP8((x)+0): \
 643               case REP8((x)+8)
 644 
 645   case CS_segment:
 646   case SS_segment:
 647   case DS_segment:
 648   case ES_segment:
 649   case FS_segment:
 650   case GS_segment:
 651     // Seems dubious
 652     LP64_ONLY(assert(false, "shouldn't have that prefix"));
 653     assert(ip == inst+1, "only one prefix allowed");
 654     goto again_after_prefix;
 655 
 656   case 0x67:
 657   case REX:
 658   case REX_B:
 659   case REX_X:
 660   case REX_XB:
 661   case REX_R:
 662   case REX_RB:
 663   case REX_RX:
 664   case REX_RXB:
 665     NOT_LP64(assert(false, "64bit prefixes"));
 666     goto again_after_prefix;
 667 
 668   case REX_W:
 669   case REX_WB:
 670   case REX_WX:
 671   case REX_WXB:
 672   case REX_WR:
 673   case REX_WRB:
 674   case REX_WRX:
 675   case REX_WRXB:
 676     NOT_LP64(assert(false, "64bit prefixes"));
 677     is_64bit = true;
 678     goto again_after_prefix;
 679 
 680   case 0xFF: // pushq a; decl a; incl a; call a; jmp a
 681   case 0x88: // movb a, r
 682   case 0x89: // movl a, r
 683   case 0x8A: // movb r, a
 684   case 0x8B: // movl r, a
 685   case 0x8F: // popl a
 686     debug_only(has_disp32 = true);
 687     break;
 688 
 689   case 0x68: // pushq #32
 690     if (which == end_pc_operand) {
 691       return ip + 4;
 692     }
 693     assert(which == imm_operand && !is_64bit, "pushl has no disp32 or 64bit immediate");
 694     return ip;                  // not produced by emit_operand
 695 
 696   case 0x66: // movw ... (size prefix)
 697     again_after_size_prefix2:
 698     switch (0xFF & *ip++) {
 699     case REX:
 700     case REX_B:
 701     case REX_X:
 702     case REX_XB:
 703     case REX_R:
 704     case REX_RB:
 705     case REX_RX:
 706     case REX_RXB:
 707     case REX_W:
 708     case REX_WB:
 709     case REX_WX:
 710     case REX_WXB:
 711     case REX_WR:
 712     case REX_WRB:
 713     case REX_WRX:
 714     case REX_WRXB:
 715       NOT_LP64(assert(false, "64bit prefix found"));
 716       goto again_after_size_prefix2;
 717     case 0x8B: // movw r, a
 718     case 0x89: // movw a, r
 719       debug_only(has_disp32 = true);
 720       break;
 721     case 0xC7: // movw a, #16
 722       debug_only(has_disp32 = true);
 723       tail_size = 2;  // the imm16
 724       break;
 725     case 0x0F: // several SSE/SSE2 variants
 726       ip--;    // reparse the 0x0F
 727       goto again_after_prefix;
 728     default:
 729       ShouldNotReachHere();
 730     }
 731     break;
 732 
 733   case REP8(0xB8): // movl/q r, #32/#64(oop?)
 734     if (which == end_pc_operand)  return ip + (is_64bit ? 8 : 4);
 735     // these asserts are somewhat nonsensical
 736 #ifndef _LP64
 737     assert(which == imm_operand || which == disp32_operand,
 738            "which %d is_64_bit %d ip " INTPTR_FORMAT, which, is_64bit, p2i(ip));
 739 #else
 740     assert((which == call32_operand || which == imm_operand) && is_64bit ||
 741            which == narrow_oop_operand && !is_64bit,
 742            "which %d is_64_bit %d ip " INTPTR_FORMAT, which, is_64bit, p2i(ip));
 743 #endif // _LP64
 744     return ip;
 745 
 746   case 0x69: // imul r, a, #32
 747   case 0xC7: // movl a, #32(oop?)
 748     tail_size = 4;
 749     debug_only(has_disp32 = true); // has both kinds of operands!
 750     break;
 751 
 752   case 0x0F: // movx..., etc.
 753     switch (0xFF & *ip++) {
 754     case 0x3A: // pcmpestri
 755       tail_size = 1;
 756     case 0x38: // ptest, pmovzxbw
 757       ip++; // skip opcode
 758       debug_only(has_disp32 = true); // has both kinds of operands!
 759       break;
 760 
 761     case 0x70: // pshufd r, r/a, #8
 762       debug_only(has_disp32 = true); // has both kinds of operands!
 763     case 0x73: // psrldq r, #8
 764       tail_size = 1;
 765       break;
 766 
 767     case 0x12: // movlps
 768     case 0x28: // movaps
 769     case 0x2E: // ucomiss
 770     case 0x2F: // comiss
 771     case 0x54: // andps
 772     case 0x55: // andnps
 773     case 0x56: // orps
 774     case 0x57: // xorps
 775     case 0x58: // addpd
 776     case 0x59: // mulpd
 777     case 0x6E: // movd
 778     case 0x7E: // movd
 779     case 0xAE: // ldmxcsr, stmxcsr, fxrstor, fxsave, clflush
 780     case 0xFE: // paddd
 781       debug_only(has_disp32 = true);
 782       break;
 783 
 784     case 0xAD: // shrd r, a, %cl
 785     case 0xAF: // imul r, a
 786     case 0xBE: // movsbl r, a (movsxb)
 787     case 0xBF: // movswl r, a (movsxw)
 788     case 0xB6: // movzbl r, a (movzxb)
 789     case 0xB7: // movzwl r, a (movzxw)
 790     case REP16(0x40): // cmovl cc, r, a
 791     case 0xB0: // cmpxchgb
 792     case 0xB1: // cmpxchg
 793     case 0xC1: // xaddl
 794     case 0xC7: // cmpxchg8
 795     case REP16(0x90): // setcc a
 796       debug_only(has_disp32 = true);
 797       // fall out of the switch to decode the address
 798       break;
 799 
 800     case 0xC4: // pinsrw r, a, #8
 801       debug_only(has_disp32 = true);
 802     case 0xC5: // pextrw r, r, #8
 803       tail_size = 1;  // the imm8
 804       break;
 805 
 806     case 0xAC: // shrd r, a, #8
 807       debug_only(has_disp32 = true);
 808       tail_size = 1;  // the imm8
 809       break;
 810 
 811     case REP16(0x80): // jcc rdisp32
 812       if (which == end_pc_operand)  return ip + 4;
 813       assert(which == call32_operand, "jcc has no disp32 or imm");
 814       return ip;
 815     default:
 816       ShouldNotReachHere();
 817     }
 818     break;
 819 
 820   case 0x81: // addl a, #32; addl r, #32
 821     // also: orl, adcl, sbbl, andl, subl, xorl, cmpl
 822     // on 32bit in the case of cmpl, the imm might be an oop
 823     tail_size = 4;
 824     debug_only(has_disp32 = true); // has both kinds of operands!
 825     break;
 826 
 827   case 0x83: // addl a, #8; addl r, #8
 828     // also: orl, adcl, sbbl, andl, subl, xorl, cmpl
 829     debug_only(has_disp32 = true); // has both kinds of operands!
 830     tail_size = 1;
 831     break;
 832 
 833   case 0x9B:
 834     switch (0xFF & *ip++) {
 835     case 0xD9: // fnstcw a
 836       debug_only(has_disp32 = true);
 837       break;
 838     default:
 839       ShouldNotReachHere();
 840     }
 841     break;
 842 
 843   case REP4(0x00): // addb a, r; addl a, r; addb r, a; addl r, a
 844   case REP4(0x10): // adc...
 845   case REP4(0x20): // and...
 846   case REP4(0x30): // xor...
 847   case REP4(0x08): // or...
 848   case REP4(0x18): // sbb...
 849   case REP4(0x28): // sub...
 850   case 0xF7: // mull a
 851   case 0x8D: // lea r, a
 852   case 0x87: // xchg r, a
 853   case REP4(0x38): // cmp...
 854   case 0x85: // test r, a
 855     debug_only(has_disp32 = true); // has both kinds of operands!
 856     break;
 857 
 858   case 0xC1: // sal a, #8; sar a, #8; shl a, #8; shr a, #8
 859   case 0xC6: // movb a, #8
 860   case 0x80: // cmpb a, #8
 861   case 0x6B: // imul r, a, #8
 862     debug_only(has_disp32 = true); // has both kinds of operands!
 863     tail_size = 1; // the imm8
 864     break;
 865 
 866   case 0xC4: // VEX_3bytes
 867   case 0xC5: // VEX_2bytes
 868     assert((UseAVX > 0), "shouldn't have VEX prefix");
 869     assert(ip == inst+1, "no prefixes allowed");
 870     // C4 and C5 are also used as opcodes for PINSRW and PEXTRW instructions
 871     // but they have prefix 0x0F and processed when 0x0F processed above.
 872     //
 873     // In 32-bit mode the VEX first byte C4 and C5 alias onto LDS and LES
 874     // instructions (these instructions are not supported in 64-bit mode).
 875     // To distinguish them bits [7:6] are set in the VEX second byte since
 876     // ModRM byte can not be of the form 11xxxxxx in 32-bit mode. To set
 877     // those VEX bits REX and vvvv bits are inverted.
 878     //
 879     // Fortunately C2 doesn't generate these instructions so we don't need
 880     // to check for them in product version.
 881 
 882     // Check second byte
 883     NOT_LP64(assert((0xC0 & *ip) == 0xC0, "shouldn't have LDS and LES instructions"));
 884 
 885     int vex_opcode;
 886     // First byte
 887     if ((0xFF & *inst) == VEX_3bytes) {
 888       vex_opcode = VEX_OPCODE_MASK & *ip;
 889       ip++; // third byte
 890       is_64bit = ((VEX_W & *ip) == VEX_W);
 891     } else {
 892       vex_opcode = VEX_OPCODE_0F;
 893     }
 894     ip++; // opcode
 895     // To find the end of instruction (which == end_pc_operand).
 896     switch (vex_opcode) {
 897       case VEX_OPCODE_0F:
 898         switch (0xFF & *ip) {
 899         case 0x70: // pshufd r, r/a, #8
 900         case 0x71: // ps[rl|ra|ll]w r, #8
 901         case 0x72: // ps[rl|ra|ll]d r, #8
 902         case 0x73: // ps[rl|ra|ll]q r, #8
 903         case 0xC2: // cmp[ps|pd|ss|sd] r, r, r/a, #8
 904         case 0xC4: // pinsrw r, r, r/a, #8
 905         case 0xC5: // pextrw r/a, r, #8
 906         case 0xC6: // shufp[s|d] r, r, r/a, #8
 907           tail_size = 1;  // the imm8
 908           break;
 909         }
 910         break;
 911       case VEX_OPCODE_0F_3A:
 912         tail_size = 1;
 913         break;
 914     }
 915     ip++; // skip opcode
 916     debug_only(has_disp32 = true); // has both kinds of operands!
 917     break;
 918 
 919   case 0x62: // EVEX_4bytes
 920     assert((UseAVX > 0), "shouldn't have EVEX prefix");
 921     assert(ip == inst+1, "no prefixes allowed");
 922     // no EVEX collisions, all instructions that have 0x62 opcodes
 923     // have EVEX versions and are subopcodes of 0x66
 924     ip++; // skip P0 and exmaine W in P1
 925     is_64bit = ((VEX_W & *ip) == VEX_W);
 926     ip++; // move to P2
 927     ip++; // skip P2, move to opcode
 928     // To find the end of instruction (which == end_pc_operand).
 929     switch (0xFF & *ip) {
 930     case 0x22: // pinsrd r, r/a, #8
 931     case 0x61: // pcmpestri r, r/a, #8
 932     case 0x70: // pshufd r, r/a, #8
 933     case 0x73: // psrldq r, #8
 934       tail_size = 1;  // the imm8
 935       break;
 936     default:
 937       break;
 938     }
 939     ip++; // skip opcode
 940     debug_only(has_disp32 = true); // has both kinds of operands!
 941     break;
 942 
 943   case 0xD1: // sal a, 1; sar a, 1; shl a, 1; shr a, 1
 944   case 0xD3: // sal a, %cl; sar a, %cl; shl a, %cl; shr a, %cl
 945   case 0xD9: // fld_s a; fst_s a; fstp_s a; fldcw a
 946   case 0xDD: // fld_d a; fst_d a; fstp_d a
 947   case 0xDB: // fild_s a; fistp_s a; fld_x a; fstp_x a
 948   case 0xDF: // fild_d a; fistp_d a
 949   case 0xD8: // fadd_s a; fsubr_s a; fmul_s a; fdivr_s a; fcomp_s a
 950   case 0xDC: // fadd_d a; fsubr_d a; fmul_d a; fdivr_d a; fcomp_d a
 951   case 0xDE: // faddp_d a; fsubrp_d a; fmulp_d a; fdivrp_d a; fcompp_d a
 952     debug_only(has_disp32 = true);
 953     break;
 954 
 955   case 0xE8: // call rdisp32
 956   case 0xE9: // jmp  rdisp32
 957     if (which == end_pc_operand)  return ip + 4;
 958     assert(which == call32_operand, "call has no disp32 or imm");
 959     return ip;
 960 
 961   case 0xF0:                    // Lock
 962     assert(os::is_MP(), "only on MP");
 963     goto again_after_prefix;
 964 
 965   case 0xF3:                    // For SSE
 966   case 0xF2:                    // For SSE2
 967     switch (0xFF & *ip++) {
 968     case REX:
 969     case REX_B:
 970     case REX_X:
 971     case REX_XB:
 972     case REX_R:
 973     case REX_RB:
 974     case REX_RX:
 975     case REX_RXB:
 976     case REX_W:
 977     case REX_WB:
 978     case REX_WX:
 979     case REX_WXB:
 980     case REX_WR:
 981     case REX_WRB:
 982     case REX_WRX:
 983     case REX_WRXB:
 984       NOT_LP64(assert(false, "found 64bit prefix"));
 985       ip++;
 986     default:
 987       ip++;
 988     }
 989     debug_only(has_disp32 = true); // has both kinds of operands!
 990     break;
 991 
 992   default:
 993     ShouldNotReachHere();
 994 
 995 #undef REP8
 996 #undef REP16
 997   }
 998 
 999   assert(which != call32_operand, "instruction is not a call, jmp, or jcc");
1000 #ifdef _LP64
1001   assert(which != imm_operand, "instruction is not a movq reg, imm64");
1002 #else
1003   // assert(which != imm_operand || has_imm32, "instruction has no imm32 field");
1004   assert(which != imm_operand || has_disp32, "instruction has no imm32 field");
1005 #endif // LP64
1006   assert(which != disp32_operand || has_disp32, "instruction has no disp32 field");
1007 
1008   // parse the output of emit_operand
1009   int op2 = 0xFF & *ip++;
1010   int base = op2 & 0x07;
1011   int op3 = -1;
1012   const int b100 = 4;
1013   const int b101 = 5;
1014   if (base == b100 && (op2 >> 6) != 3) {
1015     op3 = 0xFF & *ip++;
1016     base = op3 & 0x07;   // refetch the base
1017   }
1018   // now ip points at the disp (if any)
1019 
1020   switch (op2 >> 6) {
1021   case 0:
1022     // [00 reg  100][ss index base]
1023     // [00 reg  100][00   100  esp]
1024     // [00 reg base]
1025     // [00 reg  100][ss index  101][disp32]
1026     // [00 reg  101]               [disp32]
1027 
1028     if (base == b101) {
1029       if (which == disp32_operand)
1030         return ip;              // caller wants the disp32
1031       ip += 4;                  // skip the disp32
1032     }
1033     break;
1034 
1035   case 1:
1036     // [01 reg  100][ss index base][disp8]
1037     // [01 reg  100][00   100  esp][disp8]
1038     // [01 reg base]               [disp8]
1039     ip += 1;                    // skip the disp8
1040     break;
1041 
1042   case 2:
1043     // [10 reg  100][ss index base][disp32]
1044     // [10 reg  100][00   100  esp][disp32]
1045     // [10 reg base]               [disp32]
1046     if (which == disp32_operand)
1047       return ip;                // caller wants the disp32
1048     ip += 4;                    // skip the disp32
1049     break;
1050 
1051   case 3:
1052     // [11 reg base]  (not a memory addressing mode)
1053     break;
1054   }
1055 
1056   if (which == end_pc_operand) {
1057     return ip + tail_size;
1058   }
1059 
1060 #ifdef _LP64
1061   assert(which == narrow_oop_operand && !is_64bit, "instruction is not a movl adr, imm32");
1062 #else
1063   assert(which == imm_operand, "instruction has only an imm field");
1064 #endif // LP64
1065   return ip;
1066 }
1067 
1068 address Assembler::locate_next_instruction(address inst) {
1069   // Secretly share code with locate_operand:
1070   return locate_operand(inst, end_pc_operand);
1071 }
1072 
1073 
1074 #ifdef ASSERT
1075 void Assembler::check_relocation(RelocationHolder const& rspec, int format) {
1076   address inst = inst_mark();
1077   assert(inst != NULL && inst < pc(), "must point to beginning of instruction");
1078   address opnd;
1079 
1080   Relocation* r = rspec.reloc();
1081   if (r->type() == relocInfo::none) {
1082     return;
1083   } else if (r->is_call() || format == call32_operand) {
1084     // assert(format == imm32_operand, "cannot specify a nonzero format");
1085     opnd = locate_operand(inst, call32_operand);
1086   } else if (r->is_data()) {
1087     assert(format == imm_operand || format == disp32_operand
1088            LP64_ONLY(|| format == narrow_oop_operand), "format ok");
1089     opnd = locate_operand(inst, (WhichOperand)format);
1090   } else {
1091     assert(format == imm_operand, "cannot specify a format");
1092     return;
1093   }
1094   assert(opnd == pc(), "must put operand where relocs can find it");
1095 }
1096 #endif // ASSERT
1097 
1098 void Assembler::emit_operand32(Register reg, Address adr) {
1099   assert(reg->encoding() < 8, "no extended registers");
1100   assert(!adr.base_needs_rex() && !adr.index_needs_rex(), "no extended registers");
1101   emit_operand(reg, adr._base, adr._index, adr._scale, adr._disp,
1102                adr._rspec);
1103 }
1104 
1105 void Assembler::emit_operand(Register reg, Address adr,
1106                              int rip_relative_correction) {
1107   emit_operand(reg, adr._base, adr._index, adr._scale, adr._disp,
1108                adr._rspec,
1109                rip_relative_correction);
1110 }
1111 
1112 void Assembler::emit_operand(XMMRegister reg, Address adr) {
1113   emit_operand(reg, adr._base, adr._index, adr._scale, adr._disp,
1114                adr._rspec);
1115 }
1116 
1117 // MMX operations
1118 void Assembler::emit_operand(MMXRegister reg, Address adr) {
1119   assert(!adr.base_needs_rex() && !adr.index_needs_rex(), "no extended registers");
1120   emit_operand((Register)reg, adr._base, adr._index, adr._scale, adr._disp, adr._rspec);
1121 }
1122 
1123 // work around gcc (3.2.1-7a) bug
1124 void Assembler::emit_operand(Address adr, MMXRegister reg) {
1125   assert(!adr.base_needs_rex() && !adr.index_needs_rex(), "no extended registers");
1126   emit_operand((Register)reg, adr._base, adr._index, adr._scale, adr._disp, adr._rspec);
1127 }
1128 
1129 
1130 void Assembler::emit_farith(int b1, int b2, int i) {
1131   assert(isByte(b1) && isByte(b2), "wrong opcode");
1132   assert(0 <= i &&  i < 8, "illegal stack offset");
1133   emit_int8(b1);
1134   emit_int8(b2 + i);
1135 }
1136 
1137 
1138 // Now the Assembler instructions (identical for 32/64 bits)
1139 
1140 void Assembler::adcl(Address dst, int32_t imm32) {
1141   InstructionMark im(this);
1142   prefix(dst);
1143   emit_arith_operand(0x81, rdx, dst, imm32);
1144 }
1145 
1146 void Assembler::adcl(Address dst, Register src) {
1147   InstructionMark im(this);
1148   prefix(dst, src);
1149   emit_int8(0x11);
1150   emit_operand(src, dst);
1151 }
1152 
1153 void Assembler::adcl(Register dst, int32_t imm32) {
1154   prefix(dst);
1155   emit_arith(0x81, 0xD0, dst, imm32);
1156 }
1157 
1158 void Assembler::adcl(Register dst, Address src) {
1159   InstructionMark im(this);
1160   prefix(src, dst);
1161   emit_int8(0x13);
1162   emit_operand(dst, src);
1163 }
1164 
1165 void Assembler::adcl(Register dst, Register src) {
1166   (void) prefix_and_encode(dst->encoding(), src->encoding());
1167   emit_arith(0x13, 0xC0, dst, src);
1168 }
1169 
1170 void Assembler::addl(Address dst, int32_t imm32) {
1171   InstructionMark im(this);
1172   prefix(dst);
1173   emit_arith_operand(0x81, rax, dst, imm32);
1174 }
1175 
1176 void Assembler::addb(Address dst, int imm8) {
1177   InstructionMark im(this);
1178   prefix(dst);
1179   emit_int8((unsigned char)0x80);
1180   emit_operand(rax, dst, 1);
1181   emit_int8(imm8);
1182 }
1183 
1184 void Assembler::addw(Address dst, int imm16) {
1185   InstructionMark im(this);
1186   emit_int8(0x66);
1187   prefix(dst);
1188   emit_int8((unsigned char)0x81);
1189   emit_operand(rax, dst, 2);
1190   emit_int16(imm16);
1191 }
1192 
1193 void Assembler::addl(Address dst, Register src) {
1194   InstructionMark im(this);
1195   prefix(dst, src);
1196   emit_int8(0x01);
1197   emit_operand(src, dst);
1198 }
1199 
1200 void Assembler::addl(Register dst, int32_t imm32) {
1201   prefix(dst);
1202   emit_arith(0x81, 0xC0, dst, imm32);
1203 }
1204 
1205 void Assembler::addl(Register dst, Address src) {
1206   InstructionMark im(this);
1207   prefix(src, dst);
1208   emit_int8(0x03);
1209   emit_operand(dst, src);
1210 }
1211 
1212 void Assembler::addl(Register dst, Register src) {
1213   (void) prefix_and_encode(dst->encoding(), src->encoding());
1214   emit_arith(0x03, 0xC0, dst, src);
1215 }
1216 
1217 void Assembler::addr_nop_4() {
1218   assert(UseAddressNop, "no CPU support");
1219   // 4 bytes: NOP DWORD PTR [EAX+0]
1220   emit_int8(0x0F);
1221   emit_int8(0x1F);
1222   emit_int8(0x40); // emit_rm(cbuf, 0x1, EAX_enc, EAX_enc);
1223   emit_int8(0);    // 8-bits offset (1 byte)
1224 }
1225 
1226 void Assembler::addr_nop_5() {
1227   assert(UseAddressNop, "no CPU support");
1228   // 5 bytes: NOP DWORD PTR [EAX+EAX*0+0] 8-bits offset
1229   emit_int8(0x0F);
1230   emit_int8(0x1F);
1231   emit_int8(0x44); // emit_rm(cbuf, 0x1, EAX_enc, 0x4);
1232   emit_int8(0x00); // emit_rm(cbuf, 0x0, EAX_enc, EAX_enc);
1233   emit_int8(0);    // 8-bits offset (1 byte)
1234 }
1235 
1236 void Assembler::addr_nop_7() {
1237   assert(UseAddressNop, "no CPU support");
1238   // 7 bytes: NOP DWORD PTR [EAX+0] 32-bits offset
1239   emit_int8(0x0F);
1240   emit_int8(0x1F);
1241   emit_int8((unsigned char)0x80);
1242                    // emit_rm(cbuf, 0x2, EAX_enc, EAX_enc);
1243   emit_int32(0);   // 32-bits offset (4 bytes)
1244 }
1245 
1246 void Assembler::addr_nop_8() {
1247   assert(UseAddressNop, "no CPU support");
1248   // 8 bytes: NOP DWORD PTR [EAX+EAX*0+0] 32-bits offset
1249   emit_int8(0x0F);
1250   emit_int8(0x1F);
1251   emit_int8((unsigned char)0x84);
1252                    // emit_rm(cbuf, 0x2, EAX_enc, 0x4);
1253   emit_int8(0x00); // emit_rm(cbuf, 0x0, EAX_enc, EAX_enc);
1254   emit_int32(0);   // 32-bits offset (4 bytes)
1255 }
1256 
1257 void Assembler::addsd(XMMRegister dst, XMMRegister src) {
1258   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1259   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
1260   attributes.set_rex_vex_w_reverted();
1261   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
1262   emit_int8(0x58);
1263   emit_int8((unsigned char)(0xC0 | encode));
1264 }
1265 
1266 void Assembler::addsd(XMMRegister dst, Address src) {
1267   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1268   InstructionMark im(this);
1269   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
1270   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
1271   attributes.set_rex_vex_w_reverted();
1272   simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
1273   emit_int8(0x58);
1274   emit_operand(dst, src);
1275 }
1276 
1277 void Assembler::addss(XMMRegister dst, XMMRegister src) {
1278   NOT_LP64(assert(VM_Version::supports_sse(), ""));
1279   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
1280   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
1281   emit_int8(0x58);
1282   emit_int8((unsigned char)(0xC0 | encode));
1283 }
1284 
1285 void Assembler::addss(XMMRegister dst, Address src) {
1286   NOT_LP64(assert(VM_Version::supports_sse(), ""));
1287   InstructionMark im(this);
1288   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
1289   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
1290   simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
1291   emit_int8(0x58);
1292   emit_operand(dst, src);
1293 }
1294 
1295 void Assembler::aesdec(XMMRegister dst, Address src) {
1296   assert(VM_Version::supports_aes(), "");
1297   InstructionMark im(this);
1298   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
1299   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
1300   emit_int8((unsigned char)0xDE);
1301   emit_operand(dst, src);
1302 }
1303 
1304 void Assembler::aesdec(XMMRegister dst, XMMRegister src) {
1305   assert(VM_Version::supports_aes(), "");
1306   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
1307   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
1308   emit_int8((unsigned char)0xDE);
1309   emit_int8(0xC0 | encode);
1310 }
1311 
1312 void Assembler::aesdeclast(XMMRegister dst, Address src) {
1313   assert(VM_Version::supports_aes(), "");
1314   InstructionMark im(this);
1315   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
1316   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
1317   emit_int8((unsigned char)0xDF);
1318   emit_operand(dst, src);
1319 }
1320 
1321 void Assembler::aesdeclast(XMMRegister dst, XMMRegister src) {
1322   assert(VM_Version::supports_aes(), "");
1323   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
1324   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
1325   emit_int8((unsigned char)0xDF);
1326   emit_int8((unsigned char)(0xC0 | encode));
1327 }
1328 
1329 void Assembler::aesenc(XMMRegister dst, Address src) {
1330   assert(VM_Version::supports_aes(), "");
1331   InstructionMark im(this);
1332   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
1333   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
1334   emit_int8((unsigned char)0xDC);
1335   emit_operand(dst, src);
1336 }
1337 
1338 void Assembler::aesenc(XMMRegister dst, XMMRegister src) {
1339   assert(VM_Version::supports_aes(), "");
1340   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
1341   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
1342   emit_int8((unsigned char)0xDC);
1343   emit_int8(0xC0 | encode);
1344 }
1345 
1346 void Assembler::aesenclast(XMMRegister dst, Address src) {
1347   assert(VM_Version::supports_aes(), "");
1348   InstructionMark im(this);
1349   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
1350   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
1351   emit_int8((unsigned char)0xDD);
1352   emit_operand(dst, src);
1353 }
1354 
1355 void Assembler::aesenclast(XMMRegister dst, XMMRegister src) {
1356   assert(VM_Version::supports_aes(), "");
1357   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
1358   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
1359   emit_int8((unsigned char)0xDD);
1360   emit_int8((unsigned char)(0xC0 | encode));
1361 }
1362 
1363 void Assembler::andl(Address dst, int32_t imm32) {
1364   InstructionMark im(this);
1365   prefix(dst);
1366   emit_int8((unsigned char)0x81);
1367   emit_operand(rsp, dst, 4);
1368   emit_int32(imm32);
1369 }
1370 
1371 void Assembler::andl(Register dst, int32_t imm32) {
1372   prefix(dst);
1373   emit_arith(0x81, 0xE0, dst, imm32);
1374 }
1375 
1376 void Assembler::andl(Register dst, Address src) {
1377   InstructionMark im(this);
1378   prefix(src, dst);
1379   emit_int8(0x23);
1380   emit_operand(dst, src);
1381 }
1382 
1383 void Assembler::andl(Register dst, Register src) {
1384   (void) prefix_and_encode(dst->encoding(), src->encoding());
1385   emit_arith(0x23, 0xC0, dst, src);
1386 }
1387 
1388 void Assembler::andnl(Register dst, Register src1, Register src2) {
1389   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1390   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
1391   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
1392   emit_int8((unsigned char)0xF2);
1393   emit_int8((unsigned char)(0xC0 | encode));
1394 }
1395 
1396 void Assembler::andnl(Register dst, Register src1, Address src2) {
1397   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1398   InstructionMark im(this);
1399   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
1400   vex_prefix(src2, src1->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
1401   emit_int8((unsigned char)0xF2);
1402   emit_operand(dst, src2);
1403 }
1404 
1405 void Assembler::bsfl(Register dst, Register src) {
1406   int encode = prefix_and_encode(dst->encoding(), src->encoding());
1407   emit_int8(0x0F);
1408   emit_int8((unsigned char)0xBC);
1409   emit_int8((unsigned char)(0xC0 | encode));
1410 }
1411 
1412 void Assembler::bsrl(Register dst, Register src) {
1413   int encode = prefix_and_encode(dst->encoding(), src->encoding());
1414   emit_int8(0x0F);
1415   emit_int8((unsigned char)0xBD);
1416   emit_int8((unsigned char)(0xC0 | encode));
1417 }
1418 
1419 void Assembler::bswapl(Register reg) { // bswap
1420   int encode = prefix_and_encode(reg->encoding());
1421   emit_int8(0x0F);
1422   emit_int8((unsigned char)(0xC8 | encode));
1423 }
1424 
1425 void Assembler::blsil(Register dst, Register src) {
1426   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1427   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
1428   int encode = vex_prefix_and_encode(rbx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
1429   emit_int8((unsigned char)0xF3);
1430   emit_int8((unsigned char)(0xC0 | encode));
1431 }
1432 
1433 void Assembler::blsil(Register dst, Address src) {
1434   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1435   InstructionMark im(this);
1436   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
1437   vex_prefix(src, dst->encoding(), rbx->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
1438   emit_int8((unsigned char)0xF3);
1439   emit_operand(rbx, src);
1440 }
1441 
1442 void Assembler::blsmskl(Register dst, Register src) {
1443   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1444   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
1445   int encode = vex_prefix_and_encode(rdx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
1446   emit_int8((unsigned char)0xF3);
1447   emit_int8((unsigned char)(0xC0 | encode));
1448 }
1449 
1450 void Assembler::blsmskl(Register dst, Address src) {
1451   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1452   InstructionMark im(this);
1453   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
1454   vex_prefix(src, dst->encoding(), rdx->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
1455   emit_int8((unsigned char)0xF3);
1456   emit_operand(rdx, src);
1457 }
1458 
1459 void Assembler::blsrl(Register dst, Register src) {
1460   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1461   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
1462   int encode = vex_prefix_and_encode(rcx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
1463   emit_int8((unsigned char)0xF3);
1464   emit_int8((unsigned char)(0xC0 | encode));
1465 }
1466 
1467 void Assembler::blsrl(Register dst, Address src) {
1468   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1469   InstructionMark im(this);
1470   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
1471   vex_prefix(src, dst->encoding(), rcx->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
1472   emit_int8((unsigned char)0xF3);
1473   emit_operand(rcx, src);
1474 }
1475 
1476 void Assembler::call(Label& L, relocInfo::relocType rtype) {
1477   // suspect disp32 is always good
1478   int operand = LP64_ONLY(disp32_operand) NOT_LP64(imm_operand);
1479 
1480   if (L.is_bound()) {
1481     const int long_size = 5;
1482     int offs = (int)( target(L) - pc() );
1483     assert(offs <= 0, "assembler error");
1484     InstructionMark im(this);
1485     // 1110 1000 #32-bit disp
1486     emit_int8((unsigned char)0xE8);
1487     emit_data(offs - long_size, rtype, operand);
1488   } else {
1489     InstructionMark im(this);
1490     // 1110 1000 #32-bit disp
1491     L.add_patch_at(code(), locator());
1492 
1493     emit_int8((unsigned char)0xE8);
1494     emit_data(int(0), rtype, operand);
1495   }
1496 }
1497 
1498 void Assembler::call(Register dst) {
1499   int encode = prefix_and_encode(dst->encoding());
1500   emit_int8((unsigned char)0xFF);
1501   emit_int8((unsigned char)(0xD0 | encode));
1502 }
1503 
1504 
1505 void Assembler::call(Address adr) {
1506   InstructionMark im(this);
1507   prefix(adr);
1508   emit_int8((unsigned char)0xFF);
1509   emit_operand(rdx, adr);
1510 }
1511 
1512 void Assembler::call_literal(address entry, RelocationHolder const& rspec) {
1513   assert(entry != NULL, "call most probably wrong");
1514   InstructionMark im(this);
1515   emit_int8((unsigned char)0xE8);
1516   intptr_t disp = entry - (pc() + sizeof(int32_t));
1517   assert(is_simm32(disp), "must be 32bit offset (call2)");
1518   // Technically, should use call32_operand, but this format is
1519   // implied by the fact that we're emitting a call instruction.
1520 
1521   int operand = LP64_ONLY(disp32_operand) NOT_LP64(call32_operand);
1522   emit_data((int) disp, rspec, operand);
1523 }
1524 
1525 void Assembler::cdql() {
1526   emit_int8((unsigned char)0x99);
1527 }
1528 
1529 void Assembler::cld() {
1530   emit_int8((unsigned char)0xFC);
1531 }
1532 
1533 void Assembler::cmovl(Condition cc, Register dst, Register src) {
1534   NOT_LP64(guarantee(VM_Version::supports_cmov(), "illegal instruction"));
1535   int encode = prefix_and_encode(dst->encoding(), src->encoding());
1536   emit_int8(0x0F);
1537   emit_int8(0x40 | cc);
1538   emit_int8((unsigned char)(0xC0 | encode));
1539 }
1540 
1541 
1542 void Assembler::cmovl(Condition cc, Register dst, Address src) {
1543   NOT_LP64(guarantee(VM_Version::supports_cmov(), "illegal instruction"));
1544   prefix(src, dst);
1545   emit_int8(0x0F);
1546   emit_int8(0x40 | cc);
1547   emit_operand(dst, src);
1548 }
1549 
1550 void Assembler::cmpb(Address dst, int imm8) {
1551   InstructionMark im(this);
1552   prefix(dst);
1553   emit_int8((unsigned char)0x80);
1554   emit_operand(rdi, dst, 1);
1555   emit_int8(imm8);
1556 }
1557 
1558 void Assembler::cmpl(Address dst, int32_t imm32) {
1559   InstructionMark im(this);
1560   prefix(dst);
1561   emit_int8((unsigned char)0x81);
1562   emit_operand(rdi, dst, 4);
1563   emit_int32(imm32);
1564 }
1565 
1566 void Assembler::cmpl(Register dst, int32_t imm32) {
1567   prefix(dst);
1568   emit_arith(0x81, 0xF8, dst, imm32);
1569 }
1570 
1571 void Assembler::cmpl(Register dst, Register src) {
1572   (void) prefix_and_encode(dst->encoding(), src->encoding());
1573   emit_arith(0x3B, 0xC0, dst, src);
1574 }
1575 
1576 void Assembler::cmpl(Register dst, Address  src) {
1577   InstructionMark im(this);
1578   prefix(src, dst);
1579   emit_int8((unsigned char)0x3B);
1580   emit_operand(dst, src);
1581 }
1582 
1583 void Assembler::cmpw(Address dst, int imm16) {
1584   InstructionMark im(this);
1585   assert(!dst.base_needs_rex() && !dst.index_needs_rex(), "no extended registers");
1586   emit_int8(0x66);
1587   emit_int8((unsigned char)0x81);
1588   emit_operand(rdi, dst, 2);
1589   emit_int16(imm16);
1590 }
1591 
1592 // The 32-bit cmpxchg compares the value at adr with the contents of rax,
1593 // and stores reg into adr if so; otherwise, the value at adr is loaded into rax,.
1594 // The ZF is set if the compared values were equal, and cleared otherwise.
1595 void Assembler::cmpxchgl(Register reg, Address adr) { // cmpxchg
1596   InstructionMark im(this);
1597   prefix(adr, reg);
1598   emit_int8(0x0F);
1599   emit_int8((unsigned char)0xB1);
1600   emit_operand(reg, adr);
1601 }
1602 
1603 // The 8-bit cmpxchg compares the value at adr with the contents of rax,
1604 // and stores reg into adr if so; otherwise, the value at adr is loaded into rax,.
1605 // The ZF is set if the compared values were equal, and cleared otherwise.
1606 void Assembler::cmpxchgb(Register reg, Address adr) { // cmpxchg
1607   InstructionMark im(this);
1608   prefix(adr, reg, true);
1609   emit_int8(0x0F);
1610   emit_int8((unsigned char)0xB0);
1611   emit_operand(reg, adr);
1612 }
1613 
1614 void Assembler::comisd(XMMRegister dst, Address src) {
1615   // NOTE: dbx seems to decode this as comiss even though the
1616   // 0x66 is there. Strangly ucomisd comes out correct
1617   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1618   InstructionMark im(this);
1619   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);;
1620   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
1621   attributes.set_rex_vex_w_reverted();
1622   simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
1623   emit_int8(0x2F);
1624   emit_operand(dst, src);
1625 }
1626 
1627 void Assembler::comisd(XMMRegister dst, XMMRegister src) {
1628   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1629   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1630   attributes.set_rex_vex_w_reverted();
1631   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
1632   emit_int8(0x2F);
1633   emit_int8((unsigned char)(0xC0 | encode));
1634 }
1635 
1636 void Assembler::comiss(XMMRegister dst, Address src) {
1637   NOT_LP64(assert(VM_Version::supports_sse(), ""));
1638   InstructionMark im(this);
1639   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1640   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
1641   simd_prefix(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
1642   emit_int8(0x2F);
1643   emit_operand(dst, src);
1644 }
1645 
1646 void Assembler::comiss(XMMRegister dst, XMMRegister src) {
1647   NOT_LP64(assert(VM_Version::supports_sse(), ""));
1648   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1649   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
1650   emit_int8(0x2F);
1651   emit_int8((unsigned char)(0xC0 | encode));
1652 }
1653 
1654 void Assembler::cpuid() {
1655   emit_int8(0x0F);
1656   emit_int8((unsigned char)0xA2);
1657 }
1658 
1659 // Opcode / Instruction                      Op /  En  64 - Bit Mode     Compat / Leg Mode Description                  Implemented
1660 // F2 0F 38 F0 / r       CRC32 r32, r / m8   RM        Valid             Valid             Accumulate CRC32 on r / m8.  v
1661 // F2 REX 0F 38 F0 / r   CRC32 r32, r / m8*  RM        Valid             N.E.              Accumulate CRC32 on r / m8.  -
1662 // F2 REX.W 0F 38 F0 / r CRC32 r64, r / m8   RM        Valid             N.E.              Accumulate CRC32 on r / m8.  -
1663 //
1664 // F2 0F 38 F1 / r       CRC32 r32, r / m16  RM        Valid             Valid             Accumulate CRC32 on r / m16. v
1665 //
1666 // F2 0F 38 F1 / r       CRC32 r32, r / m32  RM        Valid             Valid             Accumulate CRC32 on r / m32. v
1667 //
1668 // F2 REX.W 0F 38 F1 / r CRC32 r64, r / m64  RM        Valid             N.E.              Accumulate CRC32 on r / m64. v
1669 void Assembler::crc32(Register crc, Register v, int8_t sizeInBytes) {
1670   assert(VM_Version::supports_sse4_2(), "");
1671   int8_t w = 0x01;
1672   Prefix p = Prefix_EMPTY;
1673 
1674   emit_int8((int8_t)0xF2);
1675   switch (sizeInBytes) {
1676   case 1:
1677     w = 0;
1678     break;
1679   case 2:
1680   case 4:
1681     break;
1682   LP64_ONLY(case 8:)
1683     // This instruction is not valid in 32 bits
1684     // Note:
1685     // http://www.intel.com/content/dam/www/public/us/en/documents/manuals/64-ia-32-architectures-software-developer-instruction-set-reference-manual-325383.pdf
1686     //
1687     // Page B - 72   Vol. 2C says
1688     // qwreg2 to qwreg            1111 0010 : 0100 1R0B : 0000 1111 : 0011 1000 : 1111 0000 : 11 qwreg1 qwreg2
1689     // mem64 to qwreg             1111 0010 : 0100 1R0B : 0000 1111 : 0011 1000 : 1111 0000 : mod qwreg r / m
1690     //                                                                            F0!!!
1691     // while 3 - 208 Vol. 2A
1692     // F2 REX.W 0F 38 F1 / r       CRC32 r64, r / m64             RM         Valid      N.E.Accumulate CRC32 on r / m64.
1693     //
1694     // the 0 on a last bit is reserved for a different flavor of this instruction :
1695     // F2 REX.W 0F 38 F0 / r       CRC32 r64, r / m8              RM         Valid      N.E.Accumulate CRC32 on r / m8.
1696     p = REX_W;
1697     break;
1698   default:
1699     assert(0, "Unsupported value for a sizeInBytes argument");
1700     break;
1701   }
1702   LP64_ONLY(prefix(crc, v, p);)
1703   emit_int8((int8_t)0x0F);
1704   emit_int8(0x38);
1705   emit_int8((int8_t)(0xF0 | w));
1706   emit_int8(0xC0 | ((crc->encoding() & 0x7) << 3) | (v->encoding() & 7));
1707 }
1708 
1709 void Assembler::crc32(Register crc, Address adr, int8_t sizeInBytes) {
1710   assert(VM_Version::supports_sse4_2(), "");
1711   InstructionMark im(this);
1712   int8_t w = 0x01;
1713   Prefix p = Prefix_EMPTY;
1714 
1715   emit_int8((int8_t)0xF2);
1716   switch (sizeInBytes) {
1717   case 1:
1718     w = 0;
1719     break;
1720   case 2:
1721   case 4:
1722     break;
1723   LP64_ONLY(case 8:)
1724     // This instruction is not valid in 32 bits
1725     p = REX_W;
1726     break;
1727   default:
1728     assert(0, "Unsupported value for a sizeInBytes argument");
1729     break;
1730   }
1731   LP64_ONLY(prefix(crc, adr, p);)
1732   emit_int8((int8_t)0x0F);
1733   emit_int8(0x38);
1734   emit_int8((int8_t)(0xF0 | w));
1735   emit_operand(crc, adr);
1736 }
1737 
1738 void Assembler::cvtdq2pd(XMMRegister dst, XMMRegister src) {
1739   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1740   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
1741   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
1742   emit_int8((unsigned char)0xE6);
1743   emit_int8((unsigned char)(0xC0 | encode));
1744 }
1745 
1746 void Assembler::cvtdq2ps(XMMRegister dst, XMMRegister src) {
1747   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1748   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
1749   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
1750   emit_int8(0x5B);
1751   emit_int8((unsigned char)(0xC0 | encode));
1752 }
1753 
1754 void Assembler::cvtsd2ss(XMMRegister dst, XMMRegister src) {
1755   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1756   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
1757   attributes.set_rex_vex_w_reverted();
1758   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
1759   emit_int8(0x5A);
1760   emit_int8((unsigned char)(0xC0 | encode));
1761 }
1762 
1763 void Assembler::cvtsd2ss(XMMRegister dst, Address src) {
1764   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1765   InstructionMark im(this);
1766   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
1767   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
1768   attributes.set_rex_vex_w_reverted();
1769   simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
1770   emit_int8(0x5A);
1771   emit_operand(dst, src);
1772 }
1773 
1774 void Assembler::cvtsi2sdl(XMMRegister dst, Register src) {
1775   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1776   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1777   int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
1778   emit_int8(0x2A);
1779   emit_int8((unsigned char)(0xC0 | encode));
1780 }
1781 
1782 void Assembler::cvtsi2sdl(XMMRegister dst, Address src) {
1783   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1784   InstructionMark im(this);
1785   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1786   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
1787   simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
1788   emit_int8(0x2A);
1789   emit_operand(dst, src);
1790 }
1791 
1792 void Assembler::cvtsi2ssl(XMMRegister dst, Register src) {
1793   NOT_LP64(assert(VM_Version::supports_sse(), ""));
1794   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1795   int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
1796   emit_int8(0x2A);
1797   emit_int8((unsigned char)(0xC0 | encode));
1798 }
1799 
1800 void Assembler::cvtsi2ssl(XMMRegister dst, Address src) {
1801   NOT_LP64(assert(VM_Version::supports_sse(), ""));
1802   InstructionMark im(this);
1803   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1804   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
1805   simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
1806   emit_int8(0x2A);
1807   emit_operand(dst, src);
1808 }
1809 
1810 void Assembler::cvtsi2ssq(XMMRegister dst, Register src) {
1811   NOT_LP64(assert(VM_Version::supports_sse(), ""));
1812   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1813   int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
1814   emit_int8(0x2A);
1815   emit_int8((unsigned char)(0xC0 | encode));
1816 }
1817 
1818 void Assembler::cvtss2sd(XMMRegister dst, XMMRegister src) {
1819   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1820   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
1821   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
1822   emit_int8(0x5A);
1823   emit_int8((unsigned char)(0xC0 | encode));
1824 }
1825 
1826 void Assembler::cvtss2sd(XMMRegister dst, Address src) {
1827   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1828   InstructionMark im(this);
1829   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
1830   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
1831   simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
1832   emit_int8(0x5A);
1833   emit_operand(dst, src);
1834 }
1835 
1836 
1837 void Assembler::cvttsd2sil(Register dst, XMMRegister src) {
1838   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1839   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1840   int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
1841   emit_int8(0x2C);
1842   emit_int8((unsigned char)(0xC0 | encode));
1843 }
1844 
1845 void Assembler::cvttss2sil(Register dst, XMMRegister src) {
1846   NOT_LP64(assert(VM_Version::supports_sse(), ""));
1847   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1848   int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
1849   emit_int8(0x2C);
1850   emit_int8((unsigned char)(0xC0 | encode));
1851 }
1852 
1853 void Assembler::cvttpd2dq(XMMRegister dst, XMMRegister src) {
1854   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1855   int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_128bit;
1856   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
1857   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
1858   emit_int8((unsigned char)0xE6);
1859   emit_int8((unsigned char)(0xC0 | encode));
1860 }
1861 
1862 void Assembler::decl(Address dst) {
1863   // Don't use it directly. Use MacroAssembler::decrement() instead.
1864   InstructionMark im(this);
1865   prefix(dst);
1866   emit_int8((unsigned char)0xFF);
1867   emit_operand(rcx, dst);
1868 }
1869 
1870 void Assembler::divsd(XMMRegister dst, Address src) {
1871   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1872   InstructionMark im(this);
1873   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
1874   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
1875   attributes.set_rex_vex_w_reverted();
1876   simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
1877   emit_int8(0x5E);
1878   emit_operand(dst, src);
1879 }
1880 
1881 void Assembler::divsd(XMMRegister dst, XMMRegister src) {
1882   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1883   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
1884   attributes.set_rex_vex_w_reverted();
1885   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
1886   emit_int8(0x5E);
1887   emit_int8((unsigned char)(0xC0 | encode));
1888 }
1889 
1890 void Assembler::divss(XMMRegister dst, Address src) {
1891   NOT_LP64(assert(VM_Version::supports_sse(), ""));
1892   InstructionMark im(this);
1893   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
1894   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
1895   simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
1896   emit_int8(0x5E);
1897   emit_operand(dst, src);
1898 }
1899 
1900 void Assembler::divss(XMMRegister dst, XMMRegister src) {
1901   NOT_LP64(assert(VM_Version::supports_sse(), ""));
1902   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
1903   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
1904   emit_int8(0x5E);
1905   emit_int8((unsigned char)(0xC0 | encode));
1906 }
1907 
1908 void Assembler::emms() {
1909   NOT_LP64(assert(VM_Version::supports_mmx(), ""));
1910   emit_int8(0x0F);
1911   emit_int8(0x77);
1912 }
1913 
1914 void Assembler::hlt() {
1915   emit_int8((unsigned char)0xF4);
1916 }
1917 
1918 void Assembler::idivl(Register src) {
1919   int encode = prefix_and_encode(src->encoding());
1920   emit_int8((unsigned char)0xF7);
1921   emit_int8((unsigned char)(0xF8 | encode));
1922 }
1923 
1924 void Assembler::divl(Register src) { // Unsigned
1925   int encode = prefix_and_encode(src->encoding());
1926   emit_int8((unsigned char)0xF7);
1927   emit_int8((unsigned char)(0xF0 | encode));
1928 }
1929 
1930 void Assembler::imull(Register src) {
1931   int encode = prefix_and_encode(src->encoding());
1932   emit_int8((unsigned char)0xF7);
1933   emit_int8((unsigned char)(0xE8 | encode));
1934 }
1935 
1936 void Assembler::imull(Register dst, Register src) {
1937   int encode = prefix_and_encode(dst->encoding(), src->encoding());
1938   emit_int8(0x0F);
1939   emit_int8((unsigned char)0xAF);
1940   emit_int8((unsigned char)(0xC0 | encode));
1941 }
1942 
1943 
1944 void Assembler::imull(Register dst, Register src, int value) {
1945   int encode = prefix_and_encode(dst->encoding(), src->encoding());
1946   if (is8bit(value)) {
1947     emit_int8(0x6B);
1948     emit_int8((unsigned char)(0xC0 | encode));
1949     emit_int8(value & 0xFF);
1950   } else {
1951     emit_int8(0x69);
1952     emit_int8((unsigned char)(0xC0 | encode));
1953     emit_int32(value);
1954   }
1955 }
1956 
1957 void Assembler::imull(Register dst, Address src) {
1958   InstructionMark im(this);
1959   prefix(src, dst);
1960   emit_int8(0x0F);
1961   emit_int8((unsigned char) 0xAF);
1962   emit_operand(dst, src);
1963 }
1964 
1965 
1966 void Assembler::incl(Address dst) {
1967   // Don't use it directly. Use MacroAssembler::increment() instead.
1968   InstructionMark im(this);
1969   prefix(dst);
1970   emit_int8((unsigned char)0xFF);
1971   emit_operand(rax, dst);
1972 }
1973 
1974 void Assembler::jcc(Condition cc, Label& L, bool maybe_short) {
1975   InstructionMark im(this);
1976   assert((0 <= cc) && (cc < 16), "illegal cc");
1977   if (L.is_bound()) {
1978     address dst = target(L);
1979     assert(dst != NULL, "jcc most probably wrong");
1980 
1981     const int short_size = 2;
1982     const int long_size = 6;
1983     intptr_t offs = (intptr_t)dst - (intptr_t)pc();
1984     if (maybe_short && is8bit(offs - short_size)) {
1985       // 0111 tttn #8-bit disp
1986       emit_int8(0x70 | cc);
1987       emit_int8((offs - short_size) & 0xFF);
1988     } else {
1989       // 0000 1111 1000 tttn #32-bit disp
1990       assert(is_simm32(offs - long_size),
1991              "must be 32bit offset (call4)");
1992       emit_int8(0x0F);
1993       emit_int8((unsigned char)(0x80 | cc));
1994       emit_int32(offs - long_size);
1995     }
1996   } else {
1997     // Note: could eliminate cond. jumps to this jump if condition
1998     //       is the same however, seems to be rather unlikely case.
1999     // Note: use jccb() if label to be bound is very close to get
2000     //       an 8-bit displacement
2001     L.add_patch_at(code(), locator());
2002     emit_int8(0x0F);
2003     emit_int8((unsigned char)(0x80 | cc));
2004     emit_int32(0);
2005   }
2006 }
2007 
2008 void Assembler::jccb(Condition cc, Label& L) {
2009   if (L.is_bound()) {
2010     const int short_size = 2;
2011     address entry = target(L);
2012 #ifdef ASSERT
2013     intptr_t dist = (intptr_t)entry - ((intptr_t)pc() + short_size);
2014     intptr_t delta = short_branch_delta();
2015     if (delta != 0) {
2016       dist += (dist < 0 ? (-delta) :delta);
2017     }
2018     assert(is8bit(dist), "Dispacement too large for a short jmp");
2019 #endif
2020     intptr_t offs = (intptr_t)entry - (intptr_t)pc();
2021     // 0111 tttn #8-bit disp
2022     emit_int8(0x70 | cc);
2023     emit_int8((offs - short_size) & 0xFF);
2024   } else {
2025     InstructionMark im(this);
2026     L.add_patch_at(code(), locator());
2027     emit_int8(0x70 | cc);
2028     emit_int8(0);
2029   }
2030 }
2031 
2032 void Assembler::jmp(Address adr) {
2033   InstructionMark im(this);
2034   prefix(adr);
2035   emit_int8((unsigned char)0xFF);
2036   emit_operand(rsp, adr);
2037 }
2038 
2039 void Assembler::jmp(Label& L, bool maybe_short) {
2040   if (L.is_bound()) {
2041     address entry = target(L);
2042     assert(entry != NULL, "jmp most probably wrong");
2043     InstructionMark im(this);
2044     const int short_size = 2;
2045     const int long_size = 5;
2046     intptr_t offs = entry - pc();
2047     if (maybe_short && is8bit(offs - short_size)) {
2048       emit_int8((unsigned char)0xEB);
2049       emit_int8((offs - short_size) & 0xFF);
2050     } else {
2051       emit_int8((unsigned char)0xE9);
2052       emit_int32(offs - long_size);
2053     }
2054   } else {
2055     // By default, forward jumps are always 32-bit displacements, since
2056     // we can't yet know where the label will be bound.  If you're sure that
2057     // the forward jump will not run beyond 256 bytes, use jmpb to
2058     // force an 8-bit displacement.
2059     InstructionMark im(this);
2060     L.add_patch_at(code(), locator());
2061     emit_int8((unsigned char)0xE9);
2062     emit_int32(0);
2063   }
2064 }
2065 
2066 void Assembler::jmp(Register entry) {
2067   int encode = prefix_and_encode(entry->encoding());
2068   emit_int8((unsigned char)0xFF);
2069   emit_int8((unsigned char)(0xE0 | encode));
2070 }
2071 
2072 void Assembler::jmp_literal(address dest, RelocationHolder const& rspec) {
2073   InstructionMark im(this);
2074   emit_int8((unsigned char)0xE9);
2075   assert(dest != NULL, "must have a target");
2076   intptr_t disp = dest - (pc() + sizeof(int32_t));
2077   assert(is_simm32(disp), "must be 32bit offset (jmp)");
2078   emit_data(disp, rspec.reloc(), call32_operand);
2079 }
2080 
2081 void Assembler::jmpb(Label& L) {
2082   if (L.is_bound()) {
2083     const int short_size = 2;
2084     address entry = target(L);
2085     assert(entry != NULL, "jmp most probably wrong");
2086 #ifdef ASSERT
2087     intptr_t dist = (intptr_t)entry - ((intptr_t)pc() + short_size);
2088     intptr_t delta = short_branch_delta();
2089     if (delta != 0) {
2090       dist += (dist < 0 ? (-delta) :delta);
2091     }
2092     assert(is8bit(dist), "Dispacement too large for a short jmp");
2093 #endif
2094     intptr_t offs = entry - pc();
2095     emit_int8((unsigned char)0xEB);
2096     emit_int8((offs - short_size) & 0xFF);
2097   } else {
2098     InstructionMark im(this);
2099     L.add_patch_at(code(), locator());
2100     emit_int8((unsigned char)0xEB);
2101     emit_int8(0);
2102   }
2103 }
2104 
2105 void Assembler::ldmxcsr( Address src) {
2106   NOT_LP64(assert(VM_Version::supports_sse(), ""));
2107   InstructionMark im(this);
2108   prefix(src);
2109   emit_int8(0x0F);
2110   emit_int8((unsigned char)0xAE);
2111   emit_operand(as_Register(2), src);
2112 }
2113 
2114 void Assembler::leal(Register dst, Address src) {
2115   InstructionMark im(this);
2116 #ifdef _LP64
2117   emit_int8(0x67); // addr32
2118   prefix(src, dst);
2119 #endif // LP64
2120   emit_int8((unsigned char)0x8D);
2121   emit_operand(dst, src);
2122 }
2123 
2124 void Assembler::lfence() {
2125   emit_int8(0x0F);
2126   emit_int8((unsigned char)0xAE);
2127   emit_int8((unsigned char)0xE8);
2128 }
2129 
2130 void Assembler::lock() {
2131   emit_int8((unsigned char)0xF0);
2132 }
2133 
2134 void Assembler::lzcntl(Register dst, Register src) {
2135   assert(VM_Version::supports_lzcnt(), "encoding is treated as BSR");
2136   emit_int8((unsigned char)0xF3);
2137   int encode = prefix_and_encode(dst->encoding(), src->encoding());
2138   emit_int8(0x0F);
2139   emit_int8((unsigned char)0xBD);
2140   emit_int8((unsigned char)(0xC0 | encode));
2141 }
2142 
2143 // Emit mfence instruction
2144 void Assembler::mfence() {
2145   NOT_LP64(assert(VM_Version::supports_sse2(), "unsupported");)
2146   emit_int8(0x0F);
2147   emit_int8((unsigned char)0xAE);
2148   emit_int8((unsigned char)0xF0);
2149 }
2150 
2151 void Assembler::mov(Register dst, Register src) {
2152   LP64_ONLY(movq(dst, src)) NOT_LP64(movl(dst, src));
2153 }
2154 
2155 void Assembler::movapd(XMMRegister dst, XMMRegister src) {
2156   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2157   int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_128bit;
2158   InstructionAttr attributes(vector_len, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
2159   attributes.set_rex_vex_w_reverted();
2160   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2161   emit_int8(0x28);
2162   emit_int8((unsigned char)(0xC0 | encode));
2163 }
2164 
2165 void Assembler::movaps(XMMRegister dst, XMMRegister src) {
2166   NOT_LP64(assert(VM_Version::supports_sse(), ""));
2167   int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_128bit;
2168   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
2169   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2170   emit_int8(0x28);
2171   emit_int8((unsigned char)(0xC0 | encode));
2172 }
2173 
2174 void Assembler::movlhps(XMMRegister dst, XMMRegister src) {
2175   NOT_LP64(assert(VM_Version::supports_sse(), ""));
2176   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2177   int encode = simd_prefix_and_encode(dst, src, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2178   emit_int8(0x16);
2179   emit_int8((unsigned char)(0xC0 | encode));
2180 }
2181 
2182 void Assembler::movb(Register dst, Address src) {
2183   NOT_LP64(assert(dst->has_byte_register(), "must have byte register"));
2184   InstructionMark im(this);
2185   prefix(src, dst, true);
2186   emit_int8((unsigned char)0x8A);
2187   emit_operand(dst, src);
2188 }
2189 
2190 void Assembler::movddup(XMMRegister dst, XMMRegister src) {
2191   NOT_LP64(assert(VM_Version::supports_sse3(), ""));
2192   int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_128bit;
2193   InstructionAttr attributes(vector_len, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2194   attributes.set_rex_vex_w_reverted();
2195   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2196   emit_int8(0x12);
2197   emit_int8(0xC0 | encode);
2198 }
2199 
2200 void Assembler::kmovbl(KRegister dst, Register src) {
2201   assert(VM_Version::supports_avx512dq(), "");
2202   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2203   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2204   emit_int8((unsigned char)0x92);
2205   emit_int8((unsigned char)(0xC0 | encode));
2206 }
2207 
2208 void Assembler::kmovbl(Register dst, KRegister src) {
2209   assert(VM_Version::supports_avx512dq(), "");
2210   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2211   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2212   emit_int8((unsigned char)0x93);
2213   emit_int8((unsigned char)(0xC0 | encode));
2214 }
2215 
2216 void Assembler::kmovwl(KRegister dst, Register src) {
2217   assert(VM_Version::supports_evex(), "");
2218   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2219   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2220   emit_int8((unsigned char)0x92);
2221   emit_int8((unsigned char)(0xC0 | encode));
2222 }
2223 
2224 void Assembler::kmovwl(Register dst, KRegister src) {
2225   assert(VM_Version::supports_evex(), "");
2226   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2227   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2228   emit_int8((unsigned char)0x93);
2229   emit_int8((unsigned char)(0xC0 | encode));
2230 }
2231 
2232 void Assembler::kmovwl(KRegister dst, Address src) {
2233   assert(VM_Version::supports_evex(), "");
2234   InstructionMark im(this);
2235   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2236   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2237   emit_int8((unsigned char)0x90);
2238   emit_operand((Register)dst, src);
2239 }
2240 
2241 void Assembler::kmovdl(KRegister dst, Register src) {
2242   assert(VM_Version::supports_avx512bw(), "");
2243   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2244   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2245   emit_int8((unsigned char)0x92);
2246   emit_int8((unsigned char)(0xC0 | encode));
2247 }
2248 
2249 void Assembler::kmovdl(Register dst, KRegister src) {
2250   assert(VM_Version::supports_avx512bw(), "");
2251   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2252   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2253   emit_int8((unsigned char)0x93);
2254   emit_int8((unsigned char)(0xC0 | encode));
2255 }
2256 
2257 void Assembler::kmovql(KRegister dst, KRegister src) {
2258   assert(VM_Version::supports_avx512bw(), "");
2259   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2260   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2261   emit_int8((unsigned char)0x90);
2262   emit_int8((unsigned char)(0xC0 | encode));
2263 }
2264 
2265 void Assembler::kmovql(KRegister dst, Address src) {
2266   assert(VM_Version::supports_avx512bw(), "");
2267   InstructionMark im(this);
2268   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2269   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2270   emit_int8((unsigned char)0x90);
2271   emit_operand((Register)dst, src);
2272 }
2273 
2274 void Assembler::kmovql(Address dst, KRegister src) {
2275   assert(VM_Version::supports_avx512bw(), "");
2276   InstructionMark im(this);
2277   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2278   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2279   emit_int8((unsigned char)0x90);
2280   emit_operand((Register)src, dst);
2281 }
2282 
2283 void Assembler::kmovql(KRegister dst, Register src) {
2284   assert(VM_Version::supports_avx512bw(), "");
2285   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2286   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2287   emit_int8((unsigned char)0x92);
2288   emit_int8((unsigned char)(0xC0 | encode));
2289 }
2290 
2291 void Assembler::kmovql(Register dst, KRegister src) {
2292   assert(VM_Version::supports_avx512bw(), "");
2293   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2294   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2295   emit_int8((unsigned char)0x93);
2296   emit_int8((unsigned char)(0xC0 | encode));
2297 }
2298 
2299 void Assembler::knotwl(KRegister dst, KRegister src) {
2300   assert(VM_Version::supports_evex(), "");
2301   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2302   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2303   emit_int8((unsigned char)0x44);
2304   emit_int8((unsigned char)(0xC0 | encode));
2305 }
2306 
2307 // This instruction produces ZF or CF flags
2308 void Assembler::kortestbl(KRegister src1, KRegister src2) {
2309   assert(VM_Version::supports_avx512dq(), "");
2310   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2311   int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2312   emit_int8((unsigned char)0x98);
2313   emit_int8((unsigned char)(0xC0 | encode));
2314 }
2315 
2316 // This instruction produces ZF or CF flags
2317 void Assembler::kortestwl(KRegister src1, KRegister src2) {
2318   assert(VM_Version::supports_evex(), "");
2319   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2320   int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2321   emit_int8((unsigned char)0x98);
2322   emit_int8((unsigned char)(0xC0 | encode));
2323 }
2324 
2325 // This instruction produces ZF or CF flags
2326 void Assembler::kortestdl(KRegister src1, KRegister src2) {
2327   assert(VM_Version::supports_avx512bw(), "");
2328   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2329   int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2330   emit_int8((unsigned char)0x98);
2331   emit_int8((unsigned char)(0xC0 | encode));
2332 }
2333 
2334 // This instruction produces ZF or CF flags
2335 void Assembler::kortestql(KRegister src1, KRegister src2) {
2336   assert(VM_Version::supports_avx512bw(), "");
2337   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2338   int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2339   emit_int8((unsigned char)0x98);
2340   emit_int8((unsigned char)(0xC0 | encode));
2341 }
2342 
2343 // This instruction produces ZF or CF flags
2344 void Assembler::ktestql(KRegister src1, KRegister src2) {
2345   assert(VM_Version::supports_avx512bw(), "");
2346   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2347   int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2348   emit_int8((unsigned char)0x99);
2349   emit_int8((unsigned char)(0xC0 | encode));
2350 }
2351 
2352 void Assembler::ktestq(KRegister src1, KRegister src2) {
2353   assert(VM_Version::supports_avx512bw(), "");
2354   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2355   int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2356   emit_int8((unsigned char)0x99);
2357   emit_int8((unsigned char)(0xC0 | encode));
2358 }
2359 
2360 void Assembler::ktestd(KRegister src1, KRegister src2) {
2361   assert(VM_Version::supports_avx512bw(), "");
2362   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2363   int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2364   emit_int8((unsigned char)0x99);
2365   emit_int8((unsigned char)(0xC0 | encode));
2366 }
2367 
2368 void Assembler::movb(Address dst, int imm8) {
2369   InstructionMark im(this);
2370    prefix(dst);
2371   emit_int8((unsigned char)0xC6);
2372   emit_operand(rax, dst, 1);
2373   emit_int8(imm8);
2374 }
2375 
2376 
2377 void Assembler::movb(Address dst, Register src) {
2378   assert(src->has_byte_register(), "must have byte register");
2379   InstructionMark im(this);
2380   prefix(dst, src, true);
2381   emit_int8((unsigned char)0x88);
2382   emit_operand(src, dst);
2383 }
2384 
2385 void Assembler::movdl(XMMRegister dst, Register src) {
2386   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2387   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2388   int encode = simd_prefix_and_encode(dst, xnoreg, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2389   emit_int8(0x6E);
2390   emit_int8((unsigned char)(0xC0 | encode));
2391 }
2392 
2393 void Assembler::movdl(Register dst, XMMRegister src) {
2394   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2395   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2396   // swap src/dst to get correct prefix
2397   int encode = simd_prefix_and_encode(src, xnoreg, as_XMMRegister(dst->encoding()), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2398   emit_int8(0x7E);
2399   emit_int8((unsigned char)(0xC0 | encode));
2400 }
2401 
2402 void Assembler::movdl(XMMRegister dst, Address src) {
2403   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2404   InstructionMark im(this);
2405   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2406   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
2407   simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2408   emit_int8(0x6E);
2409   emit_operand(dst, src);
2410 }
2411 
2412 void Assembler::movdl(Address dst, XMMRegister src) {
2413   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2414   InstructionMark im(this);
2415   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2416   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
2417   simd_prefix(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2418   emit_int8(0x7E);
2419   emit_operand(src, dst);
2420 }
2421 
2422 void Assembler::movdqa(XMMRegister dst, XMMRegister src) {
2423   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2424   int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_128bit;
2425   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2426   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2427   emit_int8(0x6F);
2428   emit_int8((unsigned char)(0xC0 | encode));
2429 }
2430 
2431 void Assembler::movdqa(XMMRegister dst, Address src) {
2432   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2433   InstructionMark im(this);
2434   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2435   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2436   simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2437   emit_int8(0x6F);
2438   emit_operand(dst, src);
2439 }
2440 
2441 void Assembler::movdqu(XMMRegister dst, Address src) {
2442   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2443   InstructionMark im(this);
2444   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2445   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2446   simd_prefix(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2447   emit_int8(0x6F);
2448   emit_operand(dst, src);
2449 }
2450 
2451 void Assembler::movdqu(XMMRegister dst, XMMRegister src) {
2452   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2453   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2454   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2455   emit_int8(0x6F);
2456   emit_int8((unsigned char)(0xC0 | encode));
2457 }
2458 
2459 void Assembler::movdqu(Address dst, XMMRegister src) {
2460   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2461   InstructionMark im(this);
2462   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2463   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2464   simd_prefix(src, xnoreg, dst, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2465   emit_int8(0x7F);
2466   emit_operand(src, dst);
2467 }
2468 
2469 // Move Unaligned 256bit Vector
2470 void Assembler::vmovdqu(XMMRegister dst, XMMRegister src) {
2471   assert(UseAVX > 0, "");
2472   InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2473   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2474   emit_int8(0x6F);
2475   emit_int8((unsigned char)(0xC0 | encode));
2476 }
2477 
2478 void Assembler::vmovdqu(XMMRegister dst, Address src) {
2479   assert(UseAVX > 0, "");
2480   InstructionMark im(this);
2481   InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2482   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2483   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2484   emit_int8(0x6F);
2485   emit_operand(dst, src);
2486 }
2487 
2488 void Assembler::vmovdqu(Address dst, XMMRegister src) {
2489   assert(UseAVX > 0, "");
2490   InstructionMark im(this);
2491   InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2492   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2493   // swap src<->dst for encoding
2494   assert(src != xnoreg, "sanity");
2495   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2496   emit_int8(0x7F);
2497   emit_operand(src, dst);
2498 }
2499 
2500 // Move Unaligned EVEX enabled Vector (programmable : 8,16,32,64)
2501 void Assembler::evmovdqub(XMMRegister dst, XMMRegister src, int vector_len) {
2502   assert(VM_Version::supports_evex(), "");
2503   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
2504   attributes.set_is_evex_instruction();
2505   int prefix = (_legacy_mode_bw) ? VEX_SIMD_F2 : VEX_SIMD_F3;
2506   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), (Assembler::VexSimdPrefix)prefix, VEX_OPCODE_0F, &attributes);
2507   emit_int8(0x6F);
2508   emit_int8((unsigned char)(0xC0 | encode));
2509 }
2510 
2511 void Assembler::evmovdqub(XMMRegister dst, Address src, int vector_len) {
2512   assert(VM_Version::supports_evex(), "");
2513   InstructionMark im(this);
2514   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
2515   int prefix = (_legacy_mode_bw) ? VEX_SIMD_F2 : VEX_SIMD_F3;
2516   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2517   attributes.set_is_evex_instruction();
2518   vex_prefix(src, 0, dst->encoding(), (Assembler::VexSimdPrefix)prefix, VEX_OPCODE_0F, &attributes);
2519   emit_int8(0x6F);
2520   emit_operand(dst, src);
2521 }
2522 
2523 void Assembler::evmovdqub(Address dst, XMMRegister src, int vector_len) {
2524   assert(VM_Version::supports_evex(), "");
2525   assert(src != xnoreg, "sanity");
2526   InstructionMark im(this);
2527   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
2528   int prefix = (_legacy_mode_bw) ? VEX_SIMD_F2 : VEX_SIMD_F3;
2529   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2530   attributes.set_is_evex_instruction();
2531   vex_prefix(dst, 0, src->encoding(), (Assembler::VexSimdPrefix)prefix, VEX_OPCODE_0F, &attributes);
2532   emit_int8(0x7F);
2533   emit_operand(src, dst);
2534 }
2535 
2536 void Assembler::evmovdqub(XMMRegister dst, KRegister mask, Address src, int vector_len) {
2537   assert(VM_Version::supports_avx512vlbw(), "");
2538   assert(is_vector_masking(), "");    // For stub code use only
2539   InstructionMark im(this);
2540   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
2541   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2542   attributes.set_embedded_opmask_register_specifier(mask);
2543   attributes.set_is_evex_instruction();
2544   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2545   emit_int8(0x6F);
2546   emit_operand(dst, src);
2547 }
2548 
2549 void Assembler::evmovdquw(XMMRegister dst, Address src, int vector_len) {
2550   assert(VM_Version::supports_evex(), "");
2551   InstructionMark im(this);
2552   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
2553   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2554   attributes.set_is_evex_instruction();
2555   int prefix = (_legacy_mode_bw) ? VEX_SIMD_F2 : VEX_SIMD_F3;
2556   vex_prefix(src, 0, dst->encoding(), (Assembler::VexSimdPrefix)prefix, VEX_OPCODE_0F, &attributes);
2557   emit_int8(0x6F);
2558   emit_operand(dst, src);
2559 }
2560 
2561 void Assembler::evmovdquw(XMMRegister dst, KRegister mask, Address src, int vector_len) {
2562   assert(is_vector_masking(), "");
2563   assert(VM_Version::supports_avx512vlbw(), "");
2564   InstructionMark im(this);
2565   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
2566   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2567   attributes.set_embedded_opmask_register_specifier(mask);
2568   attributes.set_is_evex_instruction();
2569   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2570   emit_int8(0x6F);
2571   emit_operand(dst, src);
2572 }
2573 
2574 void Assembler::evmovdquw(Address dst, XMMRegister src, int vector_len) {
2575   assert(VM_Version::supports_evex(), "");
2576   assert(src != xnoreg, "sanity");
2577   InstructionMark im(this);
2578   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
2579   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2580   attributes.set_is_evex_instruction();
2581   int prefix = (_legacy_mode_bw) ? VEX_SIMD_F2 : VEX_SIMD_F3;
2582   vex_prefix(dst, 0, src->encoding(), (Assembler::VexSimdPrefix)prefix, VEX_OPCODE_0F, &attributes);
2583   emit_int8(0x7F);
2584   emit_operand(src, dst);
2585 }
2586 
2587 void Assembler::evmovdquw(Address dst, KRegister mask, XMMRegister src, int vector_len) {
2588   assert(VM_Version::supports_avx512vlbw(), "");
2589   assert(src != xnoreg, "sanity");
2590   InstructionMark im(this);
2591   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2592   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2593   attributes.set_embedded_opmask_register_specifier(mask);
2594   attributes.set_is_evex_instruction();
2595   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2596   emit_int8(0x7F);
2597   emit_operand(src, dst);
2598 }
2599 
2600 void Assembler::evmovdqul(XMMRegister dst, XMMRegister src, int vector_len) {
2601   assert(VM_Version::supports_evex(), "");
2602   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2603   attributes.set_is_evex_instruction();
2604   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2605   emit_int8(0x6F);
2606   emit_int8((unsigned char)(0xC0 | encode));
2607 }
2608 
2609 void Assembler::evmovdqul(XMMRegister dst, Address src, int vector_len) {
2610   assert(VM_Version::supports_evex(), "");
2611   InstructionMark im(this);
2612   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false , /* uses_vl */ true);
2613   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2614   attributes.set_is_evex_instruction();
2615   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2616   emit_int8(0x6F);
2617   emit_operand(dst, src);
2618 }
2619 
2620 void Assembler::evmovdqul(Address dst, XMMRegister src, int vector_len) {
2621   assert(VM_Version::supports_evex(), "");
2622   assert(src != xnoreg, "sanity");
2623   InstructionMark im(this);
2624   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2625   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2626   attributes.set_is_evex_instruction();
2627   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2628   emit_int8(0x7F);
2629   emit_operand(src, dst);
2630 }
2631 
2632 void Assembler::evmovdquq(XMMRegister dst, XMMRegister src, int vector_len) {
2633   assert(VM_Version::supports_evex(), "");
2634   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2635   attributes.set_is_evex_instruction();
2636   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2637   emit_int8(0x6F);
2638   emit_int8((unsigned char)(0xC0 | encode));
2639 }
2640 
2641 void Assembler::evmovdquq(XMMRegister dst, Address src, int vector_len) {
2642   assert(VM_Version::supports_evex(), "");
2643   InstructionMark im(this);
2644   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2645   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2646   attributes.set_is_evex_instruction();
2647   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2648   emit_int8(0x6F);
2649   emit_operand(dst, src);
2650 }
2651 
2652 void Assembler::evmovdquq(Address dst, XMMRegister src, int vector_len) {
2653   assert(VM_Version::supports_evex(), "");
2654   assert(src != xnoreg, "sanity");
2655   InstructionMark im(this);
2656   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2657   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2658   attributes.set_is_evex_instruction();
2659   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2660   emit_int8(0x7F);
2661   emit_operand(src, dst);
2662 }
2663 
2664 // Uses zero extension on 64bit
2665 
2666 void Assembler::movl(Register dst, int32_t imm32) {
2667   int encode = prefix_and_encode(dst->encoding());
2668   emit_int8((unsigned char)(0xB8 | encode));
2669   emit_int32(imm32);
2670 }
2671 
2672 void Assembler::movl(Register dst, Register src) {
2673   int encode = prefix_and_encode(dst->encoding(), src->encoding());
2674   emit_int8((unsigned char)0x8B);
2675   emit_int8((unsigned char)(0xC0 | encode));
2676 }
2677 
2678 void Assembler::movl(Register dst, Address src) {
2679   InstructionMark im(this);
2680   prefix(src, dst);
2681   emit_int8((unsigned char)0x8B);
2682   emit_operand(dst, src);
2683 }
2684 
2685 void Assembler::movl(Address dst, int32_t imm32) {
2686   InstructionMark im(this);
2687   prefix(dst);
2688   emit_int8((unsigned char)0xC7);
2689   emit_operand(rax, dst, 4);
2690   emit_int32(imm32);
2691 }
2692 
2693 void Assembler::movl(Address dst, Register src) {
2694   InstructionMark im(this);
2695   prefix(dst, src);
2696   emit_int8((unsigned char)0x89);
2697   emit_operand(src, dst);
2698 }
2699 
2700 // New cpus require to use movsd and movss to avoid partial register stall
2701 // when loading from memory. But for old Opteron use movlpd instead of movsd.
2702 // The selection is done in MacroAssembler::movdbl() and movflt().
2703 void Assembler::movlpd(XMMRegister dst, Address src) {
2704   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2705   InstructionMark im(this);
2706   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2707   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
2708   attributes.set_rex_vex_w_reverted();
2709   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2710   emit_int8(0x12);
2711   emit_operand(dst, src);
2712 }
2713 
2714 void Assembler::movq( MMXRegister dst, Address src ) {
2715   assert( VM_Version::supports_mmx(), "" );
2716   emit_int8(0x0F);
2717   emit_int8(0x6F);
2718   emit_operand(dst, src);
2719 }
2720 
2721 void Assembler::movq( Address dst, MMXRegister src ) {
2722   assert( VM_Version::supports_mmx(), "" );
2723   emit_int8(0x0F);
2724   emit_int8(0x7F);
2725   // workaround gcc (3.2.1-7a) bug
2726   // In that version of gcc with only an emit_operand(MMX, Address)
2727   // gcc will tail jump and try and reverse the parameters completely
2728   // obliterating dst in the process. By having a version available
2729   // that doesn't need to swap the args at the tail jump the bug is
2730   // avoided.
2731   emit_operand(dst, src);
2732 }
2733 
2734 void Assembler::movq(XMMRegister dst, Address src) {
2735   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2736   InstructionMark im(this);
2737   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2738   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
2739   attributes.set_rex_vex_w_reverted();
2740   simd_prefix(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2741   emit_int8(0x7E);
2742   emit_operand(dst, src);
2743 }
2744 
2745 void Assembler::movq(Address dst, XMMRegister src) {
2746   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2747   InstructionMark im(this);
2748   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2749   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
2750   attributes.set_rex_vex_w_reverted();
2751   simd_prefix(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2752   emit_int8((unsigned char)0xD6);
2753   emit_operand(src, dst);
2754 }
2755 
2756 void Assembler::movsbl(Register dst, Address src) { // movsxb
2757   InstructionMark im(this);
2758   prefix(src, dst);
2759   emit_int8(0x0F);
2760   emit_int8((unsigned char)0xBE);
2761   emit_operand(dst, src);
2762 }
2763 
2764 void Assembler::movsbl(Register dst, Register src) { // movsxb
2765   NOT_LP64(assert(src->has_byte_register(), "must have byte register"));
2766   int encode = prefix_and_encode(dst->encoding(), false, src->encoding(), true);
2767   emit_int8(0x0F);
2768   emit_int8((unsigned char)0xBE);
2769   emit_int8((unsigned char)(0xC0 | encode));
2770 }
2771 
2772 void Assembler::movsd(XMMRegister dst, XMMRegister src) {
2773   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2774   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
2775   attributes.set_rex_vex_w_reverted();
2776   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2777   emit_int8(0x10);
2778   emit_int8((unsigned char)(0xC0 | encode));
2779 }
2780 
2781 void Assembler::movsd(XMMRegister dst, Address src) {
2782   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2783   InstructionMark im(this);
2784   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
2785   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
2786   attributes.set_rex_vex_w_reverted();
2787   simd_prefix(dst, xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2788   emit_int8(0x10);
2789   emit_operand(dst, src);
2790 }
2791 
2792 void Assembler::movsd(Address dst, XMMRegister src) {
2793   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2794   InstructionMark im(this);
2795   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
2796   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
2797   attributes.set_rex_vex_w_reverted();
2798   simd_prefix(src, xnoreg, dst, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2799   emit_int8(0x11);
2800   emit_operand(src, dst);
2801 }
2802 
2803 void Assembler::movss(XMMRegister dst, XMMRegister src) {
2804   NOT_LP64(assert(VM_Version::supports_sse(), ""));
2805   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
2806   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2807   emit_int8(0x10);
2808   emit_int8((unsigned char)(0xC0 | encode));
2809 }
2810 
2811 void Assembler::movss(XMMRegister dst, Address src) {
2812   NOT_LP64(assert(VM_Version::supports_sse(), ""));
2813   InstructionMark im(this);
2814   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
2815   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
2816   simd_prefix(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2817   emit_int8(0x10);
2818   emit_operand(dst, src);
2819 }
2820 
2821 void Assembler::movss(Address dst, XMMRegister src) {
2822   NOT_LP64(assert(VM_Version::supports_sse(), ""));
2823   InstructionMark im(this);
2824   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
2825   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
2826   simd_prefix(src, xnoreg, dst, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2827   emit_int8(0x11);
2828   emit_operand(src, dst);
2829 }
2830 
2831 void Assembler::movswl(Register dst, Address src) { // movsxw
2832   InstructionMark im(this);
2833   prefix(src, dst);
2834   emit_int8(0x0F);
2835   emit_int8((unsigned char)0xBF);
2836   emit_operand(dst, src);
2837 }
2838 
2839 void Assembler::movswl(Register dst, Register src) { // movsxw
2840   int encode = prefix_and_encode(dst->encoding(), src->encoding());
2841   emit_int8(0x0F);
2842   emit_int8((unsigned char)0xBF);
2843   emit_int8((unsigned char)(0xC0 | encode));
2844 }
2845 
2846 void Assembler::movw(Address dst, int imm16) {
2847   InstructionMark im(this);
2848 
2849   emit_int8(0x66); // switch to 16-bit mode
2850   prefix(dst);
2851   emit_int8((unsigned char)0xC7);
2852   emit_operand(rax, dst, 2);
2853   emit_int16(imm16);
2854 }
2855 
2856 void Assembler::movw(Register dst, Address src) {
2857   InstructionMark im(this);
2858   emit_int8(0x66);
2859   prefix(src, dst);
2860   emit_int8((unsigned char)0x8B);
2861   emit_operand(dst, src);
2862 }
2863 
2864 void Assembler::movw(Address dst, Register src) {
2865   InstructionMark im(this);
2866   emit_int8(0x66);
2867   prefix(dst, src);
2868   emit_int8((unsigned char)0x89);
2869   emit_operand(src, dst);
2870 }
2871 
2872 void Assembler::movzbl(Register dst, Address src) { // movzxb
2873   InstructionMark im(this);
2874   prefix(src, dst);
2875   emit_int8(0x0F);
2876   emit_int8((unsigned char)0xB6);
2877   emit_operand(dst, src);
2878 }
2879 
2880 void Assembler::movzbl(Register dst, Register src) { // movzxb
2881   NOT_LP64(assert(src->has_byte_register(), "must have byte register"));
2882   int encode = prefix_and_encode(dst->encoding(), false, src->encoding(), true);
2883   emit_int8(0x0F);
2884   emit_int8((unsigned char)0xB6);
2885   emit_int8(0xC0 | encode);
2886 }
2887 
2888 void Assembler::movzwl(Register dst, Address src) { // movzxw
2889   InstructionMark im(this);
2890   prefix(src, dst);
2891   emit_int8(0x0F);
2892   emit_int8((unsigned char)0xB7);
2893   emit_operand(dst, src);
2894 }
2895 
2896 void Assembler::movzwl(Register dst, Register src) { // movzxw
2897   int encode = prefix_and_encode(dst->encoding(), src->encoding());
2898   emit_int8(0x0F);
2899   emit_int8((unsigned char)0xB7);
2900   emit_int8(0xC0 | encode);
2901 }
2902 
2903 void Assembler::mull(Address src) {
2904   InstructionMark im(this);
2905   prefix(src);
2906   emit_int8((unsigned char)0xF7);
2907   emit_operand(rsp, src);
2908 }
2909 
2910 void Assembler::mull(Register src) {
2911   int encode = prefix_and_encode(src->encoding());
2912   emit_int8((unsigned char)0xF7);
2913   emit_int8((unsigned char)(0xE0 | encode));
2914 }
2915 
2916 void Assembler::mulsd(XMMRegister dst, Address src) {
2917   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2918   InstructionMark im(this);
2919   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
2920   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
2921   attributes.set_rex_vex_w_reverted();
2922   simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2923   emit_int8(0x59);
2924   emit_operand(dst, src);
2925 }
2926 
2927 void Assembler::mulsd(XMMRegister dst, XMMRegister src) {
2928   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2929   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
2930   attributes.set_rex_vex_w_reverted();
2931   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2932   emit_int8(0x59);
2933   emit_int8((unsigned char)(0xC0 | encode));
2934 }
2935 
2936 void Assembler::mulss(XMMRegister dst, Address src) {
2937   NOT_LP64(assert(VM_Version::supports_sse(), ""));
2938   InstructionMark im(this);
2939   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
2940   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
2941   simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2942   emit_int8(0x59);
2943   emit_operand(dst, src);
2944 }
2945 
2946 void Assembler::mulss(XMMRegister dst, XMMRegister src) {
2947   NOT_LP64(assert(VM_Version::supports_sse(), ""));
2948   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
2949   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2950   emit_int8(0x59);
2951   emit_int8((unsigned char)(0xC0 | encode));
2952 }
2953 
2954 void Assembler::negl(Register dst) {
2955   int encode = prefix_and_encode(dst->encoding());
2956   emit_int8((unsigned char)0xF7);
2957   emit_int8((unsigned char)(0xD8 | encode));
2958 }
2959 
2960 void Assembler::nop(int i) {
2961 #ifdef ASSERT
2962   assert(i > 0, " ");
2963   // The fancy nops aren't currently recognized by debuggers making it a
2964   // pain to disassemble code while debugging. If asserts are on clearly
2965   // speed is not an issue so simply use the single byte traditional nop
2966   // to do alignment.
2967 
2968   for (; i > 0 ; i--) emit_int8((unsigned char)0x90);
2969   return;
2970 
2971 #endif // ASSERT
2972 
2973   if (UseAddressNop && VM_Version::is_intel()) {
2974     //
2975     // Using multi-bytes nops "0x0F 0x1F [address]" for Intel
2976     //  1: 0x90
2977     //  2: 0x66 0x90
2978     //  3: 0x66 0x66 0x90 (don't use "0x0F 0x1F 0x00" - need patching safe padding)
2979     //  4: 0x0F 0x1F 0x40 0x00
2980     //  5: 0x0F 0x1F 0x44 0x00 0x00
2981     //  6: 0x66 0x0F 0x1F 0x44 0x00 0x00
2982     //  7: 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00
2983     //  8: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
2984     //  9: 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
2985     // 10: 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
2986     // 11: 0x66 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
2987 
2988     // The rest coding is Intel specific - don't use consecutive address nops
2989 
2990     // 12: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
2991     // 13: 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
2992     // 14: 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
2993     // 15: 0x66 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
2994 
2995     while(i >= 15) {
2996       // For Intel don't generate consecutive addess nops (mix with regular nops)
2997       i -= 15;
2998       emit_int8(0x66);   // size prefix
2999       emit_int8(0x66);   // size prefix
3000       emit_int8(0x66);   // size prefix
3001       addr_nop_8();
3002       emit_int8(0x66);   // size prefix
3003       emit_int8(0x66);   // size prefix
3004       emit_int8(0x66);   // size prefix
3005       emit_int8((unsigned char)0x90);
3006                          // nop
3007     }
3008     switch (i) {
3009       case 14:
3010         emit_int8(0x66); // size prefix
3011       case 13:
3012         emit_int8(0x66); // size prefix
3013       case 12:
3014         addr_nop_8();
3015         emit_int8(0x66); // size prefix
3016         emit_int8(0x66); // size prefix
3017         emit_int8(0x66); // size prefix
3018         emit_int8((unsigned char)0x90);
3019                          // nop
3020         break;
3021       case 11:
3022         emit_int8(0x66); // size prefix
3023       case 10:
3024         emit_int8(0x66); // size prefix
3025       case 9:
3026         emit_int8(0x66); // size prefix
3027       case 8:
3028         addr_nop_8();
3029         break;
3030       case 7:
3031         addr_nop_7();
3032         break;
3033       case 6:
3034         emit_int8(0x66); // size prefix
3035       case 5:
3036         addr_nop_5();
3037         break;
3038       case 4:
3039         addr_nop_4();
3040         break;
3041       case 3:
3042         // Don't use "0x0F 0x1F 0x00" - need patching safe padding
3043         emit_int8(0x66); // size prefix
3044       case 2:
3045         emit_int8(0x66); // size prefix
3046       case 1:
3047         emit_int8((unsigned char)0x90);
3048                          // nop
3049         break;
3050       default:
3051         assert(i == 0, " ");
3052     }
3053     return;
3054   }
3055   if (UseAddressNop && VM_Version::is_amd()) {
3056     //
3057     // Using multi-bytes nops "0x0F 0x1F [address]" for AMD.
3058     //  1: 0x90
3059     //  2: 0x66 0x90
3060     //  3: 0x66 0x66 0x90 (don't use "0x0F 0x1F 0x00" - need patching safe padding)
3061     //  4: 0x0F 0x1F 0x40 0x00
3062     //  5: 0x0F 0x1F 0x44 0x00 0x00
3063     //  6: 0x66 0x0F 0x1F 0x44 0x00 0x00
3064     //  7: 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00
3065     //  8: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
3066     //  9: 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
3067     // 10: 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
3068     // 11: 0x66 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
3069 
3070     // The rest coding is AMD specific - use consecutive address nops
3071 
3072     // 12: 0x66 0x0F 0x1F 0x44 0x00 0x00 0x66 0x0F 0x1F 0x44 0x00 0x00
3073     // 13: 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00 0x66 0x0F 0x1F 0x44 0x00 0x00
3074     // 14: 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00
3075     // 15: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00
3076     // 16: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
3077     //     Size prefixes (0x66) are added for larger sizes
3078 
3079     while(i >= 22) {
3080       i -= 11;
3081       emit_int8(0x66); // size prefix
3082       emit_int8(0x66); // size prefix
3083       emit_int8(0x66); // size prefix
3084       addr_nop_8();
3085     }
3086     // Generate first nop for size between 21-12
3087     switch (i) {
3088       case 21:
3089         i -= 1;
3090         emit_int8(0x66); // size prefix
3091       case 20:
3092       case 19:
3093         i -= 1;
3094         emit_int8(0x66); // size prefix
3095       case 18:
3096       case 17:
3097         i -= 1;
3098         emit_int8(0x66); // size prefix
3099       case 16:
3100       case 15:
3101         i -= 8;
3102         addr_nop_8();
3103         break;
3104       case 14:
3105       case 13:
3106         i -= 7;
3107         addr_nop_7();
3108         break;
3109       case 12:
3110         i -= 6;
3111         emit_int8(0x66); // size prefix
3112         addr_nop_5();
3113         break;
3114       default:
3115         assert(i < 12, " ");
3116     }
3117 
3118     // Generate second nop for size between 11-1
3119     switch (i) {
3120       case 11:
3121         emit_int8(0x66); // size prefix
3122       case 10:
3123         emit_int8(0x66); // size prefix
3124       case 9:
3125         emit_int8(0x66); // size prefix
3126       case 8:
3127         addr_nop_8();
3128         break;
3129       case 7:
3130         addr_nop_7();
3131         break;
3132       case 6:
3133         emit_int8(0x66); // size prefix
3134       case 5:
3135         addr_nop_5();
3136         break;
3137       case 4:
3138         addr_nop_4();
3139         break;
3140       case 3:
3141         // Don't use "0x0F 0x1F 0x00" - need patching safe padding
3142         emit_int8(0x66); // size prefix
3143       case 2:
3144         emit_int8(0x66); // size prefix
3145       case 1:
3146         emit_int8((unsigned char)0x90);
3147                          // nop
3148         break;
3149       default:
3150         assert(i == 0, " ");
3151     }
3152     return;
3153   }
3154 
3155   // Using nops with size prefixes "0x66 0x90".
3156   // From AMD Optimization Guide:
3157   //  1: 0x90
3158   //  2: 0x66 0x90
3159   //  3: 0x66 0x66 0x90
3160   //  4: 0x66 0x66 0x66 0x90
3161   //  5: 0x66 0x66 0x90 0x66 0x90
3162   //  6: 0x66 0x66 0x90 0x66 0x66 0x90
3163   //  7: 0x66 0x66 0x66 0x90 0x66 0x66 0x90
3164   //  8: 0x66 0x66 0x66 0x90 0x66 0x66 0x66 0x90
3165   //  9: 0x66 0x66 0x90 0x66 0x66 0x90 0x66 0x66 0x90
3166   // 10: 0x66 0x66 0x66 0x90 0x66 0x66 0x90 0x66 0x66 0x90
3167   //
3168   while(i > 12) {
3169     i -= 4;
3170     emit_int8(0x66); // size prefix
3171     emit_int8(0x66);
3172     emit_int8(0x66);
3173     emit_int8((unsigned char)0x90);
3174                      // nop
3175   }
3176   // 1 - 12 nops
3177   if(i > 8) {
3178     if(i > 9) {
3179       i -= 1;
3180       emit_int8(0x66);
3181     }
3182     i -= 3;
3183     emit_int8(0x66);
3184     emit_int8(0x66);
3185     emit_int8((unsigned char)0x90);
3186   }
3187   // 1 - 8 nops
3188   if(i > 4) {
3189     if(i > 6) {
3190       i -= 1;
3191       emit_int8(0x66);
3192     }
3193     i -= 3;
3194     emit_int8(0x66);
3195     emit_int8(0x66);
3196     emit_int8((unsigned char)0x90);
3197   }
3198   switch (i) {
3199     case 4:
3200       emit_int8(0x66);
3201     case 3:
3202       emit_int8(0x66);
3203     case 2:
3204       emit_int8(0x66);
3205     case 1:
3206       emit_int8((unsigned char)0x90);
3207       break;
3208     default:
3209       assert(i == 0, " ");
3210   }
3211 }
3212 
3213 void Assembler::notl(Register dst) {
3214   int encode = prefix_and_encode(dst->encoding());
3215   emit_int8((unsigned char)0xF7);
3216   emit_int8((unsigned char)(0xD0 | encode));
3217 }
3218 
3219 void Assembler::orl(Address dst, int32_t imm32) {
3220   InstructionMark im(this);
3221   prefix(dst);
3222   emit_arith_operand(0x81, rcx, dst, imm32);
3223 }
3224 
3225 void Assembler::orl(Register dst, int32_t imm32) {
3226   prefix(dst);
3227   emit_arith(0x81, 0xC8, dst, imm32);
3228 }
3229 
3230 void Assembler::orl(Register dst, Address src) {
3231   InstructionMark im(this);
3232   prefix(src, dst);
3233   emit_int8(0x0B);
3234   emit_operand(dst, src);
3235 }
3236 
3237 void Assembler::orl(Register dst, Register src) {
3238   (void) prefix_and_encode(dst->encoding(), src->encoding());
3239   emit_arith(0x0B, 0xC0, dst, src);
3240 }
3241 
3242 void Assembler::orl(Address dst, Register src) {
3243   InstructionMark im(this);
3244   prefix(dst, src);
3245   emit_int8(0x09);
3246   emit_operand(src, dst);
3247 }
3248 
3249 void Assembler::packuswb(XMMRegister dst, Address src) {
3250   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3251   assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
3252   InstructionMark im(this);
3253   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
3254   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
3255   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3256   emit_int8(0x67);
3257   emit_operand(dst, src);
3258 }
3259 
3260 void Assembler::packuswb(XMMRegister dst, XMMRegister src) {
3261   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3262   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
3263   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3264   emit_int8(0x67);
3265   emit_int8((unsigned char)(0xC0 | encode));
3266 }
3267 
3268 void Assembler::vpackuswb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
3269   assert(UseAVX > 0, "some form of AVX must be enabled");
3270   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
3271   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3272   emit_int8(0x67);
3273   emit_int8((unsigned char)(0xC0 | encode));
3274 }
3275 
3276 void Assembler::vpermq(XMMRegister dst, XMMRegister src, int imm8, int vector_len) {
3277   assert(VM_Version::supports_avx2(), "");
3278   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
3279   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3280   emit_int8(0x00);
3281   emit_int8(0xC0 | encode);
3282   emit_int8(imm8);
3283 }
3284 
3285 void Assembler::vperm2i128(XMMRegister dst,  XMMRegister nds, XMMRegister src, int imm8) {
3286   assert(VM_Version::supports_avx2(), "");
3287   InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3288   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3289   emit_int8(0x46);
3290   emit_int8(0xC0 | encode);
3291   emit_int8(imm8);
3292 }
3293 
3294 
3295 void Assembler::pause() {
3296   emit_int8((unsigned char)0xF3);
3297   emit_int8((unsigned char)0x90);
3298 }
3299 
3300 void Assembler::pcmpestri(XMMRegister dst, Address src, int imm8) {
3301   assert(VM_Version::supports_sse4_2(), "");
3302   InstructionMark im(this);
3303   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3304   simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3305   emit_int8(0x61);
3306   emit_operand(dst, src);
3307   emit_int8(imm8);
3308 }
3309 
3310 void Assembler::pcmpestri(XMMRegister dst, XMMRegister src, int imm8) {
3311   assert(VM_Version::supports_sse4_2(), "");
3312   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3313   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3314   emit_int8(0x61);
3315   emit_int8((unsigned char)(0xC0 | encode));
3316   emit_int8(imm8);
3317 }
3318 
3319 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
3320 void Assembler::pcmpeqb(XMMRegister dst, XMMRegister src) {
3321   assert(VM_Version::supports_sse2(), "");
3322   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3323   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3324   emit_int8(0x74);
3325   emit_int8((unsigned char)(0xC0 | encode));
3326 }
3327 
3328 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
3329 void Assembler::vpcmpeqb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
3330   assert(VM_Version::supports_avx(), "");
3331   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3332   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3333   emit_int8(0x74);
3334   emit_int8((unsigned char)(0xC0 | encode));
3335 }
3336 
3337 // In this context, kdst is written the mask used to process the equal components
3338 void Assembler::evpcmpeqb(KRegister kdst, XMMRegister nds, XMMRegister src, int vector_len) {
3339   assert(VM_Version::supports_avx512bw(), "");
3340   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
3341   attributes.set_is_evex_instruction();
3342   int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3343   emit_int8(0x74);
3344   emit_int8((unsigned char)(0xC0 | encode));
3345 }
3346 
3347 void Assembler::evpcmpgtb(KRegister kdst, XMMRegister nds, Address src, int vector_len) {
3348   assert(VM_Version::supports_avx512vlbw(), "");
3349   InstructionMark im(this);
3350   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3351   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
3352   attributes.set_is_evex_instruction();
3353   int dst_enc = kdst->encoding();
3354   vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3355   emit_int8(0x64);
3356   emit_operand(as_Register(dst_enc), src);
3357 }
3358 
3359 void Assembler::evpcmpgtb(KRegister kdst, KRegister mask, XMMRegister nds, Address src, int vector_len) {
3360   assert(is_vector_masking(), "");
3361   assert(VM_Version::supports_avx512vlbw(), "");
3362   InstructionMark im(this);
3363   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
3364   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
3365   attributes.set_embedded_opmask_register_specifier(mask);
3366   attributes.set_is_evex_instruction();
3367   int dst_enc = kdst->encoding();
3368   vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3369   emit_int8(0x64);
3370   emit_operand(as_Register(dst_enc), src);
3371 }
3372 
3373 void Assembler::evpcmpuw(KRegister kdst, XMMRegister nds, XMMRegister src, ComparisonPredicate vcc, int vector_len) {
3374   assert(VM_Version::supports_avx512vlbw(), "");
3375   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3376   attributes.set_is_evex_instruction();
3377   int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3378   emit_int8(0x3E);
3379   emit_int8((unsigned char)(0xC0 | encode));
3380   emit_int8(vcc);
3381 }
3382 
3383 void Assembler::evpcmpuw(KRegister kdst, KRegister mask, XMMRegister nds, XMMRegister src, ComparisonPredicate vcc, int vector_len) {
3384   assert(is_vector_masking(), "");
3385   assert(VM_Version::supports_avx512vlbw(), "");
3386   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
3387   attributes.set_embedded_opmask_register_specifier(mask);
3388   attributes.set_is_evex_instruction();
3389   int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3390   emit_int8(0x3E);
3391   emit_int8((unsigned char)(0xC0 | encode));
3392   emit_int8(vcc);
3393 }
3394 
3395 void Assembler::evpcmpuw(KRegister kdst, XMMRegister nds, Address src, ComparisonPredicate vcc, int vector_len) {
3396   assert(VM_Version::supports_avx512vlbw(), "");
3397   InstructionMark im(this);
3398   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3399   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
3400   attributes.set_is_evex_instruction();
3401   int dst_enc = kdst->encoding();
3402   vex_prefix(src, nds->encoding(), kdst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3403   emit_int8(0x3E);
3404   emit_operand(as_Register(dst_enc), src);
3405   emit_int8(vcc);
3406 }
3407 
3408 void Assembler::evpcmpeqb(KRegister kdst, XMMRegister nds, Address src, int vector_len) {
3409   assert(VM_Version::supports_avx512bw(), "");
3410   InstructionMark im(this);
3411   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3412   attributes.set_is_evex_instruction();
3413   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
3414   int dst_enc = kdst->encoding();
3415   vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3416   emit_int8(0x74);
3417   emit_operand(as_Register(dst_enc), src);
3418 }
3419 
3420 void Assembler::evpcmpeqb(KRegister kdst, KRegister mask, XMMRegister nds, Address src, int vector_len) {
3421   assert(VM_Version::supports_avx512vlbw(), "");
3422   assert(is_vector_masking(), "");    // For stub code use only
3423   InstructionMark im(this);
3424   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_reg_mask */ false, /* uses_vl */ false);
3425   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
3426   attributes.set_embedded_opmask_register_specifier(mask);
3427   attributes.set_is_evex_instruction();
3428   vex_prefix(src, nds->encoding(), kdst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3429   emit_int8(0x74);
3430   emit_operand(as_Register(kdst->encoding()), src);
3431 }
3432 
3433 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
3434 void Assembler::pcmpeqw(XMMRegister dst, XMMRegister src) {
3435   assert(VM_Version::supports_sse2(), "");
3436   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3437   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3438   emit_int8(0x75);
3439   emit_int8((unsigned char)(0xC0 | encode));
3440 }
3441 
3442 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
3443 void Assembler::vpcmpeqw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
3444   assert(VM_Version::supports_avx(), "");
3445   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3446   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3447   emit_int8(0x75);
3448   emit_int8((unsigned char)(0xC0 | encode));
3449 }
3450 
3451 // In this context, kdst is written the mask used to process the equal components
3452 void Assembler::evpcmpeqw(KRegister kdst, XMMRegister nds, XMMRegister src, int vector_len) {
3453   assert(VM_Version::supports_avx512bw(), "");
3454   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
3455   attributes.set_is_evex_instruction();
3456   int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3457   emit_int8(0x75);
3458   emit_int8((unsigned char)(0xC0 | encode));
3459 }
3460 
3461 void Assembler::evpcmpeqw(KRegister kdst, XMMRegister nds, Address src, int vector_len) {
3462   assert(VM_Version::supports_avx512bw(), "");
3463   InstructionMark im(this);
3464   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
3465   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
3466   attributes.set_is_evex_instruction();
3467   int dst_enc = kdst->encoding();
3468   vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3469   emit_int8(0x75);
3470   emit_operand(as_Register(dst_enc), src);
3471 }
3472 
3473 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
3474 void Assembler::pcmpeqd(XMMRegister dst, XMMRegister src) {
3475   assert(VM_Version::supports_sse2(), "");
3476   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3477   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3478   emit_int8(0x76);
3479   emit_int8((unsigned char)(0xC0 | encode));
3480 }
3481 
3482 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
3483 void Assembler::vpcmpeqd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
3484   assert(VM_Version::supports_avx(), "");
3485   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3486   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3487   emit_int8(0x76);
3488   emit_int8((unsigned char)(0xC0 | encode));
3489 }
3490 
3491 // In this context, kdst is written the mask used to process the equal components
3492 void Assembler::evpcmpeqd(KRegister kdst, XMMRegister nds, XMMRegister src, int vector_len) {
3493   assert(VM_Version::supports_evex(), "");
3494   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
3495   attributes.set_is_evex_instruction();
3496   int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3497   emit_int8(0x76);
3498   emit_int8((unsigned char)(0xC0 | encode));
3499 }
3500 
3501 void Assembler::evpcmpeqd(KRegister kdst, XMMRegister nds, Address src, int vector_len) {
3502   assert(VM_Version::supports_evex(), "");
3503   InstructionMark im(this);
3504   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
3505   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
3506   attributes.set_is_evex_instruction();
3507   int dst_enc = kdst->encoding();
3508   vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3509   emit_int8(0x76);
3510   emit_operand(as_Register(dst_enc), src);
3511 }
3512 
3513 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
3514 void Assembler::pcmpeqq(XMMRegister dst, XMMRegister src) {
3515   assert(VM_Version::supports_sse4_1(), "");
3516   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3517   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3518   emit_int8(0x29);
3519   emit_int8((unsigned char)(0xC0 | encode));
3520 }
3521 
3522 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
3523 void Assembler::vpcmpeqq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
3524   assert(VM_Version::supports_avx(), "");
3525   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3526   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3527   emit_int8(0x29);
3528   emit_int8((unsigned char)(0xC0 | encode));
3529 }
3530 
3531 // In this context, kdst is written the mask used to process the equal components
3532 void Assembler::evpcmpeqq(KRegister kdst, XMMRegister nds, XMMRegister src, int vector_len) {
3533   assert(VM_Version::supports_evex(), "");
3534   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
3535   attributes.set_is_evex_instruction();
3536   int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3537   emit_int8(0x29);
3538   emit_int8((unsigned char)(0xC0 | encode));
3539 }
3540 
3541 // In this context, kdst is written the mask used to process the equal components
3542 void Assembler::evpcmpeqq(KRegister kdst, XMMRegister nds, Address src, int vector_len) {
3543   assert(VM_Version::supports_evex(), "");
3544   InstructionMark im(this);
3545   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
3546   attributes.set_is_evex_instruction();
3547   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
3548   int dst_enc = kdst->encoding();
3549   vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3550   emit_int8(0x29);
3551   emit_operand(as_Register(dst_enc), src);
3552 }
3553 
3554 void Assembler::pmovmskb(Register dst, XMMRegister src) {
3555   assert(VM_Version::supports_sse2(), "");
3556   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3557   int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3558   emit_int8((unsigned char)0xD7);
3559   emit_int8((unsigned char)(0xC0 | encode));
3560 }
3561 
3562 void Assembler::vpmovmskb(Register dst, XMMRegister src) {
3563   assert(VM_Version::supports_avx2(), "");
3564   InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3565   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3566   emit_int8((unsigned char)0xD7);
3567   emit_int8((unsigned char)(0xC0 | encode));
3568 }
3569 
3570 void Assembler::pextrd(Register dst, XMMRegister src, int imm8) {
3571   assert(VM_Version::supports_sse4_1(), "");
3572   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
3573   int encode = simd_prefix_and_encode(src, xnoreg, as_XMMRegister(dst->encoding()), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3574   emit_int8(0x16);
3575   emit_int8((unsigned char)(0xC0 | encode));
3576   emit_int8(imm8);
3577 }
3578 
3579 void Assembler::pextrd(Address dst, XMMRegister src, int imm8) {
3580   assert(VM_Version::supports_sse4_1(), "");
3581   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
3582   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
3583   simd_prefix(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3584   emit_int8(0x16);
3585   emit_operand(src, dst);
3586   emit_int8(imm8);
3587 }
3588 
3589 void Assembler::pextrq(Register dst, XMMRegister src, int imm8) {
3590   assert(VM_Version::supports_sse4_1(), "");
3591   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
3592   int encode = simd_prefix_and_encode(src, xnoreg, as_XMMRegister(dst->encoding()), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3593   emit_int8(0x16);
3594   emit_int8((unsigned char)(0xC0 | encode));
3595   emit_int8(imm8);
3596 }
3597 
3598 void Assembler::pextrq(Address dst, XMMRegister src, int imm8) {
3599   assert(VM_Version::supports_sse4_1(), "");
3600   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
3601   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
3602   simd_prefix(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3603   emit_int8(0x16);
3604   emit_operand(src, dst);
3605   emit_int8(imm8);
3606 }
3607 
3608 void Assembler::pextrw(Register dst, XMMRegister src, int imm8) {
3609   assert(VM_Version::supports_sse2(), "");
3610   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
3611   int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3612   emit_int8((unsigned char)0xC5);
3613   emit_int8((unsigned char)(0xC0 | encode));
3614   emit_int8(imm8);
3615 }
3616 
3617 void Assembler::pextrw(Address dst, XMMRegister src, int imm8) {
3618   assert(VM_Version::supports_sse4_1(), "");
3619   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
3620   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_16bit);
3621   simd_prefix(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3622   emit_int8((unsigned char)0x15);
3623   emit_operand(src, dst);
3624   emit_int8(imm8);
3625 }
3626 
3627 void Assembler::pextrb(Address dst, XMMRegister src, int imm8) {
3628   assert(VM_Version::supports_sse4_1(), "");
3629   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
3630   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_8bit);
3631   simd_prefix(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3632   emit_int8(0x14);
3633   emit_operand(src, dst);
3634   emit_int8(imm8);
3635 }
3636 
3637 void Assembler::pinsrd(XMMRegister dst, Register src, int imm8) {
3638   assert(VM_Version::supports_sse4_1(), "");
3639   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
3640   int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3641   emit_int8(0x22);
3642   emit_int8((unsigned char)(0xC0 | encode));
3643   emit_int8(imm8);
3644 }
3645 
3646 void Assembler::pinsrd(XMMRegister dst, Address src, int imm8) {
3647   assert(VM_Version::supports_sse4_1(), "");
3648   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
3649   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
3650   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3651   emit_int8(0x22);
3652   emit_operand(dst,src);
3653   emit_int8(imm8);
3654 }
3655 
3656 void Assembler::pinsrq(XMMRegister dst, Register src, int imm8) {
3657   assert(VM_Version::supports_sse4_1(), "");
3658   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
3659   int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3660   emit_int8(0x22);
3661   emit_int8((unsigned char)(0xC0 | encode));
3662   emit_int8(imm8);
3663 }
3664 
3665 void Assembler::pinsrq(XMMRegister dst, Address src, int imm8) {
3666   assert(VM_Version::supports_sse4_1(), "");
3667   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
3668   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
3669   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3670   emit_int8(0x22);
3671   emit_operand(dst, src);
3672   emit_int8(imm8);
3673 }
3674 
3675 void Assembler::pinsrw(XMMRegister dst, Register src, int imm8) {
3676   assert(VM_Version::supports_sse2(), "");
3677   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
3678   int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3679   emit_int8((unsigned char)0xC4);
3680   emit_int8((unsigned char)(0xC0 | encode));
3681   emit_int8(imm8);
3682 }
3683 
3684 void Assembler::pinsrw(XMMRegister dst, Address src, int imm8) {
3685   assert(VM_Version::supports_sse2(), "");
3686   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
3687   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_16bit);
3688   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3689   emit_int8((unsigned char)0xC4);
3690   emit_operand(dst, src);
3691   emit_int8(imm8);
3692 }
3693 
3694 void Assembler::pinsrb(XMMRegister dst, Address src, int imm8) {
3695   assert(VM_Version::supports_sse4_1(), "");
3696   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
3697   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_8bit);
3698   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3699   emit_int8(0x20);
3700   emit_operand(dst, src);
3701   emit_int8(imm8);
3702 }
3703 
3704 void Assembler::pmovzxbw(XMMRegister dst, Address src) {
3705   assert(VM_Version::supports_sse4_1(), "");
3706   InstructionMark im(this);
3707   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
3708   attributes.set_address_attributes(/* tuple_type */ EVEX_HVM, /* input_size_in_bits */ EVEX_NObit);
3709   simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3710   emit_int8(0x30);
3711   emit_operand(dst, src);
3712 }
3713 
3714 void Assembler::pmovzxbw(XMMRegister dst, XMMRegister src) {
3715   assert(VM_Version::supports_sse4_1(), "");
3716   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
3717   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3718   emit_int8(0x30);
3719   emit_int8((unsigned char)(0xC0 | encode));
3720 }
3721 
3722 void Assembler::vpmovzxbw(XMMRegister dst, Address src, int vector_len) {
3723   assert(VM_Version::supports_avx(), "");
3724   InstructionMark im(this);
3725   assert(dst != xnoreg, "sanity");
3726   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
3727   attributes.set_address_attributes(/* tuple_type */ EVEX_HVM, /* input_size_in_bits */ EVEX_NObit);
3728   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3729   emit_int8(0x30);
3730   emit_operand(dst, src);
3731 }
3732 
3733 void Assembler::evpmovzxbw(XMMRegister dst, KRegister mask, Address src, int vector_len) {
3734   assert(is_vector_masking(), "");
3735   assert(VM_Version::supports_avx512vlbw(), "");
3736   assert(dst != xnoreg, "sanity");
3737   InstructionMark im(this);
3738   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
3739   attributes.set_address_attributes(/* tuple_type */ EVEX_HVM, /* input_size_in_bits */ EVEX_NObit);
3740   attributes.set_embedded_opmask_register_specifier(mask);
3741   attributes.set_is_evex_instruction();
3742   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3743   emit_int8(0x30);
3744   emit_operand(dst, src);
3745 }
3746 
3747 void Assembler::evpmovwb(Address dst, XMMRegister src, int vector_len) {
3748   assert(VM_Version::supports_avx512vlbw(), "");
3749   assert(src != xnoreg, "sanity");
3750   InstructionMark im(this);
3751   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3752   attributes.set_address_attributes(/* tuple_type */ EVEX_HVM, /* input_size_in_bits */ EVEX_NObit);
3753   attributes.set_is_evex_instruction();
3754   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
3755   emit_int8(0x30);
3756   emit_operand(src, dst);
3757 }
3758 
3759 void Assembler::evpmovwb(Address dst, KRegister mask, XMMRegister src, int vector_len) {
3760   assert(is_vector_masking(), "");
3761   assert(VM_Version::supports_avx512vlbw(), "");
3762   assert(src != xnoreg, "sanity");
3763   InstructionMark im(this);
3764   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
3765   attributes.set_address_attributes(/* tuple_type */ EVEX_HVM, /* input_size_in_bits */ EVEX_NObit);
3766   attributes.set_embedded_opmask_register_specifier(mask);
3767   attributes.set_is_evex_instruction();
3768   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
3769   emit_int8(0x30);
3770   emit_operand(src, dst);
3771 }
3772 
3773 // generic
3774 void Assembler::pop(Register dst) {
3775   int encode = prefix_and_encode(dst->encoding());
3776   emit_int8(0x58 | encode);
3777 }
3778 
3779 void Assembler::popcntl(Register dst, Address src) {
3780   assert(VM_Version::supports_popcnt(), "must support");
3781   InstructionMark im(this);
3782   emit_int8((unsigned char)0xF3);
3783   prefix(src, dst);
3784   emit_int8(0x0F);
3785   emit_int8((unsigned char)0xB8);
3786   emit_operand(dst, src);
3787 }
3788 
3789 void Assembler::popcntl(Register dst, Register src) {
3790   assert(VM_Version::supports_popcnt(), "must support");
3791   emit_int8((unsigned char)0xF3);
3792   int encode = prefix_and_encode(dst->encoding(), src->encoding());
3793   emit_int8(0x0F);
3794   emit_int8((unsigned char)0xB8);
3795   emit_int8((unsigned char)(0xC0 | encode));
3796 }
3797 
3798 void Assembler::popf() {
3799   emit_int8((unsigned char)0x9D);
3800 }
3801 
3802 #ifndef _LP64 // no 32bit push/pop on amd64
3803 void Assembler::popl(Address dst) {
3804   // NOTE: this will adjust stack by 8byte on 64bits
3805   InstructionMark im(this);
3806   prefix(dst);
3807   emit_int8((unsigned char)0x8F);
3808   emit_operand(rax, dst);
3809 }
3810 #endif
3811 
3812 void Assembler::prefetch_prefix(Address src) {
3813   prefix(src);
3814   emit_int8(0x0F);
3815 }
3816 
3817 void Assembler::prefetchnta(Address src) {
3818   NOT_LP64(assert(VM_Version::supports_sse(), "must support"));
3819   InstructionMark im(this);
3820   prefetch_prefix(src);
3821   emit_int8(0x18);
3822   emit_operand(rax, src); // 0, src
3823 }
3824 
3825 void Assembler::prefetchr(Address src) {
3826   assert(VM_Version::supports_3dnow_prefetch(), "must support");
3827   InstructionMark im(this);
3828   prefetch_prefix(src);
3829   emit_int8(0x0D);
3830   emit_operand(rax, src); // 0, src
3831 }
3832 
3833 void Assembler::prefetcht0(Address src) {
3834   NOT_LP64(assert(VM_Version::supports_sse(), "must support"));
3835   InstructionMark im(this);
3836   prefetch_prefix(src);
3837   emit_int8(0x18);
3838   emit_operand(rcx, src); // 1, src
3839 }
3840 
3841 void Assembler::prefetcht1(Address src) {
3842   NOT_LP64(assert(VM_Version::supports_sse(), "must support"));
3843   InstructionMark im(this);
3844   prefetch_prefix(src);
3845   emit_int8(0x18);
3846   emit_operand(rdx, src); // 2, src
3847 }
3848 
3849 void Assembler::prefetcht2(Address src) {
3850   NOT_LP64(assert(VM_Version::supports_sse(), "must support"));
3851   InstructionMark im(this);
3852   prefetch_prefix(src);
3853   emit_int8(0x18);
3854   emit_operand(rbx, src); // 3, src
3855 }
3856 
3857 void Assembler::prefetchw(Address src) {
3858   assert(VM_Version::supports_3dnow_prefetch(), "must support");
3859   InstructionMark im(this);
3860   prefetch_prefix(src);
3861   emit_int8(0x0D);
3862   emit_operand(rcx, src); // 1, src
3863 }
3864 
3865 void Assembler::prefix(Prefix p) {
3866   emit_int8(p);
3867 }
3868 
3869 void Assembler::pshufb(XMMRegister dst, XMMRegister src) {
3870   assert(VM_Version::supports_ssse3(), "");
3871   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
3872   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3873   emit_int8(0x00);
3874   emit_int8((unsigned char)(0xC0 | encode));
3875 }
3876 
3877 void Assembler::vpshufb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
3878   assert(vector_len == AVX_128bit? VM_Version::supports_avx() :
3879          vector_len == AVX_256bit? VM_Version::supports_avx2() :
3880          0, "");
3881   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
3882   int encode = simd_prefix_and_encode(dst, nds, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3883   emit_int8(0x00);
3884   emit_int8((unsigned char)(0xC0 | encode));
3885 }
3886 
3887 void Assembler::pshufb(XMMRegister dst, Address src) {
3888   assert(VM_Version::supports_ssse3(), "");
3889   InstructionMark im(this);
3890   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
3891   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
3892   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3893   emit_int8(0x00);
3894   emit_operand(dst, src);
3895 }
3896 
3897 void Assembler::pshufd(XMMRegister dst, XMMRegister src, int mode) {
3898   assert(isByte(mode), "invalid value");
3899   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3900   int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_128bit;
3901   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
3902   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3903   emit_int8(0x70);
3904   emit_int8((unsigned char)(0xC0 | encode));
3905   emit_int8(mode & 0xFF);
3906 }
3907 
3908 void Assembler::vpshufd(XMMRegister dst, XMMRegister src, int mode, int vector_len) {
3909   assert(vector_len == AVX_128bit? VM_Version::supports_avx() :
3910          vector_len == AVX_256bit? VM_Version::supports_avx2() :
3911          0, "");
3912   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3913   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
3914   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3915   emit_int8(0x70);
3916   emit_int8((unsigned char)(0xC0 | encode));
3917   emit_int8(mode & 0xFF);
3918 }
3919 
3920 void Assembler::pshufd(XMMRegister dst, Address src, int mode) {
3921   assert(isByte(mode), "invalid value");
3922   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3923   assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
3924   InstructionMark im(this);
3925   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
3926   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
3927   simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3928   emit_int8(0x70);
3929   emit_operand(dst, src);
3930   emit_int8(mode & 0xFF);
3931 }
3932 
3933 void Assembler::pshuflw(XMMRegister dst, XMMRegister src, int mode) {
3934   assert(isByte(mode), "invalid value");
3935   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3936   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
3937   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
3938   emit_int8(0x70);
3939   emit_int8((unsigned char)(0xC0 | encode));
3940   emit_int8(mode & 0xFF);
3941 }
3942 
3943 void Assembler::pshuflw(XMMRegister dst, Address src, int mode) {
3944   assert(isByte(mode), "invalid value");
3945   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3946   assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
3947   InstructionMark im(this);
3948   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
3949   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
3950   simd_prefix(dst, xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
3951   emit_int8(0x70);
3952   emit_operand(dst, src);
3953   emit_int8(mode & 0xFF);
3954 }
3955 
3956 void Assembler::psrldq(XMMRegister dst, int shift) {
3957   // Shift left 128 bit value in dst XMMRegister by shift number of bytes.
3958   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3959   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
3960   int encode = simd_prefix_and_encode(xmm3, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3961   emit_int8(0x73);
3962   emit_int8((unsigned char)(0xC0 | encode));
3963   emit_int8(shift);
3964 }
3965 
3966 void Assembler::pslldq(XMMRegister dst, int shift) {
3967   // Shift left 128 bit value in dst XMMRegister by shift number of bytes.
3968   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3969   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
3970   // XMM7 is for /7 encoding: 66 0F 73 /7 ib
3971   int encode = simd_prefix_and_encode(xmm7, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3972   emit_int8(0x73);
3973   emit_int8((unsigned char)(0xC0 | encode));
3974   emit_int8(shift);
3975 }
3976 
3977 void Assembler::ptest(XMMRegister dst, Address src) {
3978   assert(VM_Version::supports_sse4_1(), "");
3979   assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
3980   InstructionMark im(this);
3981   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3982   simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3983   emit_int8(0x17);
3984   emit_operand(dst, src);
3985 }
3986 
3987 void Assembler::ptest(XMMRegister dst, XMMRegister src) {
3988   assert(VM_Version::supports_sse4_1(), "");
3989   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3990   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3991   emit_int8(0x17);
3992   emit_int8((unsigned char)(0xC0 | encode));
3993 }
3994 
3995 void Assembler::vptest(XMMRegister dst, Address src) {
3996   assert(VM_Version::supports_avx(), "");
3997   InstructionMark im(this);
3998   InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3999   assert(dst != xnoreg, "sanity");
4000   // swap src<->dst for encoding
4001   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4002   emit_int8(0x17);
4003   emit_operand(dst, src);
4004 }
4005 
4006 void Assembler::vptest(XMMRegister dst, XMMRegister src) {
4007   assert(VM_Version::supports_avx(), "");
4008   InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
4009   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4010   emit_int8(0x17);
4011   emit_int8((unsigned char)(0xC0 | encode));
4012 }
4013 
4014 void Assembler::punpcklbw(XMMRegister dst, Address src) {
4015   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4016   assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
4017   InstructionMark im(this);
4018   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_vlbw, /* no_mask_reg */ false, /* uses_vl */ true);
4019   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
4020   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4021   emit_int8(0x60);
4022   emit_operand(dst, src);
4023 }
4024 
4025 void Assembler::punpcklbw(XMMRegister dst, XMMRegister src) {
4026   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4027   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_vlbw, /* no_mask_reg */ false, /* uses_vl */ true);
4028   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4029   emit_int8(0x60);
4030   emit_int8((unsigned char)(0xC0 | encode));
4031 }
4032 
4033 void Assembler::punpckldq(XMMRegister dst, Address src) {
4034   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4035   assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
4036   InstructionMark im(this);
4037   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4038   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
4039   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4040   emit_int8(0x62);
4041   emit_operand(dst, src);
4042 }
4043 
4044 void Assembler::punpckldq(XMMRegister dst, XMMRegister src) {
4045   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4046   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4047   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4048   emit_int8(0x62);
4049   emit_int8((unsigned char)(0xC0 | encode));
4050 }
4051 
4052 void Assembler::punpcklqdq(XMMRegister dst, XMMRegister src) {
4053   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4054   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4055   attributes.set_rex_vex_w_reverted();
4056   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4057   emit_int8(0x6C);
4058   emit_int8((unsigned char)(0xC0 | encode));
4059 }
4060 
4061 void Assembler::push(int32_t imm32) {
4062   // in 64bits we push 64bits onto the stack but only
4063   // take a 32bit immediate
4064   emit_int8(0x68);
4065   emit_int32(imm32);
4066 }
4067 
4068 void Assembler::push(Register src) {
4069   int encode = prefix_and_encode(src->encoding());
4070 
4071   emit_int8(0x50 | encode);
4072 }
4073 
4074 void Assembler::pushf() {
4075   emit_int8((unsigned char)0x9C);
4076 }
4077 
4078 #ifndef _LP64 // no 32bit push/pop on amd64
4079 void Assembler::pushl(Address src) {
4080   // Note this will push 64bit on 64bit
4081   InstructionMark im(this);
4082   prefix(src);
4083   emit_int8((unsigned char)0xFF);
4084   emit_operand(rsi, src);
4085 }
4086 #endif
4087 
4088 void Assembler::rcll(Register dst, int imm8) {
4089   assert(isShiftCount(imm8), "illegal shift count");
4090   int encode = prefix_and_encode(dst->encoding());
4091   if (imm8 == 1) {
4092     emit_int8((unsigned char)0xD1);
4093     emit_int8((unsigned char)(0xD0 | encode));
4094   } else {
4095     emit_int8((unsigned char)0xC1);
4096     emit_int8((unsigned char)0xD0 | encode);
4097     emit_int8(imm8);
4098   }
4099 }
4100 
4101 void Assembler::rcpps(XMMRegister dst, XMMRegister src) {
4102   NOT_LP64(assert(VM_Version::supports_sse(), ""));
4103   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
4104   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
4105   emit_int8(0x53);
4106   emit_int8((unsigned char)(0xC0 | encode));
4107 }
4108 
4109 void Assembler::rcpss(XMMRegister dst, XMMRegister src) {
4110   NOT_LP64(assert(VM_Version::supports_sse(), ""));
4111   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
4112   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
4113   emit_int8(0x53);
4114   emit_int8((unsigned char)(0xC0 | encode));
4115 }
4116 
4117 void Assembler::rdtsc() {
4118   emit_int8((unsigned char)0x0F);
4119   emit_int8((unsigned char)0x31);
4120 }
4121 
4122 // copies data from [esi] to [edi] using rcx pointer sized words
4123 // generic
4124 void Assembler::rep_mov() {
4125   emit_int8((unsigned char)0xF3);
4126   // MOVSQ
4127   LP64_ONLY(prefix(REX_W));
4128   emit_int8((unsigned char)0xA5);
4129 }
4130 
4131 // sets rcx bytes with rax, value at [edi]
4132 void Assembler::rep_stosb() {
4133   emit_int8((unsigned char)0xF3); // REP
4134   LP64_ONLY(prefix(REX_W));
4135   emit_int8((unsigned char)0xAA); // STOSB
4136 }
4137 
4138 // sets rcx pointer sized words with rax, value at [edi]
4139 // generic
4140 void Assembler::rep_stos() {
4141   emit_int8((unsigned char)0xF3); // REP
4142   LP64_ONLY(prefix(REX_W));       // LP64:STOSQ, LP32:STOSD
4143   emit_int8((unsigned char)0xAB);
4144 }
4145 
4146 // scans rcx pointer sized words at [edi] for occurance of rax,
4147 // generic
4148 void Assembler::repne_scan() { // repne_scan
4149   emit_int8((unsigned char)0xF2);
4150   // SCASQ
4151   LP64_ONLY(prefix(REX_W));
4152   emit_int8((unsigned char)0xAF);
4153 }
4154 
4155 #ifdef _LP64
4156 // scans rcx 4 byte words at [edi] for occurance of rax,
4157 // generic
4158 void Assembler::repne_scanl() { // repne_scan
4159   emit_int8((unsigned char)0xF2);
4160   // SCASL
4161   emit_int8((unsigned char)0xAF);
4162 }
4163 #endif
4164 
4165 void Assembler::ret(int imm16) {
4166   if (imm16 == 0) {
4167     emit_int8((unsigned char)0xC3);
4168   } else {
4169     emit_int8((unsigned char)0xC2);
4170     emit_int16(imm16);
4171   }
4172 }
4173 
4174 void Assembler::sahf() {
4175 #ifdef _LP64
4176   // Not supported in 64bit mode
4177   ShouldNotReachHere();
4178 #endif
4179   emit_int8((unsigned char)0x9E);
4180 }
4181 
4182 void Assembler::sarl(Register dst, int imm8) {
4183   int encode = prefix_and_encode(dst->encoding());
4184   assert(isShiftCount(imm8), "illegal shift count");
4185   if (imm8 == 1) {
4186     emit_int8((unsigned char)0xD1);
4187     emit_int8((unsigned char)(0xF8 | encode));
4188   } else {
4189     emit_int8((unsigned char)0xC1);
4190     emit_int8((unsigned char)(0xF8 | encode));
4191     emit_int8(imm8);
4192   }
4193 }
4194 
4195 void Assembler::sarl(Register dst) {
4196   int encode = prefix_and_encode(dst->encoding());
4197   emit_int8((unsigned char)0xD3);
4198   emit_int8((unsigned char)(0xF8 | encode));
4199 }
4200 
4201 void Assembler::sbbl(Address dst, int32_t imm32) {
4202   InstructionMark im(this);
4203   prefix(dst);
4204   emit_arith_operand(0x81, rbx, dst, imm32);
4205 }
4206 
4207 void Assembler::sbbl(Register dst, int32_t imm32) {
4208   prefix(dst);
4209   emit_arith(0x81, 0xD8, dst, imm32);
4210 }
4211 
4212 
4213 void Assembler::sbbl(Register dst, Address src) {
4214   InstructionMark im(this);
4215   prefix(src, dst);
4216   emit_int8(0x1B);
4217   emit_operand(dst, src);
4218 }
4219 
4220 void Assembler::sbbl(Register dst, Register src) {
4221   (void) prefix_and_encode(dst->encoding(), src->encoding());
4222   emit_arith(0x1B, 0xC0, dst, src);
4223 }
4224 
4225 void Assembler::setb(Condition cc, Register dst) {
4226   assert(0 <= cc && cc < 16, "illegal cc");
4227   int encode = prefix_and_encode(dst->encoding(), true);
4228   emit_int8(0x0F);
4229   emit_int8((unsigned char)0x90 | cc);
4230   emit_int8((unsigned char)(0xC0 | encode));
4231 }
4232 
4233 void Assembler::palignr(XMMRegister dst, XMMRegister src, int imm8) {
4234   assert(VM_Version::supports_ssse3(), "");
4235   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ false);
4236   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
4237   emit_int8((unsigned char)0x0F);
4238   emit_int8((unsigned char)(0xC0 | encode));
4239   emit_int8(imm8);
4240 }
4241 
4242 void Assembler::vpalignr(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8, int vector_len) {
4243   assert(vector_len == AVX_128bit? VM_Version::supports_avx() :
4244          vector_len == AVX_256bit? VM_Version::supports_avx2() :
4245          0, "");
4246   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
4247   int encode = simd_prefix_and_encode(dst, nds, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
4248   emit_int8((unsigned char)0x0F);
4249   emit_int8((unsigned char)(0xC0 | encode));
4250   emit_int8(imm8);
4251 }
4252 
4253 void Assembler::pblendw(XMMRegister dst, XMMRegister src, int imm8) {
4254   assert(VM_Version::supports_sse4_1(), "");
4255   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
4256   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
4257   emit_int8((unsigned char)0x0E);
4258   emit_int8((unsigned char)(0xC0 | encode));
4259   emit_int8(imm8);
4260 }
4261 
4262 void Assembler::sha1rnds4(XMMRegister dst, XMMRegister src, int imm8) {
4263   assert(VM_Version::supports_sha(), "");
4264   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
4265   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F_3A, &attributes);
4266   emit_int8((unsigned char)0xCC);
4267   emit_int8((unsigned char)(0xC0 | encode));
4268   emit_int8((unsigned char)imm8);
4269 }
4270 
4271 void Assembler::sha1nexte(XMMRegister dst, XMMRegister src) {
4272   assert(VM_Version::supports_sha(), "");
4273   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
4274   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
4275   emit_int8((unsigned char)0xC8);
4276   emit_int8((unsigned char)(0xC0 | encode));
4277 }
4278 
4279 void Assembler::sha1msg1(XMMRegister dst, XMMRegister src) {
4280   assert(VM_Version::supports_sha(), "");
4281   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
4282   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
4283   emit_int8((unsigned char)0xC9);
4284   emit_int8((unsigned char)(0xC0 | encode));
4285 }
4286 
4287 void Assembler::sha1msg2(XMMRegister dst, XMMRegister src) {
4288   assert(VM_Version::supports_sha(), "");
4289   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
4290   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
4291   emit_int8((unsigned char)0xCA);
4292   emit_int8((unsigned char)(0xC0 | encode));
4293 }
4294 
4295 // xmm0 is implicit additional source to this instruction.
4296 void Assembler::sha256rnds2(XMMRegister dst, XMMRegister src) {
4297   assert(VM_Version::supports_sha(), "");
4298   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
4299   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
4300   emit_int8((unsigned char)0xCB);
4301   emit_int8((unsigned char)(0xC0 | encode));
4302 }
4303 
4304 void Assembler::sha256msg1(XMMRegister dst, XMMRegister src) {
4305   assert(VM_Version::supports_sha(), "");
4306   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
4307   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
4308   emit_int8((unsigned char)0xCC);
4309   emit_int8((unsigned char)(0xC0 | encode));
4310 }
4311 
4312 void Assembler::sha256msg2(XMMRegister dst, XMMRegister src) {
4313   assert(VM_Version::supports_sha(), "");
4314   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
4315   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
4316   emit_int8((unsigned char)0xCD);
4317   emit_int8((unsigned char)(0xC0 | encode));
4318 }
4319 
4320 
4321 void Assembler::shll(Register dst, int imm8) {
4322   assert(isShiftCount(imm8), "illegal shift count");
4323   int encode = prefix_and_encode(dst->encoding());
4324   if (imm8 == 1 ) {
4325     emit_int8((unsigned char)0xD1);
4326     emit_int8((unsigned char)(0xE0 | encode));
4327   } else {
4328     emit_int8((unsigned char)0xC1);
4329     emit_int8((unsigned char)(0xE0 | encode));
4330     emit_int8(imm8);
4331   }
4332 }
4333 
4334 void Assembler::shll(Register dst) {
4335   int encode = prefix_and_encode(dst->encoding());
4336   emit_int8((unsigned char)0xD3);
4337   emit_int8((unsigned char)(0xE0 | encode));
4338 }
4339 
4340 void Assembler::shrl(Register dst, int imm8) {
4341   assert(isShiftCount(imm8), "illegal shift count");
4342   int encode = prefix_and_encode(dst->encoding());
4343   emit_int8((unsigned char)0xC1);
4344   emit_int8((unsigned char)(0xE8 | encode));
4345   emit_int8(imm8);
4346 }
4347 
4348 void Assembler::shrl(Register dst) {
4349   int encode = prefix_and_encode(dst->encoding());
4350   emit_int8((unsigned char)0xD3);
4351   emit_int8((unsigned char)(0xE8 | encode));
4352 }
4353 
4354 // copies a single word from [esi] to [edi]
4355 void Assembler::smovl() {
4356   emit_int8((unsigned char)0xA5);
4357 }
4358 
4359 void Assembler::sqrtsd(XMMRegister dst, XMMRegister src) {
4360   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4361   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
4362   attributes.set_rex_vex_w_reverted();
4363   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
4364   emit_int8(0x51);
4365   emit_int8((unsigned char)(0xC0 | encode));
4366 }
4367 
4368 void Assembler::sqrtsd(XMMRegister dst, Address src) {
4369   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4370   InstructionMark im(this);
4371   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
4372   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
4373   attributes.set_rex_vex_w_reverted();
4374   simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
4375   emit_int8(0x51);
4376   emit_operand(dst, src);
4377 }
4378 
4379 void Assembler::sqrtss(XMMRegister dst, XMMRegister src) {
4380   NOT_LP64(assert(VM_Version::supports_sse(), ""));
4381   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
4382   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
4383   emit_int8(0x51);
4384   emit_int8((unsigned char)(0xC0 | encode));
4385 }
4386 
4387 void Assembler::std() {
4388   emit_int8((unsigned char)0xFD);
4389 }
4390 
4391 void Assembler::sqrtss(XMMRegister dst, Address src) {
4392   NOT_LP64(assert(VM_Version::supports_sse(), ""));
4393   InstructionMark im(this);
4394   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
4395   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
4396   simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
4397   emit_int8(0x51);
4398   emit_operand(dst, src);
4399 }
4400 
4401 void Assembler::stmxcsr( Address dst) {
4402   NOT_LP64(assert(VM_Version::supports_sse(), ""));
4403   InstructionMark im(this);
4404   prefix(dst);
4405   emit_int8(0x0F);
4406   emit_int8((unsigned char)0xAE);
4407   emit_operand(as_Register(3), dst);
4408 }
4409 
4410 void Assembler::subl(Address dst, int32_t imm32) {
4411   InstructionMark im(this);
4412   prefix(dst);
4413   emit_arith_operand(0x81, rbp, dst, imm32);
4414 }
4415 
4416 void Assembler::subl(Address dst, Register src) {
4417   InstructionMark im(this);
4418   prefix(dst, src);
4419   emit_int8(0x29);
4420   emit_operand(src, dst);
4421 }
4422 
4423 void Assembler::subl(Register dst, int32_t imm32) {
4424   prefix(dst);
4425   emit_arith(0x81, 0xE8, dst, imm32);
4426 }
4427 
4428 // Force generation of a 4 byte immediate value even if it fits into 8bit
4429 void Assembler::subl_imm32(Register dst, int32_t imm32) {
4430   prefix(dst);
4431   emit_arith_imm32(0x81, 0xE8, dst, imm32);
4432 }
4433 
4434 void Assembler::subl(Register dst, Address src) {
4435   InstructionMark im(this);
4436   prefix(src, dst);
4437   emit_int8(0x2B);
4438   emit_operand(dst, src);
4439 }
4440 
4441 void Assembler::subl(Register dst, Register src) {
4442   (void) prefix_and_encode(dst->encoding(), src->encoding());
4443   emit_arith(0x2B, 0xC0, dst, src);
4444 }
4445 
4446 void Assembler::subsd(XMMRegister dst, XMMRegister src) {
4447   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4448   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
4449   attributes.set_rex_vex_w_reverted();
4450   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
4451   emit_int8(0x5C);
4452   emit_int8((unsigned char)(0xC0 | encode));
4453 }
4454 
4455 void Assembler::subsd(XMMRegister dst, Address src) {
4456   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4457   InstructionMark im(this);
4458   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
4459   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
4460   attributes.set_rex_vex_w_reverted();
4461   simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
4462   emit_int8(0x5C);
4463   emit_operand(dst, src);
4464 }
4465 
4466 void Assembler::subss(XMMRegister dst, XMMRegister src) {
4467   NOT_LP64(assert(VM_Version::supports_sse(), ""));
4468   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false , /* uses_vl */ false);
4469   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
4470   emit_int8(0x5C);
4471   emit_int8((unsigned char)(0xC0 | encode));
4472 }
4473 
4474 void Assembler::subss(XMMRegister dst, Address src) {
4475   NOT_LP64(assert(VM_Version::supports_sse(), ""));
4476   InstructionMark im(this);
4477   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
4478   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
4479   simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
4480   emit_int8(0x5C);
4481   emit_operand(dst, src);
4482 }
4483 
4484 void Assembler::testb(Register dst, int imm8) {
4485   NOT_LP64(assert(dst->has_byte_register(), "must have byte register"));
4486   (void) prefix_and_encode(dst->encoding(), true);
4487   emit_arith_b(0xF6, 0xC0, dst, imm8);
4488 }
4489 
4490 void Assembler::testb(Address dst, int imm8) {
4491   InstructionMark im(this);
4492   prefix(dst);
4493   emit_int8((unsigned char)0xF6);
4494   emit_operand(rax, dst, 1);
4495   emit_int8(imm8);
4496 }
4497 
4498 void Assembler::testl(Register dst, int32_t imm32) {
4499   // not using emit_arith because test
4500   // doesn't support sign-extension of
4501   // 8bit operands
4502   int encode = dst->encoding();
4503   if (encode == 0) {
4504     emit_int8((unsigned char)0xA9);
4505   } else {
4506     encode = prefix_and_encode(encode);
4507     emit_int8((unsigned char)0xF7);
4508     emit_int8((unsigned char)(0xC0 | encode));
4509   }
4510   emit_int32(imm32);
4511 }
4512 
4513 void Assembler::testl(Register dst, Register src) {
4514   (void) prefix_and_encode(dst->encoding(), src->encoding());
4515   emit_arith(0x85, 0xC0, dst, src);
4516 }
4517 
4518 void Assembler::testl(Register dst, Address src) {
4519   InstructionMark im(this);
4520   prefix(src, dst);
4521   emit_int8((unsigned char)0x85);
4522   emit_operand(dst, src);
4523 }
4524 
4525 void Assembler::tzcntl(Register dst, Register src) {
4526   assert(VM_Version::supports_bmi1(), "tzcnt instruction not supported");
4527   emit_int8((unsigned char)0xF3);
4528   int encode = prefix_and_encode(dst->encoding(), src->encoding());
4529   emit_int8(0x0F);
4530   emit_int8((unsigned char)0xBC);
4531   emit_int8((unsigned char)0xC0 | encode);
4532 }
4533 
4534 void Assembler::tzcntq(Register dst, Register src) {
4535   assert(VM_Version::supports_bmi1(), "tzcnt instruction not supported");
4536   emit_int8((unsigned char)0xF3);
4537   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
4538   emit_int8(0x0F);
4539   emit_int8((unsigned char)0xBC);
4540   emit_int8((unsigned char)(0xC0 | encode));
4541 }
4542 
4543 void Assembler::ucomisd(XMMRegister dst, Address src) {
4544   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4545   InstructionMark im(this);
4546   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4547   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
4548   attributes.set_rex_vex_w_reverted();
4549   simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4550   emit_int8(0x2E);
4551   emit_operand(dst, src);
4552 }
4553 
4554 void Assembler::ucomisd(XMMRegister dst, XMMRegister src) {
4555   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4556   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4557   attributes.set_rex_vex_w_reverted();
4558   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4559   emit_int8(0x2E);
4560   emit_int8((unsigned char)(0xC0 | encode));
4561 }
4562 
4563 void Assembler::ucomiss(XMMRegister dst, Address src) {
4564   NOT_LP64(assert(VM_Version::supports_sse(), ""));
4565   InstructionMark im(this);
4566   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4567   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
4568   simd_prefix(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
4569   emit_int8(0x2E);
4570   emit_operand(dst, src);
4571 }
4572 
4573 void Assembler::ucomiss(XMMRegister dst, XMMRegister src) {
4574   NOT_LP64(assert(VM_Version::supports_sse(), ""));
4575   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4576   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
4577   emit_int8(0x2E);
4578   emit_int8((unsigned char)(0xC0 | encode));
4579 }
4580 
4581 void Assembler::xabort(int8_t imm8) {
4582   emit_int8((unsigned char)0xC6);
4583   emit_int8((unsigned char)0xF8);
4584   emit_int8((unsigned char)(imm8 & 0xFF));
4585 }
4586 
4587 void Assembler::xaddb(Address dst, Register src) {
4588   InstructionMark im(this);
4589   prefix(dst, src, true);
4590   emit_int8(0x0F);
4591   emit_int8((unsigned char)0xC0);
4592   emit_operand(src, dst);
4593 }
4594 
4595 void Assembler::xaddw(Address dst, Register src) {
4596   InstructionMark im(this);
4597   emit_int8(0x66);
4598   prefix(dst, src);
4599   emit_int8(0x0F);
4600   emit_int8((unsigned char)0xC1);
4601   emit_operand(src, dst);
4602 }
4603 
4604 void Assembler::xaddl(Address dst, Register src) {
4605   InstructionMark im(this);
4606   prefix(dst, src);
4607   emit_int8(0x0F);
4608   emit_int8((unsigned char)0xC1);
4609   emit_operand(src, dst);
4610 }
4611 
4612 void Assembler::xbegin(Label& abort, relocInfo::relocType rtype) {
4613   InstructionMark im(this);
4614   relocate(rtype);
4615   if (abort.is_bound()) {
4616     address entry = target(abort);
4617     assert(entry != NULL, "abort entry NULL");
4618     intptr_t offset = entry - pc();
4619     emit_int8((unsigned char)0xC7);
4620     emit_int8((unsigned char)0xF8);
4621     emit_int32(offset - 6); // 2 opcode + 4 address
4622   } else {
4623     abort.add_patch_at(code(), locator());
4624     emit_int8((unsigned char)0xC7);
4625     emit_int8((unsigned char)0xF8);
4626     emit_int32(0);
4627   }
4628 }
4629 
4630 void Assembler::xchgb(Register dst, Address src) { // xchg
4631   InstructionMark im(this);
4632   prefix(src, dst, true);
4633   emit_int8((unsigned char)0x86);
4634   emit_operand(dst, src);
4635 }
4636 
4637 void Assembler::xchgw(Register dst, Address src) { // xchg
4638   InstructionMark im(this);
4639   emit_int8(0x66);
4640   prefix(src, dst);
4641   emit_int8((unsigned char)0x87);
4642   emit_operand(dst, src);
4643 }
4644 
4645 void Assembler::xchgl(Register dst, Address src) { // xchg
4646   InstructionMark im(this);
4647   prefix(src, dst);
4648   emit_int8((unsigned char)0x87);
4649   emit_operand(dst, src);
4650 }
4651 
4652 void Assembler::xchgl(Register dst, Register src) {
4653   int encode = prefix_and_encode(dst->encoding(), src->encoding());
4654   emit_int8((unsigned char)0x87);
4655   emit_int8((unsigned char)(0xC0 | encode));
4656 }
4657 
4658 void Assembler::xend() {
4659   emit_int8((unsigned char)0x0F);
4660   emit_int8((unsigned char)0x01);
4661   emit_int8((unsigned char)0xD5);
4662 }
4663 
4664 void Assembler::xgetbv() {
4665   emit_int8(0x0F);
4666   emit_int8(0x01);
4667   emit_int8((unsigned char)0xD0);
4668 }
4669 
4670 void Assembler::xorl(Register dst, int32_t imm32) {
4671   prefix(dst);
4672   emit_arith(0x81, 0xF0, dst, imm32);
4673 }
4674 
4675 void Assembler::xorl(Register dst, Address src) {
4676   InstructionMark im(this);
4677   prefix(src, dst);
4678   emit_int8(0x33);
4679   emit_operand(dst, src);
4680 }
4681 
4682 void Assembler::xorl(Register dst, Register src) {
4683   (void) prefix_and_encode(dst->encoding(), src->encoding());
4684   emit_arith(0x33, 0xC0, dst, src);
4685 }
4686 
4687 void Assembler::xorb(Register dst, Address src) {
4688   InstructionMark im(this);
4689   prefix(src, dst);
4690   emit_int8(0x32);
4691   emit_operand(dst, src);
4692 }
4693 
4694 // AVX 3-operands scalar float-point arithmetic instructions
4695 
4696 void Assembler::vaddsd(XMMRegister dst, XMMRegister nds, Address src) {
4697   assert(VM_Version::supports_avx(), "");
4698   InstructionMark im(this);
4699   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
4700   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
4701   attributes.set_rex_vex_w_reverted();
4702   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
4703   emit_int8(0x58);
4704   emit_operand(dst, src);
4705 }
4706 
4707 void Assembler::vaddsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
4708   assert(VM_Version::supports_avx(), "");
4709   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
4710   attributes.set_rex_vex_w_reverted();
4711   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
4712   emit_int8(0x58);
4713   emit_int8((unsigned char)(0xC0 | encode));
4714 }
4715 
4716 void Assembler::vaddss(XMMRegister dst, XMMRegister nds, Address src) {
4717   assert(VM_Version::supports_avx(), "");
4718   InstructionMark im(this);
4719   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
4720   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
4721   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
4722   emit_int8(0x58);
4723   emit_operand(dst, src);
4724 }
4725 
4726 void Assembler::vaddss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
4727   assert(VM_Version::supports_avx(), "");
4728   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
4729   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
4730   emit_int8(0x58);
4731   emit_int8((unsigned char)(0xC0 | encode));
4732 }
4733 
4734 void Assembler::vdivsd(XMMRegister dst, XMMRegister nds, Address src) {
4735   assert(VM_Version::supports_avx(), "");
4736   InstructionMark im(this);
4737   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
4738   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
4739   attributes.set_rex_vex_w_reverted();
4740   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
4741   emit_int8(0x5E);
4742   emit_operand(dst, src);
4743 }
4744 
4745 void Assembler::vdivsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
4746   assert(VM_Version::supports_avx(), "");
4747   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
4748   attributes.set_rex_vex_w_reverted();
4749   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
4750   emit_int8(0x5E);
4751   emit_int8((unsigned char)(0xC0 | encode));
4752 }
4753 
4754 void Assembler::vdivss(XMMRegister dst, XMMRegister nds, Address src) {
4755   assert(VM_Version::supports_avx(), "");
4756   InstructionMark im(this);
4757   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
4758   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
4759   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
4760   emit_int8(0x5E);
4761   emit_operand(dst, src);
4762 }
4763 
4764 void Assembler::vdivss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
4765   assert(VM_Version::supports_avx(), "");
4766   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
4767   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
4768   emit_int8(0x5E);
4769   emit_int8((unsigned char)(0xC0 | encode));
4770 }
4771 
4772 void Assembler::vfmadd231sd(XMMRegister dst, XMMRegister src1, XMMRegister src2) {
4773   assert(VM_Version::supports_fma(), "");
4774   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
4775   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4776   emit_int8((unsigned char)0xB9);
4777   emit_int8((unsigned char)(0xC0 | encode));
4778 }
4779 
4780 void Assembler::vfmadd231ss(XMMRegister dst, XMMRegister src1, XMMRegister src2) {
4781   assert(VM_Version::supports_fma(), "");
4782   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
4783   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4784   emit_int8((unsigned char)0xB9);
4785   emit_int8((unsigned char)(0xC0 | encode));
4786 }
4787 
4788 void Assembler::vmulsd(XMMRegister dst, XMMRegister nds, Address src) {
4789   assert(VM_Version::supports_avx(), "");
4790   InstructionMark im(this);
4791   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
4792   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
4793   attributes.set_rex_vex_w_reverted();
4794   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
4795   emit_int8(0x59);
4796   emit_operand(dst, src);
4797 }
4798 
4799 void Assembler::vmulsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
4800   assert(VM_Version::supports_avx(), "");
4801   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
4802   attributes.set_rex_vex_w_reverted();
4803   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
4804   emit_int8(0x59);
4805   emit_int8((unsigned char)(0xC0 | encode));
4806 }
4807 
4808 void Assembler::vmulss(XMMRegister dst, XMMRegister nds, Address src) {
4809   assert(VM_Version::supports_avx(), "");
4810   InstructionMark im(this);
4811   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
4812   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
4813   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
4814   emit_int8(0x59);
4815   emit_operand(dst, src);
4816 }
4817 
4818 void Assembler::vmulss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
4819   assert(VM_Version::supports_avx(), "");
4820   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
4821   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
4822   emit_int8(0x59);
4823   emit_int8((unsigned char)(0xC0 | encode));
4824 }
4825 
4826 void Assembler::vsubsd(XMMRegister dst, XMMRegister nds, Address src) {
4827   assert(VM_Version::supports_avx(), "");
4828   InstructionMark im(this);
4829   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
4830   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
4831   attributes.set_rex_vex_w_reverted();
4832   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
4833   emit_int8(0x5C);
4834   emit_operand(dst, src);
4835 }
4836 
4837 void Assembler::vsubsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
4838   assert(VM_Version::supports_avx(), "");
4839   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
4840   attributes.set_rex_vex_w_reverted();
4841   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
4842   emit_int8(0x5C);
4843   emit_int8((unsigned char)(0xC0 | encode));
4844 }
4845 
4846 void Assembler::vsubss(XMMRegister dst, XMMRegister nds, Address src) {
4847   assert(VM_Version::supports_avx(), "");
4848   InstructionMark im(this);
4849   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
4850   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
4851   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
4852   emit_int8(0x5C);
4853   emit_operand(dst, src);
4854 }
4855 
4856 void Assembler::vsubss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
4857   assert(VM_Version::supports_avx(), "");
4858   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
4859   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
4860   emit_int8(0x5C);
4861   emit_int8((unsigned char)(0xC0 | encode));
4862 }
4863 
4864 //====================VECTOR ARITHMETIC=====================================
4865 
4866 // Float-point vector arithmetic
4867 
4868 void Assembler::addpd(XMMRegister dst, XMMRegister src) {
4869   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4870   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4871   attributes.set_rex_vex_w_reverted();
4872   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4873   emit_int8(0x58);
4874   emit_int8((unsigned char)(0xC0 | encode));
4875 }
4876 
4877 void Assembler::addpd(XMMRegister dst, Address src) {
4878   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4879   InstructionMark im(this);
4880   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4881   attributes.set_rex_vex_w_reverted();
4882   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
4883   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4884   emit_int8(0x58);
4885   emit_operand(dst, src);
4886 }
4887 
4888 
4889 void Assembler::addps(XMMRegister dst, XMMRegister src) {
4890   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4891   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4892   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
4893   emit_int8(0x58);
4894   emit_int8((unsigned char)(0xC0 | encode));
4895 }
4896 
4897 void Assembler::vaddpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4898   assert(VM_Version::supports_avx(), "");
4899   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4900   attributes.set_rex_vex_w_reverted();
4901   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4902   emit_int8(0x58);
4903   emit_int8((unsigned char)(0xC0 | encode));
4904 }
4905 
4906 void Assembler::vaddps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4907   assert(VM_Version::supports_avx(), "");
4908   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4909   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
4910   emit_int8(0x58);
4911   emit_int8((unsigned char)(0xC0 | encode));
4912 }
4913 
4914 void Assembler::vaddpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4915   assert(VM_Version::supports_avx(), "");
4916   InstructionMark im(this);
4917   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4918   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
4919   attributes.set_rex_vex_w_reverted();
4920   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4921   emit_int8(0x58);
4922   emit_operand(dst, src);
4923 }
4924 
4925 void Assembler::vaddps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4926   assert(VM_Version::supports_avx(), "");
4927   InstructionMark im(this);
4928   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4929   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
4930   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
4931   emit_int8(0x58);
4932   emit_operand(dst, src);
4933 }
4934 
4935 void Assembler::subpd(XMMRegister dst, XMMRegister src) {
4936   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4937   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4938   attributes.set_rex_vex_w_reverted();
4939   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4940   emit_int8(0x5C);
4941   emit_int8((unsigned char)(0xC0 | encode));
4942 }
4943 
4944 void Assembler::subps(XMMRegister dst, XMMRegister src) {
4945   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4946   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4947   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
4948   emit_int8(0x5C);
4949   emit_int8((unsigned char)(0xC0 | encode));
4950 }
4951 
4952 void Assembler::vsubpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4953   assert(VM_Version::supports_avx(), "");
4954   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4955   attributes.set_rex_vex_w_reverted();
4956   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4957   emit_int8(0x5C);
4958   emit_int8((unsigned char)(0xC0 | encode));
4959 }
4960 
4961 void Assembler::vsubps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4962   assert(VM_Version::supports_avx(), "");
4963   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4964   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
4965   emit_int8(0x5C);
4966   emit_int8((unsigned char)(0xC0 | encode));
4967 }
4968 
4969 void Assembler::vsubpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4970   assert(VM_Version::supports_avx(), "");
4971   InstructionMark im(this);
4972   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4973   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
4974   attributes.set_rex_vex_w_reverted();
4975   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4976   emit_int8(0x5C);
4977   emit_operand(dst, src);
4978 }
4979 
4980 void Assembler::vsubps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4981   assert(VM_Version::supports_avx(), "");
4982   InstructionMark im(this);
4983   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4984   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
4985   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
4986   emit_int8(0x5C);
4987   emit_operand(dst, src);
4988 }
4989 
4990 void Assembler::mulpd(XMMRegister dst, XMMRegister src) {
4991   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4992   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4993   attributes.set_rex_vex_w_reverted();
4994   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4995   emit_int8(0x59);
4996   emit_int8((unsigned char)(0xC0 | encode));
4997 }
4998 
4999 void Assembler::mulpd(XMMRegister dst, Address src) {
5000   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5001   InstructionMark im(this);
5002   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5003   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
5004   attributes.set_rex_vex_w_reverted();
5005   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5006   emit_int8(0x59);
5007   emit_operand(dst, src);
5008 }
5009 
5010 void Assembler::mulps(XMMRegister dst, XMMRegister src) {
5011   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5012   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5013   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
5014   emit_int8(0x59);
5015   emit_int8((unsigned char)(0xC0 | encode));
5016 }
5017 
5018 void Assembler::vmulpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5019   assert(VM_Version::supports_avx(), "");
5020   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5021   attributes.set_rex_vex_w_reverted();
5022   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5023   emit_int8(0x59);
5024   emit_int8((unsigned char)(0xC0 | encode));
5025 }
5026 
5027 void Assembler::vmulps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5028   assert(VM_Version::supports_avx(), "");
5029   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5030   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
5031   emit_int8(0x59);
5032   emit_int8((unsigned char)(0xC0 | encode));
5033 }
5034 
5035 void Assembler::vmulpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5036   assert(VM_Version::supports_avx(), "");
5037   InstructionMark im(this);
5038   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5039   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
5040   attributes.set_rex_vex_w_reverted();
5041   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5042   emit_int8(0x59);
5043   emit_operand(dst, src);
5044 }
5045 
5046 void Assembler::vmulps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5047   assert(VM_Version::supports_avx(), "");
5048   InstructionMark im(this);
5049   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5050   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
5051   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
5052   emit_int8(0x59);
5053   emit_operand(dst, src);
5054 }
5055 
5056 void Assembler::divpd(XMMRegister dst, XMMRegister src) {
5057   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5058   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5059   attributes.set_rex_vex_w_reverted();
5060   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5061   emit_int8(0x5E);
5062   emit_int8((unsigned char)(0xC0 | encode));
5063 }
5064 
5065 void Assembler::divps(XMMRegister dst, XMMRegister src) {
5066   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5067   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5068   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
5069   emit_int8(0x5E);
5070   emit_int8((unsigned char)(0xC0 | encode));
5071 }
5072 
5073 void Assembler::vdivpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5074   assert(VM_Version::supports_avx(), "");
5075   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5076   attributes.set_rex_vex_w_reverted();
5077   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5078   emit_int8(0x5E);
5079   emit_int8((unsigned char)(0xC0 | encode));
5080 }
5081 
5082 void Assembler::vdivps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5083   assert(VM_Version::supports_avx(), "");
5084   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5085   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
5086   emit_int8(0x5E);
5087   emit_int8((unsigned char)(0xC0 | encode));
5088 }
5089 
5090 void Assembler::vdivpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5091   assert(VM_Version::supports_avx(), "");
5092   InstructionMark im(this);
5093   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5094   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
5095   attributes.set_rex_vex_w_reverted();
5096   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5097   emit_int8(0x5E);
5098   emit_operand(dst, src);
5099 }
5100 
5101 void Assembler::vdivps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5102   assert(VM_Version::supports_avx(), "");
5103   InstructionMark im(this);
5104   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5105   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
5106   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
5107   emit_int8(0x5E);
5108   emit_operand(dst, src);
5109 }
5110 
5111 void Assembler::vsqrtpd(XMMRegister dst, XMMRegister src, int vector_len) {
5112   assert(VM_Version::supports_avx(), "");
5113   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5114   attributes.set_rex_vex_w_reverted();
5115   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5116   emit_int8(0x51);
5117   emit_int8((unsigned char)(0xC0 | encode));
5118 }
5119 
5120 void Assembler::vsqrtpd(XMMRegister dst, Address src, int vector_len) {
5121   assert(VM_Version::supports_avx(), "");
5122   InstructionMark im(this);
5123   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5124   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
5125   attributes.set_rex_vex_w_reverted();
5126   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5127   emit_int8(0x51);
5128   emit_operand(dst, src);
5129 }
5130 
5131 void Assembler::andpd(XMMRegister dst, XMMRegister src) {
5132   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5133   InstructionAttr attributes(AVX_128bit, /* rex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
5134   attributes.set_rex_vex_w_reverted();
5135   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5136   emit_int8(0x54);
5137   emit_int8((unsigned char)(0xC0 | encode));
5138 }
5139 
5140 void Assembler::andps(XMMRegister dst, XMMRegister src) {
5141   NOT_LP64(assert(VM_Version::supports_sse(), ""));
5142   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
5143   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
5144   emit_int8(0x54);
5145   emit_int8((unsigned char)(0xC0 | encode));
5146 }
5147 
5148 void Assembler::andps(XMMRegister dst, Address src) {
5149   NOT_LP64(assert(VM_Version::supports_sse(), ""));
5150   InstructionMark im(this);
5151   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
5152   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
5153   simd_prefix(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
5154   emit_int8(0x54);
5155   emit_operand(dst, src);
5156 }
5157 
5158 void Assembler::andpd(XMMRegister dst, Address src) {
5159   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5160   InstructionMark im(this);
5161   InstructionAttr attributes(AVX_128bit, /* rex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
5162   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
5163   attributes.set_rex_vex_w_reverted();
5164   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5165   emit_int8(0x54);
5166   emit_operand(dst, src);
5167 }
5168 
5169 void Assembler::vandpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5170   assert(VM_Version::supports_avx(), "");
5171   InstructionAttr attributes(vector_len, /* vex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
5172   attributes.set_rex_vex_w_reverted();
5173   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5174   emit_int8(0x54);
5175   emit_int8((unsigned char)(0xC0 | encode));
5176 }
5177 
5178 void Assembler::vandps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5179   assert(VM_Version::supports_avx(), "");
5180   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
5181   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
5182   emit_int8(0x54);
5183   emit_int8((unsigned char)(0xC0 | encode));
5184 }
5185 
5186 void Assembler::vandpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5187   assert(VM_Version::supports_avx(), "");
5188   InstructionMark im(this);
5189   InstructionAttr attributes(vector_len, /* vex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
5190   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
5191   attributes.set_rex_vex_w_reverted();
5192   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5193   emit_int8(0x54);
5194   emit_operand(dst, src);
5195 }
5196 
5197 void Assembler::vandps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5198   assert(VM_Version::supports_avx(), "");
5199   InstructionMark im(this);
5200   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
5201   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
5202   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
5203   emit_int8(0x54);
5204   emit_operand(dst, src);
5205 }
5206 
5207 void Assembler::unpckhpd(XMMRegister dst, XMMRegister src) {
5208   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5209   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5210   attributes.set_rex_vex_w_reverted();
5211   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5212   emit_int8(0x15);
5213   emit_int8((unsigned char)(0xC0 | encode));
5214 }
5215 
5216 void Assembler::unpcklpd(XMMRegister dst, XMMRegister src) {
5217   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5218   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5219   attributes.set_rex_vex_w_reverted();
5220   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5221   emit_int8(0x14);
5222   emit_int8((unsigned char)(0xC0 | encode));
5223 }
5224 
5225 void Assembler::xorpd(XMMRegister dst, XMMRegister src) {
5226   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5227   InstructionAttr attributes(AVX_128bit, /* rex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
5228   attributes.set_rex_vex_w_reverted();
5229   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5230   emit_int8(0x57);
5231   emit_int8((unsigned char)(0xC0 | encode));
5232 }
5233 
5234 void Assembler::xorps(XMMRegister dst, XMMRegister src) {
5235   NOT_LP64(assert(VM_Version::supports_sse(), ""));
5236   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
5237   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
5238   emit_int8(0x57);
5239   emit_int8((unsigned char)(0xC0 | encode));
5240 }
5241 
5242 void Assembler::xorpd(XMMRegister dst, Address src) {
5243   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5244   InstructionMark im(this);
5245   InstructionAttr attributes(AVX_128bit, /* rex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
5246   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
5247   attributes.set_rex_vex_w_reverted();
5248   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5249   emit_int8(0x57);
5250   emit_operand(dst, src);
5251 }
5252 
5253 void Assembler::xorps(XMMRegister dst, Address src) {
5254   NOT_LP64(assert(VM_Version::supports_sse(), ""));
5255   InstructionMark im(this);
5256   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
5257   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
5258   simd_prefix(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
5259   emit_int8(0x57);
5260   emit_operand(dst, src);
5261 }
5262 
5263 void Assembler::vxorpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5264   assert(VM_Version::supports_avx(), "");
5265   InstructionAttr attributes(vector_len, /* vex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
5266   attributes.set_rex_vex_w_reverted();
5267   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5268   emit_int8(0x57);
5269   emit_int8((unsigned char)(0xC0 | encode));
5270 }
5271 
5272 void Assembler::vxorps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5273   assert(VM_Version::supports_avx(), "");
5274   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
5275   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
5276   emit_int8(0x57);
5277   emit_int8((unsigned char)(0xC0 | encode));
5278 }
5279 
5280 void Assembler::vxorpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5281   assert(VM_Version::supports_avx(), "");
5282   InstructionMark im(this);
5283   InstructionAttr attributes(vector_len, /* vex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
5284   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
5285   attributes.set_rex_vex_w_reverted();
5286   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5287   emit_int8(0x57);
5288   emit_operand(dst, src);
5289 }
5290 
5291 void Assembler::vxorps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5292   assert(VM_Version::supports_avx(), "");
5293   InstructionMark im(this);
5294   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
5295   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
5296   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
5297   emit_int8(0x57);
5298   emit_operand(dst, src);
5299 }
5300 
5301 // Integer vector arithmetic
5302 void Assembler::vphaddw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5303   assert(VM_Version::supports_avx() && (vector_len == 0) ||
5304          VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
5305   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
5306   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5307   emit_int8(0x01);
5308   emit_int8((unsigned char)(0xC0 | encode));
5309 }
5310 
5311 void Assembler::vphaddd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5312   assert(VM_Version::supports_avx() && (vector_len == 0) ||
5313          VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
5314   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
5315   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5316   emit_int8(0x02);
5317   emit_int8((unsigned char)(0xC0 | encode));
5318 }
5319 
5320 void Assembler::paddb(XMMRegister dst, XMMRegister src) {
5321   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5322   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5323   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5324   emit_int8((unsigned char)0xFC);
5325   emit_int8((unsigned char)(0xC0 | encode));
5326 }
5327 
5328 void Assembler::paddw(XMMRegister dst, XMMRegister src) {
5329   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5330   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5331   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5332   emit_int8((unsigned char)0xFD);
5333   emit_int8((unsigned char)(0xC0 | encode));
5334 }
5335 
5336 void Assembler::paddd(XMMRegister dst, XMMRegister src) {
5337   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5338   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5339   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5340   emit_int8((unsigned char)0xFE);
5341   emit_int8((unsigned char)(0xC0 | encode));
5342 }
5343 
5344 void Assembler::paddd(XMMRegister dst, Address src) {
5345   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5346   InstructionMark im(this);
5347   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5348   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5349   emit_int8((unsigned char)0xFE);
5350   emit_operand(dst, src);
5351 }
5352 
5353 void Assembler::paddq(XMMRegister dst, XMMRegister src) {
5354   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5355   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5356   attributes.set_rex_vex_w_reverted();
5357   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5358   emit_int8((unsigned char)0xD4);
5359   emit_int8((unsigned char)(0xC0 | encode));
5360 }
5361 
5362 void Assembler::phaddw(XMMRegister dst, XMMRegister src) {
5363   assert(VM_Version::supports_sse3(), "");
5364   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
5365   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5366   emit_int8(0x01);
5367   emit_int8((unsigned char)(0xC0 | encode));
5368 }
5369 
5370 void Assembler::phaddd(XMMRegister dst, XMMRegister src) {
5371   assert(VM_Version::supports_sse3(), "");
5372   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
5373   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5374   emit_int8(0x02);
5375   emit_int8((unsigned char)(0xC0 | encode));
5376 }
5377 
5378 void Assembler::vpaddb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5379   assert(UseAVX > 0, "requires some form of AVX");
5380   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5381   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5382   emit_int8((unsigned char)0xFC);
5383   emit_int8((unsigned char)(0xC0 | encode));
5384 }
5385 
5386 void Assembler::vpaddw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5387   assert(UseAVX > 0, "requires some form of AVX");
5388   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5389   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5390   emit_int8((unsigned char)0xFD);
5391   emit_int8((unsigned char)(0xC0 | encode));
5392 }
5393 
5394 void Assembler::vpaddd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5395   assert(UseAVX > 0, "requires some form of AVX");
5396   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5397   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5398   emit_int8((unsigned char)0xFE);
5399   emit_int8((unsigned char)(0xC0 | encode));
5400 }
5401 
5402 void Assembler::vpaddq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5403   assert(UseAVX > 0, "requires some form of AVX");
5404   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5405   attributes.set_rex_vex_w_reverted();
5406   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5407   emit_int8((unsigned char)0xD4);
5408   emit_int8((unsigned char)(0xC0 | encode));
5409 }
5410 
5411 void Assembler::vpaddb(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5412   assert(UseAVX > 0, "requires some form of AVX");
5413   InstructionMark im(this);
5414   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5415   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
5416   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5417   emit_int8((unsigned char)0xFC);
5418   emit_operand(dst, src);
5419 }
5420 
5421 void Assembler::vpaddw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5422   assert(UseAVX > 0, "requires some form of AVX");
5423   InstructionMark im(this);
5424   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5425   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
5426   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5427   emit_int8((unsigned char)0xFD);
5428   emit_operand(dst, src);
5429 }
5430 
5431 void Assembler::vpaddd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5432   assert(UseAVX > 0, "requires some form of AVX");
5433   InstructionMark im(this);
5434   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5435   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
5436   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5437   emit_int8((unsigned char)0xFE);
5438   emit_operand(dst, src);
5439 }
5440 
5441 void Assembler::vpaddq(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5442   assert(UseAVX > 0, "requires some form of AVX");
5443   InstructionMark im(this);
5444   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5445   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
5446   attributes.set_rex_vex_w_reverted();
5447   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5448   emit_int8((unsigned char)0xD4);
5449   emit_operand(dst, src);
5450 }
5451 
5452 void Assembler::psubb(XMMRegister dst, XMMRegister src) {
5453   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5454   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5455   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5456   emit_int8((unsigned char)0xF8);
5457   emit_int8((unsigned char)(0xC0 | encode));
5458 }
5459 
5460 void Assembler::psubw(XMMRegister dst, XMMRegister src) {
5461   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5462   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5463   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5464   emit_int8((unsigned char)0xF9);
5465   emit_int8((unsigned char)(0xC0 | encode));
5466 }
5467 
5468 void Assembler::psubd(XMMRegister dst, XMMRegister src) {
5469   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5470   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5471   emit_int8((unsigned char)0xFA);
5472   emit_int8((unsigned char)(0xC0 | encode));
5473 }
5474 
5475 void Assembler::psubq(XMMRegister dst, XMMRegister src) {
5476   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5477   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5478   attributes.set_rex_vex_w_reverted();
5479   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5480   emit_int8((unsigned char)0xFB);
5481   emit_int8((unsigned char)(0xC0 | encode));
5482 }
5483 
5484 void Assembler::vpsubb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5485   assert(UseAVX > 0, "requires some form of AVX");
5486   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5487   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5488   emit_int8((unsigned char)0xF8);
5489   emit_int8((unsigned char)(0xC0 | encode));
5490 }
5491 
5492 void Assembler::vpsubw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5493   assert(UseAVX > 0, "requires some form of AVX");
5494   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5495   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5496   emit_int8((unsigned char)0xF9);
5497   emit_int8((unsigned char)(0xC0 | encode));
5498 }
5499 
5500 void Assembler::vpsubd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5501   assert(UseAVX > 0, "requires some form of AVX");
5502   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5503   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5504   emit_int8((unsigned char)0xFA);
5505   emit_int8((unsigned char)(0xC0 | encode));
5506 }
5507 
5508 void Assembler::vpsubq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5509   assert(UseAVX > 0, "requires some form of AVX");
5510   InstructionAttr attributes(vector_len, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5511   attributes.set_rex_vex_w_reverted();
5512   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5513   emit_int8((unsigned char)0xFB);
5514   emit_int8((unsigned char)(0xC0 | encode));
5515 }
5516 
5517 void Assembler::vpsubb(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5518   assert(UseAVX > 0, "requires some form of AVX");
5519   InstructionMark im(this);
5520   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5521   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
5522   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5523   emit_int8((unsigned char)0xF8);
5524   emit_operand(dst, src);
5525 }
5526 
5527 void Assembler::vpsubw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5528   assert(UseAVX > 0, "requires some form of AVX");
5529   InstructionMark im(this);
5530   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5531   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
5532   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5533   emit_int8((unsigned char)0xF9);
5534   emit_operand(dst, src);
5535 }
5536 
5537 void Assembler::vpsubd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5538   assert(UseAVX > 0, "requires some form of AVX");
5539   InstructionMark im(this);
5540   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5541   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
5542   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5543   emit_int8((unsigned char)0xFA);
5544   emit_operand(dst, src);
5545 }
5546 
5547 void Assembler::vpsubq(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5548   assert(UseAVX > 0, "requires some form of AVX");
5549   InstructionMark im(this);
5550   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5551   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
5552   attributes.set_rex_vex_w_reverted();
5553   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5554   emit_int8((unsigned char)0xFB);
5555   emit_operand(dst, src);
5556 }
5557 
5558 void Assembler::pmullw(XMMRegister dst, XMMRegister src) {
5559   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5560   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5561   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5562   emit_int8((unsigned char)0xD5);
5563   emit_int8((unsigned char)(0xC0 | encode));
5564 }
5565 
5566 void Assembler::pmulld(XMMRegister dst, XMMRegister src) {
5567   assert(VM_Version::supports_sse4_1(), "");
5568   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5569   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5570   emit_int8(0x40);
5571   emit_int8((unsigned char)(0xC0 | encode));
5572 }
5573 
5574 void Assembler::vpmullw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5575   assert(UseAVX > 0, "requires some form of AVX");
5576   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5577   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5578   emit_int8((unsigned char)0xD5);
5579   emit_int8((unsigned char)(0xC0 | encode));
5580 }
5581 
5582 void Assembler::vpmulld(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5583   assert(UseAVX > 0, "requires some form of AVX");
5584   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5585   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5586   emit_int8(0x40);
5587   emit_int8((unsigned char)(0xC0 | encode));
5588 }
5589 
5590 void Assembler::vpmullq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5591   assert(UseAVX > 2, "requires some form of EVEX");
5592   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
5593   attributes.set_is_evex_instruction();
5594   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5595   emit_int8(0x40);
5596   emit_int8((unsigned char)(0xC0 | encode));
5597 }
5598 
5599 void Assembler::vpmullw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5600   assert(UseAVX > 0, "requires some form of AVX");
5601   InstructionMark im(this);
5602   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5603   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
5604   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5605   emit_int8((unsigned char)0xD5);
5606   emit_operand(dst, src);
5607 }
5608 
5609 void Assembler::vpmulld(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5610   assert(UseAVX > 0, "requires some form of AVX");
5611   InstructionMark im(this);
5612   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5613   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
5614   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5615   emit_int8(0x40);
5616   emit_operand(dst, src);
5617 }
5618 
5619 void Assembler::vpmullq(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5620   assert(UseAVX > 2, "requires some form of EVEX");
5621   InstructionMark im(this);
5622   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
5623   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
5624   attributes.set_is_evex_instruction();
5625   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5626   emit_int8(0x40);
5627   emit_operand(dst, src);
5628 }
5629 
5630 // Shift packed integers left by specified number of bits.
5631 void Assembler::psllw(XMMRegister dst, int shift) {
5632   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5633   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5634   // XMM6 is for /6 encoding: 66 0F 71 /6 ib
5635   int encode = simd_prefix_and_encode(xmm6, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5636   emit_int8(0x71);
5637   emit_int8((unsigned char)(0xC0 | encode));
5638   emit_int8(shift & 0xFF);
5639 }
5640 
5641 void Assembler::pslld(XMMRegister dst, int shift) {
5642   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5643   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5644   // XMM6 is for /6 encoding: 66 0F 72 /6 ib
5645   int encode = simd_prefix_and_encode(xmm6, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5646   emit_int8(0x72);
5647   emit_int8((unsigned char)(0xC0 | encode));
5648   emit_int8(shift & 0xFF);
5649 }
5650 
5651 void Assembler::psllq(XMMRegister dst, int shift) {
5652   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5653   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5654   // XMM6 is for /6 encoding: 66 0F 73 /6 ib
5655   int encode = simd_prefix_and_encode(xmm6, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5656   emit_int8(0x73);
5657   emit_int8((unsigned char)(0xC0 | encode));
5658   emit_int8(shift & 0xFF);
5659 }
5660 
5661 void Assembler::psllw(XMMRegister dst, XMMRegister shift) {
5662   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5663   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5664   int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5665   emit_int8((unsigned char)0xF1);
5666   emit_int8((unsigned char)(0xC0 | encode));
5667 }
5668 
5669 void Assembler::pslld(XMMRegister dst, XMMRegister shift) {
5670   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5671   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5672   int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5673   emit_int8((unsigned char)0xF2);
5674   emit_int8((unsigned char)(0xC0 | encode));
5675 }
5676 
5677 void Assembler::psllq(XMMRegister dst, XMMRegister shift) {
5678   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5679   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5680   attributes.set_rex_vex_w_reverted();
5681   int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5682   emit_int8((unsigned char)0xF3);
5683   emit_int8((unsigned char)(0xC0 | encode));
5684 }
5685 
5686 void Assembler::vpsllw(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
5687   assert(UseAVX > 0, "requires some form of AVX");
5688   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5689   // XMM6 is for /6 encoding: 66 0F 71 /6 ib
5690   int encode = vex_prefix_and_encode(xmm6->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5691   emit_int8(0x71);
5692   emit_int8((unsigned char)(0xC0 | encode));
5693   emit_int8(shift & 0xFF);
5694 }
5695 
5696 void Assembler::vpslld(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
5697   assert(UseAVX > 0, "requires some form of AVX");
5698   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5699   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5700   // XMM6 is for /6 encoding: 66 0F 72 /6 ib
5701   int encode = vex_prefix_and_encode(xmm6->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5702   emit_int8(0x72);
5703   emit_int8((unsigned char)(0xC0 | encode));
5704   emit_int8(shift & 0xFF);
5705 }
5706 
5707 void Assembler::vpsllq(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
5708   assert(UseAVX > 0, "requires some form of AVX");
5709   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5710   attributes.set_rex_vex_w_reverted();
5711   // XMM6 is for /6 encoding: 66 0F 73 /6 ib
5712   int encode = vex_prefix_and_encode(xmm6->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5713   emit_int8(0x73);
5714   emit_int8((unsigned char)(0xC0 | encode));
5715   emit_int8(shift & 0xFF);
5716 }
5717 
5718 void Assembler::vpsllw(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
5719   assert(UseAVX > 0, "requires some form of AVX");
5720   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5721   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5722   emit_int8((unsigned char)0xF1);
5723   emit_int8((unsigned char)(0xC0 | encode));
5724 }
5725 
5726 void Assembler::vpslld(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
5727   assert(UseAVX > 0, "requires some form of AVX");
5728   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5729   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5730   emit_int8((unsigned char)0xF2);
5731   emit_int8((unsigned char)(0xC0 | encode));
5732 }
5733 
5734 void Assembler::vpsllq(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
5735   assert(UseAVX > 0, "requires some form of AVX");
5736   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5737   attributes.set_rex_vex_w_reverted();
5738   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5739   emit_int8((unsigned char)0xF3);
5740   emit_int8((unsigned char)(0xC0 | encode));
5741 }
5742 
5743 // Shift packed integers logically right by specified number of bits.
5744 void Assembler::psrlw(XMMRegister dst, int shift) {
5745   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5746   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5747   // XMM2 is for /2 encoding: 66 0F 71 /2 ib
5748   int encode = simd_prefix_and_encode(xmm2, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5749   emit_int8(0x71);
5750   emit_int8((unsigned char)(0xC0 | encode));
5751   emit_int8(shift & 0xFF);
5752 }
5753 
5754 void Assembler::psrld(XMMRegister dst, int shift) {
5755   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5756   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5757   // XMM2 is for /2 encoding: 66 0F 72 /2 ib
5758   int encode = simd_prefix_and_encode(xmm2, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5759   emit_int8(0x72);
5760   emit_int8((unsigned char)(0xC0 | encode));
5761   emit_int8(shift & 0xFF);
5762 }
5763 
5764 void Assembler::psrlq(XMMRegister dst, int shift) {
5765   // Do not confuse it with psrldq SSE2 instruction which
5766   // shifts 128 bit value in xmm register by number of bytes.
5767   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5768   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5769   attributes.set_rex_vex_w_reverted();
5770   // XMM2 is for /2 encoding: 66 0F 73 /2 ib
5771   int encode = simd_prefix_and_encode(xmm2, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5772   emit_int8(0x73);
5773   emit_int8((unsigned char)(0xC0 | encode));
5774   emit_int8(shift & 0xFF);
5775 }
5776 
5777 void Assembler::psrlw(XMMRegister dst, XMMRegister shift) {
5778   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5779   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5780   int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5781   emit_int8((unsigned char)0xD1);
5782   emit_int8((unsigned char)(0xC0 | encode));
5783 }
5784 
5785 void Assembler::psrld(XMMRegister dst, XMMRegister shift) {
5786   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5787   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5788   int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5789   emit_int8((unsigned char)0xD2);
5790   emit_int8((unsigned char)(0xC0 | encode));
5791 }
5792 
5793 void Assembler::psrlq(XMMRegister dst, XMMRegister shift) {
5794   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5795   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5796   attributes.set_rex_vex_w_reverted();
5797   int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5798   emit_int8((unsigned char)0xD3);
5799   emit_int8((unsigned char)(0xC0 | encode));
5800 }
5801 
5802 void Assembler::vpsrlw(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
5803   assert(UseAVX > 0, "requires some form of AVX");
5804   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5805   // XMM2 is for /2 encoding: 66 0F 71 /2 ib
5806   int encode = vex_prefix_and_encode(xmm2->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5807   emit_int8(0x71);
5808   emit_int8((unsigned char)(0xC0 | encode));
5809   emit_int8(shift & 0xFF);
5810 }
5811 
5812 void Assembler::vpsrld(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
5813   assert(UseAVX > 0, "requires some form of AVX");
5814   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5815   // XMM2 is for /2 encoding: 66 0F 72 /2 ib
5816   int encode = vex_prefix_and_encode(xmm2->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5817   emit_int8(0x72);
5818   emit_int8((unsigned char)(0xC0 | encode));
5819   emit_int8(shift & 0xFF);
5820 }
5821 
5822 void Assembler::vpsrlq(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
5823   assert(UseAVX > 0, "requires some form of AVX");
5824   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5825   attributes.set_rex_vex_w_reverted();
5826   // XMM2 is for /2 encoding: 66 0F 73 /2 ib
5827   int encode = vex_prefix_and_encode(xmm2->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5828   emit_int8(0x73);
5829   emit_int8((unsigned char)(0xC0 | encode));
5830   emit_int8(shift & 0xFF);
5831 }
5832 
5833 void Assembler::vpsrlw(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
5834   assert(UseAVX > 0, "requires some form of AVX");
5835   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5836   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5837   emit_int8((unsigned char)0xD1);
5838   emit_int8((unsigned char)(0xC0 | encode));
5839 }
5840 
5841 void Assembler::vpsrld(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
5842   assert(UseAVX > 0, "requires some form of AVX");
5843   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5844   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5845   emit_int8((unsigned char)0xD2);
5846   emit_int8((unsigned char)(0xC0 | encode));
5847 }
5848 
5849 void Assembler::vpsrlq(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
5850   assert(UseAVX > 0, "requires some form of AVX");
5851   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5852   attributes.set_rex_vex_w_reverted();
5853   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5854   emit_int8((unsigned char)0xD3);
5855   emit_int8((unsigned char)(0xC0 | encode));
5856 }
5857 
5858 // Shift packed integers arithmetically right by specified number of bits.
5859 void Assembler::psraw(XMMRegister dst, int shift) {
5860   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5861   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5862   // XMM4 is for /4 encoding: 66 0F 71 /4 ib
5863   int encode = simd_prefix_and_encode(xmm4, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5864   emit_int8(0x71);
5865   emit_int8((unsigned char)(0xC0 | encode));
5866   emit_int8(shift & 0xFF);
5867 }
5868 
5869 void Assembler::psrad(XMMRegister dst, int shift) {
5870   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5871   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5872   // XMM4 is for /4 encoding: 66 0F 72 /4 ib
5873   int encode = simd_prefix_and_encode(xmm4, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5874   emit_int8(0x72);
5875   emit_int8((unsigned char)(0xC0 | encode));
5876   emit_int8(shift & 0xFF);
5877 }
5878 
5879 void Assembler::psraw(XMMRegister dst, XMMRegister shift) {
5880   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5881   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5882   int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5883   emit_int8((unsigned char)0xE1);
5884   emit_int8((unsigned char)(0xC0 | encode));
5885 }
5886 
5887 void Assembler::psrad(XMMRegister dst, XMMRegister shift) {
5888   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5889   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5890   int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5891   emit_int8((unsigned char)0xE2);
5892   emit_int8((unsigned char)(0xC0 | encode));
5893 }
5894 
5895 void Assembler::vpsraw(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
5896   assert(UseAVX > 0, "requires some form of AVX");
5897   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5898   // XMM4 is for /4 encoding: 66 0F 71 /4 ib
5899   int encode = vex_prefix_and_encode(xmm4->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5900   emit_int8(0x71);
5901   emit_int8((unsigned char)(0xC0 | encode));
5902   emit_int8(shift & 0xFF);
5903 }
5904 
5905 void Assembler::vpsrad(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
5906   assert(UseAVX > 0, "requires some form of AVX");
5907   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5908   // XMM4 is for /4 encoding: 66 0F 71 /4 ib
5909   int encode = vex_prefix_and_encode(xmm4->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5910   emit_int8(0x72);
5911   emit_int8((unsigned char)(0xC0 | encode));
5912   emit_int8(shift & 0xFF);
5913 }
5914 
5915 void Assembler::vpsraw(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
5916   assert(UseAVX > 0, "requires some form of AVX");
5917   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5918   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5919   emit_int8((unsigned char)0xE1);
5920   emit_int8((unsigned char)(0xC0 | encode));
5921 }
5922 
5923 void Assembler::vpsrad(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
5924   assert(UseAVX > 0, "requires some form of AVX");
5925   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5926   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5927   emit_int8((unsigned char)0xE2);
5928   emit_int8((unsigned char)(0xC0 | encode));
5929 }
5930 
5931 
5932 // logical operations packed integers
5933 void Assembler::pand(XMMRegister dst, XMMRegister src) {
5934   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5935   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5936   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5937   emit_int8((unsigned char)0xDB);
5938   emit_int8((unsigned char)(0xC0 | encode));
5939 }
5940 
5941 void Assembler::vpand(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5942   assert(UseAVX > 0, "requires some form of AVX");
5943   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5944   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5945   emit_int8((unsigned char)0xDB);
5946   emit_int8((unsigned char)(0xC0 | encode));
5947 }
5948 
5949 void Assembler::vpand(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5950   assert(UseAVX > 0, "requires some form of AVX");
5951   InstructionMark im(this);
5952   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5953   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
5954   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5955   emit_int8((unsigned char)0xDB);
5956   emit_operand(dst, src);
5957 }
5958 
5959 void Assembler::pandn(XMMRegister dst, XMMRegister src) {
5960   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5961   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5962   attributes.set_rex_vex_w_reverted();
5963   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5964   emit_int8((unsigned char)0xDF);
5965   emit_int8((unsigned char)(0xC0 | encode));
5966 }
5967 
5968 void Assembler::por(XMMRegister dst, XMMRegister src) {
5969   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5970   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5971   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5972   emit_int8((unsigned char)0xEB);
5973   emit_int8((unsigned char)(0xC0 | encode));
5974 }
5975 
5976 void Assembler::vpor(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5977   assert(UseAVX > 0, "requires some form of AVX");
5978   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5979   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5980   emit_int8((unsigned char)0xEB);
5981   emit_int8((unsigned char)(0xC0 | encode));
5982 }
5983 
5984 void Assembler::vpor(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5985   assert(UseAVX > 0, "requires some form of AVX");
5986   InstructionMark im(this);
5987   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5988   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
5989   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5990   emit_int8((unsigned char)0xEB);
5991   emit_operand(dst, src);
5992 }
5993 
5994 void Assembler::pxor(XMMRegister dst, XMMRegister src) {
5995   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5996   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5997   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5998   emit_int8((unsigned char)0xEF);
5999   emit_int8((unsigned char)(0xC0 | encode));
6000 }
6001 
6002 void Assembler::vpxor(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
6003   assert(UseAVX > 0, "requires some form of AVX");
6004   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6005   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6006   emit_int8((unsigned char)0xEF);
6007   emit_int8((unsigned char)(0xC0 | encode));
6008 }
6009 
6010 void Assembler::vpxor(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
6011   assert(UseAVX > 0, "requires some form of AVX");
6012   InstructionMark im(this);
6013   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6014   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
6015   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6016   emit_int8((unsigned char)0xEF);
6017   emit_operand(dst, src);
6018 }
6019 
6020 
6021 // vinserti forms
6022 
6023 void Assembler::vinserti128(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) {
6024   assert(VM_Version::supports_avx2(), "");
6025   assert(imm8 <= 0x01, "imm8: %u", imm8);
6026   int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_256bit;
6027   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6028   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6029   emit_int8(0x38);
6030   emit_int8((unsigned char)(0xC0 | encode));
6031   // 0x00 - insert into lower 128 bits
6032   // 0x01 - insert into upper 128 bits
6033   emit_int8(imm8 & 0x01);
6034 }
6035 
6036 void Assembler::vinserti128(XMMRegister dst, XMMRegister nds, Address src, uint8_t imm8) {
6037   assert(VM_Version::supports_avx2(), "");
6038   assert(dst != xnoreg, "sanity");
6039   assert(imm8 <= 0x01, "imm8: %u", imm8);
6040   int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_256bit;
6041   InstructionMark im(this);
6042   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6043   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
6044   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6045   emit_int8(0x38);
6046   emit_operand(dst, src);
6047   // 0x00 - insert into lower 128 bits
6048   // 0x01 - insert into upper 128 bits
6049   emit_int8(imm8 & 0x01);
6050 }
6051 
6052 void Assembler::vinserti32x4(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) {
6053   assert(VM_Version::supports_evex(), "");
6054   assert(imm8 <= 0x03, "imm8: %u", imm8);
6055   InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6056   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6057   emit_int8(0x38);
6058   emit_int8((unsigned char)(0xC0 | encode));
6059   // 0x00 - insert into q0 128 bits (0..127)
6060   // 0x01 - insert into q1 128 bits (128..255)
6061   // 0x02 - insert into q2 128 bits (256..383)
6062   // 0x03 - insert into q3 128 bits (384..511)
6063   emit_int8(imm8 & 0x03);
6064 }
6065 
6066 void Assembler::vinserti32x4(XMMRegister dst, XMMRegister nds, Address src, uint8_t imm8) {
6067   assert(VM_Version::supports_avx(), "");
6068   assert(dst != xnoreg, "sanity");
6069   assert(imm8 <= 0x03, "imm8: %u", imm8);
6070   int vector_len = VM_Version::supports_evex() ? AVX_512bit : AVX_256bit;
6071   InstructionMark im(this);
6072   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6073   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
6074   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6075   emit_int8(0x18);
6076   emit_operand(dst, src);
6077   // 0x00 - insert into q0 128 bits (0..127)
6078   // 0x01 - insert into q1 128 bits (128..255)
6079   // 0x02 - insert into q2 128 bits (256..383)
6080   // 0x03 - insert into q3 128 bits (384..511)
6081   emit_int8(imm8 & 0x03);
6082 }
6083 
6084 void Assembler::vinserti64x4(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) {
6085   assert(VM_Version::supports_evex(), "");
6086   assert(imm8 <= 0x01, "imm8: %u", imm8);
6087   InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6088   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6089   emit_int8(0x38);
6090   emit_int8((unsigned char)(0xC0 | encode));
6091   // 0x00 - insert into lower 256 bits
6092   // 0x01 - insert into upper 256 bits
6093   emit_int8(imm8 & 0x01);
6094 }
6095 
6096 
6097 // vinsertf forms
6098 
6099 void Assembler::vinsertf128(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) {
6100   assert(VM_Version::supports_avx(), "");
6101   assert(imm8 <= 0x01, "imm8: %u", imm8);
6102   int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_256bit;
6103   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6104   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6105   emit_int8(0x18);
6106   emit_int8((unsigned char)(0xC0 | encode));
6107   // 0x00 - insert into lower 128 bits
6108   // 0x01 - insert into upper 128 bits
6109   emit_int8(imm8 & 0x01);
6110 }
6111 
6112 void Assembler::vinsertf128(XMMRegister dst, XMMRegister nds, Address src, uint8_t imm8) {
6113   assert(VM_Version::supports_avx(), "");
6114   assert(dst != xnoreg, "sanity");
6115   assert(imm8 <= 0x01, "imm8: %u", imm8);
6116   int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_256bit;
6117   InstructionMark im(this);
6118   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6119   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
6120   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6121   emit_int8(0x18);
6122   emit_operand(dst, src);
6123   // 0x00 - insert into lower 128 bits
6124   // 0x01 - insert into upper 128 bits
6125   emit_int8(imm8 & 0x01);
6126 }
6127 
6128 void Assembler::vinsertf32x4(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) {
6129   assert(VM_Version::supports_evex(), "");
6130   assert(imm8 <= 0x03, "imm8: %u", imm8);
6131   InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6132   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6133   emit_int8(0x18);
6134   emit_int8((unsigned char)(0xC0 | encode));
6135   // 0x00 - insert into q0 128 bits (0..127)
6136   // 0x01 - insert into q1 128 bits (128..255)
6137   // 0x02 - insert into q2 128 bits (256..383)
6138   // 0x03 - insert into q3 128 bits (384..511)
6139   emit_int8(imm8 & 0x03);
6140 }
6141 
6142 void Assembler::vinsertf32x4(XMMRegister dst, XMMRegister nds, Address src, uint8_t imm8) {
6143   assert(VM_Version::supports_avx(), "");
6144   assert(dst != xnoreg, "sanity");
6145   assert(imm8 <= 0x03, "imm8: %u", imm8);
6146   int vector_len = VM_Version::supports_evex() ? AVX_512bit : AVX_256bit;
6147   InstructionMark im(this);
6148   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6149   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
6150   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6151   emit_int8(0x18);
6152   emit_operand(dst, src);
6153   // 0x00 - insert into q0 128 bits (0..127)
6154   // 0x01 - insert into q1 128 bits (128..255)
6155   // 0x02 - insert into q2 128 bits (256..383)
6156   // 0x03 - insert into q3 128 bits (384..511)
6157   emit_int8(imm8 & 0x03);
6158 }
6159 
6160 void Assembler::vinsertf64x4(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) {
6161   assert(VM_Version::supports_evex(), "");
6162   assert(imm8 <= 0x01, "imm8: %u", imm8);
6163   InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6164   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6165   emit_int8(0x1A);
6166   emit_int8((unsigned char)(0xC0 | encode));
6167   // 0x00 - insert into lower 256 bits
6168   // 0x01 - insert into upper 256 bits
6169   emit_int8(imm8 & 0x01);
6170 }
6171 
6172 void Assembler::vinsertf64x4(XMMRegister dst, XMMRegister nds, Address src, uint8_t imm8) {
6173   assert(VM_Version::supports_evex(), "");
6174   assert(dst != xnoreg, "sanity");
6175   assert(imm8 <= 0x01, "imm8: %u", imm8);
6176   InstructionMark im(this);
6177   InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6178   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_64bit);
6179   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6180   emit_int8(0x1A);
6181   emit_operand(dst, src);
6182   // 0x00 - insert into lower 256 bits
6183   // 0x01 - insert into upper 256 bits
6184   emit_int8(imm8 & 0x01);
6185 }
6186 
6187 
6188 // vextracti forms
6189 
6190 void Assembler::vextracti128(XMMRegister dst, XMMRegister src, uint8_t imm8) {
6191   assert(VM_Version::supports_avx(), "");
6192   assert(imm8 <= 0x01, "imm8: %u", imm8);
6193   int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_256bit;
6194   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6195   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6196   emit_int8(0x39);
6197   emit_int8((unsigned char)(0xC0 | encode));
6198   // 0x00 - extract from lower 128 bits
6199   // 0x01 - extract from upper 128 bits
6200   emit_int8(imm8 & 0x01);
6201 }
6202 
6203 void Assembler::vextracti128(Address dst, XMMRegister src, uint8_t imm8) {
6204   assert(VM_Version::supports_avx2(), "");
6205   assert(src != xnoreg, "sanity");
6206   assert(imm8 <= 0x01, "imm8: %u", imm8);
6207   int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_256bit;
6208   InstructionMark im(this);
6209   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6210   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
6211   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6212   emit_int8(0x39);
6213   emit_operand(src, dst);
6214   // 0x00 - extract from lower 128 bits
6215   // 0x01 - extract from upper 128 bits
6216   emit_int8(imm8 & 0x01);
6217 }
6218 
6219 void Assembler::vextracti32x4(XMMRegister dst, XMMRegister src, uint8_t imm8) {
6220   assert(VM_Version::supports_avx(), "");
6221   assert(imm8 <= 0x03, "imm8: %u", imm8);
6222   int vector_len = VM_Version::supports_evex() ? AVX_512bit : AVX_256bit;
6223   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6224   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6225   emit_int8(0x39);
6226   emit_int8((unsigned char)(0xC0 | encode));
6227   // 0x00 - extract from bits 127:0
6228   // 0x01 - extract from bits 255:128
6229   // 0x02 - extract from bits 383:256
6230   // 0x03 - extract from bits 511:384
6231   emit_int8(imm8 & 0x03);
6232 }
6233 
6234 void Assembler::vextracti32x4(Address dst, XMMRegister src, uint8_t imm8) {
6235   assert(VM_Version::supports_evex(), "");
6236   assert(src != xnoreg, "sanity");
6237   assert(imm8 <= 0x03, "imm8: %u", imm8);
6238   InstructionMark im(this);
6239   InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6240   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
6241   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6242   emit_int8(0x39);
6243   emit_operand(src, dst);
6244   // 0x00 - extract from bits 127:0
6245   // 0x01 - extract from bits 255:128
6246   // 0x02 - extract from bits 383:256
6247   // 0x03 - extract from bits 511:384
6248   emit_int8(imm8 & 0x03);
6249 }
6250 
6251 void Assembler::vextracti64x2(XMMRegister dst, XMMRegister src, uint8_t imm8) {
6252   assert(VM_Version::supports_avx512dq(), "");
6253   assert(imm8 <= 0x03, "imm8: %u", imm8);
6254   InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6255   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6256   emit_int8(0x39);
6257   emit_int8((unsigned char)(0xC0 | encode));
6258   // 0x00 - extract from bits 127:0
6259   // 0x01 - extract from bits 255:128
6260   // 0x02 - extract from bits 383:256
6261   // 0x03 - extract from bits 511:384
6262   emit_int8(imm8 & 0x03);
6263 }
6264 
6265 void Assembler::vextracti64x4(XMMRegister dst, XMMRegister src, uint8_t imm8) {
6266   assert(VM_Version::supports_evex(), "");
6267   assert(imm8 <= 0x01, "imm8: %u", imm8);
6268   InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6269   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6270   emit_int8(0x3B);
6271   emit_int8((unsigned char)(0xC0 | encode));
6272   // 0x00 - extract from lower 256 bits
6273   // 0x01 - extract from upper 256 bits
6274   emit_int8(imm8 & 0x01);
6275 }
6276 
6277 
6278 // vextractf forms
6279 
6280 void Assembler::vextractf128(XMMRegister dst, XMMRegister src, uint8_t imm8) {
6281   assert(VM_Version::supports_avx(), "");
6282   assert(imm8 <= 0x01, "imm8: %u", imm8);
6283   int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_256bit;
6284   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6285   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6286   emit_int8(0x19);
6287   emit_int8((unsigned char)(0xC0 | encode));
6288   // 0x00 - extract from lower 128 bits
6289   // 0x01 - extract from upper 128 bits
6290   emit_int8(imm8 & 0x01);
6291 }
6292 
6293 void Assembler::vextractf128(Address dst, XMMRegister src, uint8_t imm8) {
6294   assert(VM_Version::supports_avx(), "");
6295   assert(src != xnoreg, "sanity");
6296   assert(imm8 <= 0x01, "imm8: %u", imm8);
6297   int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_256bit;
6298   InstructionMark im(this);
6299   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6300   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
6301   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6302   emit_int8(0x19);
6303   emit_operand(src, dst);
6304   // 0x00 - extract from lower 128 bits
6305   // 0x01 - extract from upper 128 bits
6306   emit_int8(imm8 & 0x01);
6307 }
6308 
6309 void Assembler::vextractf32x4(XMMRegister dst, XMMRegister src, uint8_t imm8) {
6310   assert(VM_Version::supports_avx(), "");
6311   assert(imm8 <= 0x03, "imm8: %u", imm8);
6312   int vector_len = VM_Version::supports_evex() ? AVX_512bit : AVX_256bit;
6313   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6314   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6315   emit_int8(0x19);
6316   emit_int8((unsigned char)(0xC0 | encode));
6317   // 0x00 - extract from bits 127:0
6318   // 0x01 - extract from bits 255:128
6319   // 0x02 - extract from bits 383:256
6320   // 0x03 - extract from bits 511:384
6321   emit_int8(imm8 & 0x03);
6322 }
6323 
6324 void Assembler::vextractf32x4(Address dst, XMMRegister src, uint8_t imm8) {
6325   assert(VM_Version::supports_evex(), "");
6326   assert(src != xnoreg, "sanity");
6327   assert(imm8 <= 0x03, "imm8: %u", imm8);
6328   InstructionMark im(this);
6329   InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6330   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
6331   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6332   emit_int8(0x19);
6333   emit_operand(src, dst);
6334   // 0x00 - extract from bits 127:0
6335   // 0x01 - extract from bits 255:128
6336   // 0x02 - extract from bits 383:256
6337   // 0x03 - extract from bits 511:384
6338   emit_int8(imm8 & 0x03);
6339 }
6340 
6341 void Assembler::vextractf64x2(XMMRegister dst, XMMRegister src, uint8_t imm8) {
6342   assert(VM_Version::supports_avx512dq(), "");
6343   assert(imm8 <= 0x03, "imm8: %u", imm8);
6344   InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6345   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6346   emit_int8(0x19);
6347   emit_int8((unsigned char)(0xC0 | encode));
6348   // 0x00 - extract from bits 127:0
6349   // 0x01 - extract from bits 255:128
6350   // 0x02 - extract from bits 383:256
6351   // 0x03 - extract from bits 511:384
6352   emit_int8(imm8 & 0x03);
6353 }
6354 
6355 void Assembler::vextractf64x4(XMMRegister dst, XMMRegister src, uint8_t imm8) {
6356   assert(VM_Version::supports_evex(), "");
6357   assert(imm8 <= 0x01, "imm8: %u", imm8);
6358   InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6359   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6360   emit_int8(0x1B);
6361   emit_int8((unsigned char)(0xC0 | encode));
6362   // 0x00 - extract from lower 256 bits
6363   // 0x01 - extract from upper 256 bits
6364   emit_int8(imm8 & 0x01);
6365 }
6366 
6367 void Assembler::vextractf64x4(Address dst, XMMRegister src, uint8_t imm8) {
6368   assert(VM_Version::supports_evex(), "");
6369   assert(src != xnoreg, "sanity");
6370   assert(imm8 <= 0x01, "imm8: %u", imm8);
6371   InstructionMark im(this);
6372   InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6373   attributes.set_address_attributes(/* tuple_type */ EVEX_T4,/* input_size_in_bits */  EVEX_64bit);
6374   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6375   emit_int8(0x1B);
6376   emit_operand(src, dst);
6377   // 0x00 - extract from lower 256 bits
6378   // 0x01 - extract from upper 256 bits
6379   emit_int8(imm8 & 0x01);
6380 }
6381 
6382 
6383 // legacy word/dword replicate
6384 void Assembler::vpbroadcastw(XMMRegister dst, XMMRegister src) {
6385   assert(VM_Version::supports_avx2(), "");
6386   InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
6387   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6388   emit_int8(0x79);
6389   emit_int8((unsigned char)(0xC0 | encode));
6390 }
6391 
6392 void Assembler::vpbroadcastd(XMMRegister dst, XMMRegister src) {
6393   assert(VM_Version::supports_avx2(), "");
6394   InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6395   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6396   emit_int8(0x58);
6397   emit_int8((unsigned char)(0xC0 | encode));
6398 }
6399 
6400 
6401 // xmm/mem sourced byte/word/dword/qword replicate
6402 
6403 // duplicate 1-byte integer data from src into programmed locations in dest : requires AVX512BW and AVX512VL
6404 void Assembler::evpbroadcastb(XMMRegister dst, XMMRegister src, int vector_len) {
6405   assert(VM_Version::supports_evex(), "");
6406   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
6407   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6408   emit_int8(0x78);
6409   emit_int8((unsigned char)(0xC0 | encode));
6410 }
6411 
6412 void Assembler::evpbroadcastb(XMMRegister dst, Address src, int vector_len) {
6413   assert(VM_Version::supports_evex(), "");
6414   assert(dst != xnoreg, "sanity");
6415   InstructionMark im(this);
6416   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
6417   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_8bit);
6418   // swap src<->dst for encoding
6419   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6420   emit_int8(0x78);
6421   emit_operand(dst, src);
6422 }
6423 
6424 // duplicate 2-byte integer data from src into programmed locations in dest : requires AVX512BW and AVX512VL
6425 void Assembler::evpbroadcastw(XMMRegister dst, XMMRegister src, int vector_len) {
6426   assert(VM_Version::supports_evex(), "");
6427   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
6428   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6429   emit_int8(0x79);
6430   emit_int8((unsigned char)(0xC0 | encode));
6431 }
6432 
6433 void Assembler::evpbroadcastw(XMMRegister dst, Address src, int vector_len) {
6434   assert(VM_Version::supports_evex(), "");
6435   assert(dst != xnoreg, "sanity");
6436   InstructionMark im(this);
6437   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
6438   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_16bit);
6439   // swap src<->dst for encoding
6440   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6441   emit_int8(0x79);
6442   emit_operand(dst, src);
6443 }
6444 
6445 // duplicate 4-byte integer data from src into programmed locations in dest : requires AVX512VL
6446 void Assembler::evpbroadcastd(XMMRegister dst, XMMRegister src, int vector_len) {
6447   assert(VM_Version::supports_evex(), "");
6448   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6449   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6450   emit_int8(0x58);
6451   emit_int8((unsigned char)(0xC0 | encode));
6452 }
6453 
6454 void Assembler::evpbroadcastd(XMMRegister dst, Address src, int vector_len) {
6455   assert(VM_Version::supports_evex(), "");
6456   assert(dst != xnoreg, "sanity");
6457   InstructionMark im(this);
6458   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6459   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
6460   // swap src<->dst for encoding
6461   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6462   emit_int8(0x58);
6463   emit_operand(dst, src);
6464 }
6465 
6466 // duplicate 8-byte integer data from src into programmed locations in dest : requires AVX512VL
6467 void Assembler::evpbroadcastq(XMMRegister dst, XMMRegister src, int vector_len) {
6468   assert(VM_Version::supports_evex(), "");
6469   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6470   attributes.set_rex_vex_w_reverted();
6471   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6472   emit_int8(0x59);
6473   emit_int8((unsigned char)(0xC0 | encode));
6474 }
6475 
6476 void Assembler::evpbroadcastq(XMMRegister dst, Address src, int vector_len) {
6477   assert(VM_Version::supports_evex(), "");
6478   assert(dst != xnoreg, "sanity");
6479   InstructionMark im(this);
6480   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6481   attributes.set_rex_vex_w_reverted();
6482   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
6483   // swap src<->dst for encoding
6484   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6485   emit_int8(0x59);
6486   emit_operand(dst, src);
6487 }
6488 
6489 
6490 // scalar single/double precision replicate
6491 
6492 // duplicate single precision data from src into programmed locations in dest : requires AVX512VL
6493 void Assembler::evpbroadcastss(XMMRegister dst, XMMRegister src, int vector_len) {
6494   assert(VM_Version::supports_evex(), "");
6495   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6496   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6497   emit_int8(0x18);
6498   emit_int8((unsigned char)(0xC0 | encode));
6499 }
6500 
6501 void Assembler::evpbroadcastss(XMMRegister dst, Address src, int vector_len) {
6502   assert(VM_Version::supports_evex(), "");
6503   assert(dst != xnoreg, "sanity");
6504   InstructionMark im(this);
6505   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6506   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
6507   // swap src<->dst for encoding
6508   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6509   emit_int8(0x18);
6510   emit_operand(dst, src);
6511 }
6512 
6513 // duplicate double precision data from src into programmed locations in dest : requires AVX512VL
6514 void Assembler::evpbroadcastsd(XMMRegister dst, XMMRegister src, int vector_len) {
6515   assert(VM_Version::supports_evex(), "");
6516   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6517   attributes.set_rex_vex_w_reverted();
6518   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6519   emit_int8(0x19);
6520   emit_int8((unsigned char)(0xC0 | encode));
6521 }
6522 
6523 void Assembler::evpbroadcastsd(XMMRegister dst, Address src, int vector_len) {
6524   assert(VM_Version::supports_evex(), "");
6525   assert(dst != xnoreg, "sanity");
6526   InstructionMark im(this);
6527   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6528   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
6529   attributes.set_rex_vex_w_reverted();
6530   // swap src<->dst for encoding
6531   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6532   emit_int8(0x19);
6533   emit_operand(dst, src);
6534 }
6535 
6536 
6537 // gpr source broadcast forms
6538 
6539 // duplicate 1-byte integer data from src into programmed locations in dest : requires AVX512BW and AVX512VL
6540 void Assembler::evpbroadcastb(XMMRegister dst, Register src, int vector_len) {
6541   assert(VM_Version::supports_evex(), "");
6542   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
6543   attributes.set_is_evex_instruction();
6544   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6545   emit_int8(0x7A);
6546   emit_int8((unsigned char)(0xC0 | encode));
6547 }
6548 
6549 // duplicate 2-byte integer data from src into programmed locations in dest : requires AVX512BW and AVX512VL
6550 void Assembler::evpbroadcastw(XMMRegister dst, Register src, int vector_len) {
6551   assert(VM_Version::supports_evex(), "");
6552   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
6553   attributes.set_is_evex_instruction();
6554   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6555   emit_int8(0x7B);
6556   emit_int8((unsigned char)(0xC0 | encode));
6557 }
6558 
6559 // duplicate 4-byte integer data from src into programmed locations in dest : requires AVX512VL
6560 void Assembler::evpbroadcastd(XMMRegister dst, Register src, int vector_len) {
6561   assert(VM_Version::supports_evex(), "");
6562   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6563   attributes.set_is_evex_instruction();
6564   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6565   emit_int8(0x7C);
6566   emit_int8((unsigned char)(0xC0 | encode));
6567 }
6568 
6569 // duplicate 8-byte integer data from src into programmed locations in dest : requires AVX512VL
6570 void Assembler::evpbroadcastq(XMMRegister dst, Register src, int vector_len) {
6571   assert(VM_Version::supports_evex(), "");
6572   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6573   attributes.set_is_evex_instruction();
6574   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6575   emit_int8(0x7C);
6576   emit_int8((unsigned char)(0xC0 | encode));
6577 }
6578 
6579 
6580 // Carry-Less Multiplication Quadword
6581 void Assembler::pclmulqdq(XMMRegister dst, XMMRegister src, int mask) {
6582   assert(VM_Version::supports_clmul(), "");
6583   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
6584   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6585   emit_int8(0x44);
6586   emit_int8((unsigned char)(0xC0 | encode));
6587   emit_int8((unsigned char)mask);
6588 }
6589 
6590 // Carry-Less Multiplication Quadword
6591 void Assembler::vpclmulqdq(XMMRegister dst, XMMRegister nds, XMMRegister src, int mask) {
6592   assert(VM_Version::supports_avx() && VM_Version::supports_clmul(), "");
6593   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
6594   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6595   emit_int8(0x44);
6596   emit_int8((unsigned char)(0xC0 | encode));
6597   emit_int8((unsigned char)mask);
6598 }
6599 
6600 void Assembler::vzeroupper() {
6601   assert(VM_Version::supports_avx(), "");
6602   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
6603   (void)vex_prefix_and_encode(0, 0, 0, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
6604   emit_int8(0x77);
6605 }
6606 
6607 #ifndef _LP64
6608 // 32bit only pieces of the assembler
6609 
6610 void Assembler::cmp_literal32(Register src1, int32_t imm32, RelocationHolder const& rspec) {
6611   // NO PREFIX AS NEVER 64BIT
6612   InstructionMark im(this);
6613   emit_int8((unsigned char)0x81);
6614   emit_int8((unsigned char)(0xF8 | src1->encoding()));
6615   emit_data(imm32, rspec, 0);
6616 }
6617 
6618 void Assembler::cmp_literal32(Address src1, int32_t imm32, RelocationHolder const& rspec) {
6619   // NO PREFIX AS NEVER 64BIT (not even 32bit versions of 64bit regs
6620   InstructionMark im(this);
6621   emit_int8((unsigned char)0x81);
6622   emit_operand(rdi, src1);
6623   emit_data(imm32, rspec, 0);
6624 }
6625 
6626 // The 64-bit (32bit platform) cmpxchg compares the value at adr with the contents of rdx:rax,
6627 // and stores rcx:rbx into adr if so; otherwise, the value at adr is loaded
6628 // into rdx:rax.  The ZF is set if the compared values were equal, and cleared otherwise.
6629 void Assembler::cmpxchg8(Address adr) {
6630   InstructionMark im(this);
6631   emit_int8(0x0F);
6632   emit_int8((unsigned char)0xC7);
6633   emit_operand(rcx, adr);
6634 }
6635 
6636 void Assembler::decl(Register dst) {
6637   // Don't use it directly. Use MacroAssembler::decrementl() instead.
6638  emit_int8(0x48 | dst->encoding());
6639 }
6640 
6641 #endif // _LP64
6642 
6643 // 64bit typically doesn't use the x87 but needs to for the trig funcs
6644 
6645 void Assembler::fabs() {
6646   emit_int8((unsigned char)0xD9);
6647   emit_int8((unsigned char)0xE1);
6648 }
6649 
6650 void Assembler::fadd(int i) {
6651   emit_farith(0xD8, 0xC0, i);
6652 }
6653 
6654 void Assembler::fadd_d(Address src) {
6655   InstructionMark im(this);
6656   emit_int8((unsigned char)0xDC);
6657   emit_operand32(rax, src);
6658 }
6659 
6660 void Assembler::fadd_s(Address src) {
6661   InstructionMark im(this);
6662   emit_int8((unsigned char)0xD8);
6663   emit_operand32(rax, src);
6664 }
6665 
6666 void Assembler::fadda(int i) {
6667   emit_farith(0xDC, 0xC0, i);
6668 }
6669 
6670 void Assembler::faddp(int i) {
6671   emit_farith(0xDE, 0xC0, i);
6672 }
6673 
6674 void Assembler::fchs() {
6675   emit_int8((unsigned char)0xD9);
6676   emit_int8((unsigned char)0xE0);
6677 }
6678 
6679 void Assembler::fcom(int i) {
6680   emit_farith(0xD8, 0xD0, i);
6681 }
6682 
6683 void Assembler::fcomp(int i) {
6684   emit_farith(0xD8, 0xD8, i);
6685 }
6686 
6687 void Assembler::fcomp_d(Address src) {
6688   InstructionMark im(this);
6689   emit_int8((unsigned char)0xDC);
6690   emit_operand32(rbx, src);
6691 }
6692 
6693 void Assembler::fcomp_s(Address src) {
6694   InstructionMark im(this);
6695   emit_int8((unsigned char)0xD8);
6696   emit_operand32(rbx, src);
6697 }
6698 
6699 void Assembler::fcompp() {
6700   emit_int8((unsigned char)0xDE);
6701   emit_int8((unsigned char)0xD9);
6702 }
6703 
6704 void Assembler::fcos() {
6705   emit_int8((unsigned char)0xD9);
6706   emit_int8((unsigned char)0xFF);
6707 }
6708 
6709 void Assembler::fdecstp() {
6710   emit_int8((unsigned char)0xD9);
6711   emit_int8((unsigned char)0xF6);
6712 }
6713 
6714 void Assembler::fdiv(int i) {
6715   emit_farith(0xD8, 0xF0, i);
6716 }
6717 
6718 void Assembler::fdiv_d(Address src) {
6719   InstructionMark im(this);
6720   emit_int8((unsigned char)0xDC);
6721   emit_operand32(rsi, src);
6722 }
6723 
6724 void Assembler::fdiv_s(Address src) {
6725   InstructionMark im(this);
6726   emit_int8((unsigned char)0xD8);
6727   emit_operand32(rsi, src);
6728 }
6729 
6730 void Assembler::fdiva(int i) {
6731   emit_farith(0xDC, 0xF8, i);
6732 }
6733 
6734 // Note: The Intel manual (Pentium Processor User's Manual, Vol.3, 1994)
6735 //       is erroneous for some of the floating-point instructions below.
6736 
6737 void Assembler::fdivp(int i) {
6738   emit_farith(0xDE, 0xF8, i);                    // ST(0) <- ST(0) / ST(1) and pop (Intel manual wrong)
6739 }
6740 
6741 void Assembler::fdivr(int i) {
6742   emit_farith(0xD8, 0xF8, i);
6743 }
6744 
6745 void Assembler::fdivr_d(Address src) {
6746   InstructionMark im(this);
6747   emit_int8((unsigned char)0xDC);
6748   emit_operand32(rdi, src);
6749 }
6750 
6751 void Assembler::fdivr_s(Address src) {
6752   InstructionMark im(this);
6753   emit_int8((unsigned char)0xD8);
6754   emit_operand32(rdi, src);
6755 }
6756 
6757 void Assembler::fdivra(int i) {
6758   emit_farith(0xDC, 0xF0, i);
6759 }
6760 
6761 void Assembler::fdivrp(int i) {
6762   emit_farith(0xDE, 0xF0, i);                    // ST(0) <- ST(1) / ST(0) and pop (Intel manual wrong)
6763 }
6764 
6765 void Assembler::ffree(int i) {
6766   emit_farith(0xDD, 0xC0, i);
6767 }
6768 
6769 void Assembler::fild_d(Address adr) {
6770   InstructionMark im(this);
6771   emit_int8((unsigned char)0xDF);
6772   emit_operand32(rbp, adr);
6773 }
6774 
6775 void Assembler::fild_s(Address adr) {
6776   InstructionMark im(this);
6777   emit_int8((unsigned char)0xDB);
6778   emit_operand32(rax, adr);
6779 }
6780 
6781 void Assembler::fincstp() {
6782   emit_int8((unsigned char)0xD9);
6783   emit_int8((unsigned char)0xF7);
6784 }
6785 
6786 void Assembler::finit() {
6787   emit_int8((unsigned char)0x9B);
6788   emit_int8((unsigned char)0xDB);
6789   emit_int8((unsigned char)0xE3);
6790 }
6791 
6792 void Assembler::fist_s(Address adr) {
6793   InstructionMark im(this);
6794   emit_int8((unsigned char)0xDB);
6795   emit_operand32(rdx, adr);
6796 }
6797 
6798 void Assembler::fistp_d(Address adr) {
6799   InstructionMark im(this);
6800   emit_int8((unsigned char)0xDF);
6801   emit_operand32(rdi, adr);
6802 }
6803 
6804 void Assembler::fistp_s(Address adr) {
6805   InstructionMark im(this);
6806   emit_int8((unsigned char)0xDB);
6807   emit_operand32(rbx, adr);
6808 }
6809 
6810 void Assembler::fld1() {
6811   emit_int8((unsigned char)0xD9);
6812   emit_int8((unsigned char)0xE8);
6813 }
6814 
6815 void Assembler::fld_d(Address adr) {
6816   InstructionMark im(this);
6817   emit_int8((unsigned char)0xDD);
6818   emit_operand32(rax, adr);
6819 }
6820 
6821 void Assembler::fld_s(Address adr) {
6822   InstructionMark im(this);
6823   emit_int8((unsigned char)0xD9);
6824   emit_operand32(rax, adr);
6825 }
6826 
6827 
6828 void Assembler::fld_s(int index) {
6829   emit_farith(0xD9, 0xC0, index);
6830 }
6831 
6832 void Assembler::fld_x(Address adr) {
6833   InstructionMark im(this);
6834   emit_int8((unsigned char)0xDB);
6835   emit_operand32(rbp, adr);
6836 }
6837 
6838 void Assembler::fldcw(Address src) {
6839   InstructionMark im(this);
6840   emit_int8((unsigned char)0xD9);
6841   emit_operand32(rbp, src);
6842 }
6843 
6844 void Assembler::fldenv(Address src) {
6845   InstructionMark im(this);
6846   emit_int8((unsigned char)0xD9);
6847   emit_operand32(rsp, src);
6848 }
6849 
6850 void Assembler::fldlg2() {
6851   emit_int8((unsigned char)0xD9);
6852   emit_int8((unsigned char)0xEC);
6853 }
6854 
6855 void Assembler::fldln2() {
6856   emit_int8((unsigned char)0xD9);
6857   emit_int8((unsigned char)0xED);
6858 }
6859 
6860 void Assembler::fldz() {
6861   emit_int8((unsigned char)0xD9);
6862   emit_int8((unsigned char)0xEE);
6863 }
6864 
6865 void Assembler::flog() {
6866   fldln2();
6867   fxch();
6868   fyl2x();
6869 }
6870 
6871 void Assembler::flog10() {
6872   fldlg2();
6873   fxch();
6874   fyl2x();
6875 }
6876 
6877 void Assembler::fmul(int i) {
6878   emit_farith(0xD8, 0xC8, i);
6879 }
6880 
6881 void Assembler::fmul_d(Address src) {
6882   InstructionMark im(this);
6883   emit_int8((unsigned char)0xDC);
6884   emit_operand32(rcx, src);
6885 }
6886 
6887 void Assembler::fmul_s(Address src) {
6888   InstructionMark im(this);
6889   emit_int8((unsigned char)0xD8);
6890   emit_operand32(rcx, src);
6891 }
6892 
6893 void Assembler::fmula(int i) {
6894   emit_farith(0xDC, 0xC8, i);
6895 }
6896 
6897 void Assembler::fmulp(int i) {
6898   emit_farith(0xDE, 0xC8, i);
6899 }
6900 
6901 void Assembler::fnsave(Address dst) {
6902   InstructionMark im(this);
6903   emit_int8((unsigned char)0xDD);
6904   emit_operand32(rsi, dst);
6905 }
6906 
6907 void Assembler::fnstcw(Address src) {
6908   InstructionMark im(this);
6909   emit_int8((unsigned char)0x9B);
6910   emit_int8((unsigned char)0xD9);
6911   emit_operand32(rdi, src);
6912 }
6913 
6914 void Assembler::fnstsw_ax() {
6915   emit_int8((unsigned char)0xDF);
6916   emit_int8((unsigned char)0xE0);
6917 }
6918 
6919 void Assembler::fprem() {
6920   emit_int8((unsigned char)0xD9);
6921   emit_int8((unsigned char)0xF8);
6922 }
6923 
6924 void Assembler::fprem1() {
6925   emit_int8((unsigned char)0xD9);
6926   emit_int8((unsigned char)0xF5);
6927 }
6928 
6929 void Assembler::frstor(Address src) {
6930   InstructionMark im(this);
6931   emit_int8((unsigned char)0xDD);
6932   emit_operand32(rsp, src);
6933 }
6934 
6935 void Assembler::fsin() {
6936   emit_int8((unsigned char)0xD9);
6937   emit_int8((unsigned char)0xFE);
6938 }
6939 
6940 void Assembler::fsqrt() {
6941   emit_int8((unsigned char)0xD9);
6942   emit_int8((unsigned char)0xFA);
6943 }
6944 
6945 void Assembler::fst_d(Address adr) {
6946   InstructionMark im(this);
6947   emit_int8((unsigned char)0xDD);
6948   emit_operand32(rdx, adr);
6949 }
6950 
6951 void Assembler::fst_s(Address adr) {
6952   InstructionMark im(this);
6953   emit_int8((unsigned char)0xD9);
6954   emit_operand32(rdx, adr);
6955 }
6956 
6957 void Assembler::fstp_d(Address adr) {
6958   InstructionMark im(this);
6959   emit_int8((unsigned char)0xDD);
6960   emit_operand32(rbx, adr);
6961 }
6962 
6963 void Assembler::fstp_d(int index) {
6964   emit_farith(0xDD, 0xD8, index);
6965 }
6966 
6967 void Assembler::fstp_s(Address adr) {
6968   InstructionMark im(this);
6969   emit_int8((unsigned char)0xD9);
6970   emit_operand32(rbx, adr);
6971 }
6972 
6973 void Assembler::fstp_x(Address adr) {
6974   InstructionMark im(this);
6975   emit_int8((unsigned char)0xDB);
6976   emit_operand32(rdi, adr);
6977 }
6978 
6979 void Assembler::fsub(int i) {
6980   emit_farith(0xD8, 0xE0, i);
6981 }
6982 
6983 void Assembler::fsub_d(Address src) {
6984   InstructionMark im(this);
6985   emit_int8((unsigned char)0xDC);
6986   emit_operand32(rsp, src);
6987 }
6988 
6989 void Assembler::fsub_s(Address src) {
6990   InstructionMark im(this);
6991   emit_int8((unsigned char)0xD8);
6992   emit_operand32(rsp, src);
6993 }
6994 
6995 void Assembler::fsuba(int i) {
6996   emit_farith(0xDC, 0xE8, i);
6997 }
6998 
6999 void Assembler::fsubp(int i) {
7000   emit_farith(0xDE, 0xE8, i);                    // ST(0) <- ST(0) - ST(1) and pop (Intel manual wrong)
7001 }
7002 
7003 void Assembler::fsubr(int i) {
7004   emit_farith(0xD8, 0xE8, i);
7005 }
7006 
7007 void Assembler::fsubr_d(Address src) {
7008   InstructionMark im(this);
7009   emit_int8((unsigned char)0xDC);
7010   emit_operand32(rbp, src);
7011 }
7012 
7013 void Assembler::fsubr_s(Address src) {
7014   InstructionMark im(this);
7015   emit_int8((unsigned char)0xD8);
7016   emit_operand32(rbp, src);
7017 }
7018 
7019 void Assembler::fsubra(int i) {
7020   emit_farith(0xDC, 0xE0, i);
7021 }
7022 
7023 void Assembler::fsubrp(int i) {
7024   emit_farith(0xDE, 0xE0, i);                    // ST(0) <- ST(1) - ST(0) and pop (Intel manual wrong)
7025 }
7026 
7027 void Assembler::ftan() {
7028   emit_int8((unsigned char)0xD9);
7029   emit_int8((unsigned char)0xF2);
7030   emit_int8((unsigned char)0xDD);
7031   emit_int8((unsigned char)0xD8);
7032 }
7033 
7034 void Assembler::ftst() {
7035   emit_int8((unsigned char)0xD9);
7036   emit_int8((unsigned char)0xE4);
7037 }
7038 
7039 void Assembler::fucomi(int i) {
7040   // make sure the instruction is supported (introduced for P6, together with cmov)
7041   guarantee(VM_Version::supports_cmov(), "illegal instruction");
7042   emit_farith(0xDB, 0xE8, i);
7043 }
7044 
7045 void Assembler::fucomip(int i) {
7046   // make sure the instruction is supported (introduced for P6, together with cmov)
7047   guarantee(VM_Version::supports_cmov(), "illegal instruction");
7048   emit_farith(0xDF, 0xE8, i);
7049 }
7050 
7051 void Assembler::fwait() {
7052   emit_int8((unsigned char)0x9B);
7053 }
7054 
7055 void Assembler::fxch(int i) {
7056   emit_farith(0xD9, 0xC8, i);
7057 }
7058 
7059 void Assembler::fyl2x() {
7060   emit_int8((unsigned char)0xD9);
7061   emit_int8((unsigned char)0xF1);
7062 }
7063 
7064 void Assembler::frndint() {
7065   emit_int8((unsigned char)0xD9);
7066   emit_int8((unsigned char)0xFC);
7067 }
7068 
7069 void Assembler::f2xm1() {
7070   emit_int8((unsigned char)0xD9);
7071   emit_int8((unsigned char)0xF0);
7072 }
7073 
7074 void Assembler::fldl2e() {
7075   emit_int8((unsigned char)0xD9);
7076   emit_int8((unsigned char)0xEA);
7077 }
7078 
7079 // SSE SIMD prefix byte values corresponding to VexSimdPrefix encoding.
7080 static int simd_pre[4] = { 0, 0x66, 0xF3, 0xF2 };
7081 // SSE opcode second byte values (first is 0x0F) corresponding to VexOpcode encoding.
7082 static int simd_opc[4] = { 0,    0, 0x38, 0x3A };
7083 
7084 // Generate SSE legacy REX prefix and SIMD opcode based on VEX encoding.
7085 void Assembler::rex_prefix(Address adr, XMMRegister xreg, VexSimdPrefix pre, VexOpcode opc, bool rex_w) {
7086   if (pre > 0) {
7087     emit_int8(simd_pre[pre]);
7088   }
7089   if (rex_w) {
7090     prefixq(adr, xreg);
7091   } else {
7092     prefix(adr, xreg);
7093   }
7094   if (opc > 0) {
7095     emit_int8(0x0F);
7096     int opc2 = simd_opc[opc];
7097     if (opc2 > 0) {
7098       emit_int8(opc2);
7099     }
7100   }
7101 }
7102 
7103 int Assembler::rex_prefix_and_encode(int dst_enc, int src_enc, VexSimdPrefix pre, VexOpcode opc, bool rex_w) {
7104   if (pre > 0) {
7105     emit_int8(simd_pre[pre]);
7106   }
7107   int encode = (rex_w) ? prefixq_and_encode(dst_enc, src_enc) : prefix_and_encode(dst_enc, src_enc);
7108   if (opc > 0) {
7109     emit_int8(0x0F);
7110     int opc2 = simd_opc[opc];
7111     if (opc2 > 0) {
7112       emit_int8(opc2);
7113     }
7114   }
7115   return encode;
7116 }
7117 
7118 
7119 void Assembler::vex_prefix(bool vex_r, bool vex_b, bool vex_x, int nds_enc, VexSimdPrefix pre, VexOpcode opc) {
7120   int vector_len = _attributes->get_vector_len();
7121   bool vex_w = _attributes->is_rex_vex_w();
7122   if (vex_b || vex_x || vex_w || (opc == VEX_OPCODE_0F_38) || (opc == VEX_OPCODE_0F_3A)) {
7123     prefix(VEX_3bytes);
7124 
7125     int byte1 = (vex_r ? VEX_R : 0) | (vex_x ? VEX_X : 0) | (vex_b ? VEX_B : 0);
7126     byte1 = (~byte1) & 0xE0;
7127     byte1 |= opc;
7128     emit_int8(byte1);
7129 
7130     int byte2 = ((~nds_enc) & 0xf) << 3;
7131     byte2 |= (vex_w ? VEX_W : 0) | ((vector_len > 0) ? 4 : 0) | pre;
7132     emit_int8(byte2);
7133   } else {
7134     prefix(VEX_2bytes);
7135 
7136     int byte1 = vex_r ? VEX_R : 0;
7137     byte1 = (~byte1) & 0x80;
7138     byte1 |= ((~nds_enc) & 0xf) << 3;
7139     byte1 |= ((vector_len > 0 ) ? 4 : 0) | pre;
7140     emit_int8(byte1);
7141   }
7142 }
7143 
7144 // This is a 4 byte encoding
7145 void Assembler::evex_prefix(bool vex_r, bool vex_b, bool vex_x, bool evex_r, bool evex_v, int nds_enc, VexSimdPrefix pre, VexOpcode opc){
7146   // EVEX 0x62 prefix
7147   prefix(EVEX_4bytes);
7148   bool vex_w = _attributes->is_rex_vex_w();
7149   int evex_encoding = (vex_w ? VEX_W : 0);
7150   // EVEX.b is not currently used for broadcast of single element or data rounding modes
7151   _attributes->set_evex_encoding(evex_encoding);
7152 
7153   // P0: byte 2, initialized to RXBR`00mm
7154   // instead of not'd
7155   int byte2 = (vex_r ? VEX_R : 0) | (vex_x ? VEX_X : 0) | (vex_b ? VEX_B : 0) | (evex_r ? EVEX_Rb : 0);
7156   byte2 = (~byte2) & 0xF0;
7157   // confine opc opcode extensions in mm bits to lower two bits
7158   // of form {0F, 0F_38, 0F_3A}
7159   byte2 |= opc;
7160   emit_int8(byte2);
7161 
7162   // P1: byte 3 as Wvvvv1pp
7163   int byte3 = ((~nds_enc) & 0xf) << 3;
7164   // p[10] is always 1
7165   byte3 |= EVEX_F;
7166   byte3 |= (vex_w & 1) << 7;
7167   // confine pre opcode extensions in pp bits to lower two bits
7168   // of form {66, F3, F2}
7169   byte3 |= pre;
7170   emit_int8(byte3);
7171 
7172   // P2: byte 4 as zL'Lbv'aaa
7173   // kregs are implemented in the low 3 bits as aaa (hard code k1, it will be initialized for now)
7174   int byte4 = (_attributes->is_no_reg_mask()) ?
7175               0 :
7176               _attributes->get_embedded_opmask_register_specifier();
7177   // EVEX.v` for extending EVEX.vvvv or VIDX
7178   byte4 |= (evex_v ? 0: EVEX_V);
7179   // third EXEC.b for broadcast actions
7180   byte4 |= (_attributes->is_extended_context() ? EVEX_Rb : 0);
7181   // fourth EVEX.L'L for vector length : 0 is 128, 1 is 256, 2 is 512, currently we do not support 1024
7182   byte4 |= ((_attributes->get_vector_len())& 0x3) << 5;
7183   // last is EVEX.z for zero/merge actions
7184   byte4 |= (_attributes->is_clear_context() ? EVEX_Z : 0);
7185   emit_int8(byte4);
7186 }
7187 
7188 void Assembler::vex_prefix(Address adr, int nds_enc, int xreg_enc, VexSimdPrefix pre, VexOpcode opc, InstructionAttr *attributes) {
7189   bool vex_r = ((xreg_enc & 8) == 8) ? 1 : 0;
7190   bool vex_b = adr.base_needs_rex();
7191   bool vex_x = adr.index_needs_rex();
7192   set_attributes(attributes);
7193   attributes->set_current_assembler(this);
7194 
7195   // if vector length is turned off, revert to AVX for vectors smaller than 512-bit
7196   if (UseAVX > 2 && _legacy_mode_vl && attributes->uses_vl()) {
7197     switch (attributes->get_vector_len()) {
7198     case AVX_128bit:
7199     case AVX_256bit:
7200       attributes->set_is_legacy_mode();
7201       break;
7202     }
7203   }
7204 
7205   // For pure EVEX check and see if this instruction
7206   // is allowed in legacy mode and has resources which will
7207   // fit in it.  Pure EVEX instructions will use set_is_evex_instruction in their definition,
7208   // else that field is set when we encode to EVEX
7209   if (UseAVX > 2 && !attributes->is_legacy_mode() &&
7210       !_is_managed && !attributes->is_evex_instruction()) {
7211     if (!_legacy_mode_vl && attributes->get_vector_len() != AVX_512bit) {
7212       bool check_register_bank = NOT_IA32(true) IA32_ONLY(false);
7213       if (check_register_bank) {
7214         // check nds_enc and xreg_enc for upper bank usage
7215         if (nds_enc < 16 && xreg_enc < 16) {
7216           attributes->set_is_legacy_mode();
7217         }
7218       } else {
7219         attributes->set_is_legacy_mode();
7220       }
7221     }
7222   }
7223 
7224   _is_managed = false;
7225   if (UseAVX > 2 && !attributes->is_legacy_mode())
7226   {
7227     bool evex_r = (xreg_enc >= 16);
7228     bool evex_v = (nds_enc >= 16);
7229     attributes->set_is_evex_instruction();
7230     evex_prefix(vex_r, vex_b, vex_x, evex_r, evex_v, nds_enc, pre, opc);
7231   } else {
7232     if (UseAVX > 2 && attributes->is_rex_vex_w_reverted()) {
7233       attributes->set_rex_vex_w(false);
7234     }
7235     vex_prefix(vex_r, vex_b, vex_x, nds_enc, pre, opc);
7236   }
7237 }
7238 
7239 int Assembler::vex_prefix_and_encode(int dst_enc, int nds_enc, int src_enc, VexSimdPrefix pre, VexOpcode opc, InstructionAttr *attributes) {
7240   bool vex_r = ((dst_enc & 8) == 8) ? 1 : 0;
7241   bool vex_b = ((src_enc & 8) == 8) ? 1 : 0;
7242   bool vex_x = false;
7243   set_attributes(attributes);
7244   attributes->set_current_assembler(this);
7245   bool check_register_bank = NOT_IA32(true) IA32_ONLY(false);
7246 
7247   // if vector length is turned off, revert to AVX for vectors smaller than 512-bit
7248   if (UseAVX > 2 && _legacy_mode_vl && attributes->uses_vl()) {
7249     switch (attributes->get_vector_len()) {
7250     case AVX_128bit:
7251     case AVX_256bit:
7252       if (check_register_bank) {
7253         if (dst_enc >= 16 || nds_enc >= 16 || src_enc >= 16) {
7254           // up propagate arithmetic instructions to meet RA requirements
7255           attributes->set_vector_len(AVX_512bit);
7256         } else {
7257           attributes->set_is_legacy_mode();
7258         }
7259       } else {
7260         attributes->set_is_legacy_mode();
7261       }
7262       break;
7263     }
7264   }
7265 
7266   // For pure EVEX check and see if this instruction
7267   // is allowed in legacy mode and has resources which will
7268   // fit in it.  Pure EVEX instructions will use set_is_evex_instruction in their definition,
7269   // else that field is set when we encode to EVEX
7270   if (UseAVX > 2 && !attributes->is_legacy_mode() &&
7271       !_is_managed && !attributes->is_evex_instruction()) {
7272     if (!_legacy_mode_vl && attributes->get_vector_len() != AVX_512bit) {
7273       if (check_register_bank) {
7274         // check dst_enc, nds_enc and src_enc for upper bank usage
7275         if (dst_enc < 16 && nds_enc < 16 && src_enc < 16) {
7276           attributes->set_is_legacy_mode();
7277         }
7278       } else {
7279         attributes->set_is_legacy_mode();
7280       }
7281     }
7282   }
7283 
7284   _is_managed = false;
7285   if (UseAVX > 2 && !attributes->is_legacy_mode())
7286   {
7287     bool evex_r = (dst_enc >= 16);
7288     bool evex_v = (nds_enc >= 16);
7289     // can use vex_x as bank extender on rm encoding
7290     vex_x = (src_enc >= 16);
7291     attributes->set_is_evex_instruction();
7292     evex_prefix(vex_r, vex_b, vex_x, evex_r, evex_v, nds_enc, pre, opc);
7293   } else {
7294     if (UseAVX > 2 && attributes->is_rex_vex_w_reverted()) {
7295       attributes->set_rex_vex_w(false);
7296     }
7297     vex_prefix(vex_r, vex_b, vex_x, nds_enc, pre, opc);
7298   }
7299 
7300   // return modrm byte components for operands
7301   return (((dst_enc & 7) << 3) | (src_enc & 7));
7302 }
7303 
7304 
7305 void Assembler::simd_prefix(XMMRegister xreg, XMMRegister nds, Address adr, VexSimdPrefix pre,
7306                             VexOpcode opc, InstructionAttr *attributes) {
7307   if (UseAVX > 0) {
7308     int xreg_enc = xreg->encoding();
7309     int nds_enc = nds->is_valid() ? nds->encoding() : 0;
7310     vex_prefix(adr, nds_enc, xreg_enc, pre, opc, attributes);
7311   } else {
7312     assert((nds == xreg) || (nds == xnoreg), "wrong sse encoding");
7313     rex_prefix(adr, xreg, pre, opc, attributes->is_rex_vex_w());
7314   }
7315 }
7316 
7317 int Assembler::simd_prefix_and_encode(XMMRegister dst, XMMRegister nds, XMMRegister src, VexSimdPrefix pre,
7318                                       VexOpcode opc, InstructionAttr *attributes) {
7319   int dst_enc = dst->encoding();
7320   int src_enc = src->encoding();
7321   if (UseAVX > 0) {
7322     int nds_enc = nds->is_valid() ? nds->encoding() : 0;
7323     return vex_prefix_and_encode(dst_enc, nds_enc, src_enc, pre, opc, attributes);
7324   } else {
7325     assert((nds == dst) || (nds == src) || (nds == xnoreg), "wrong sse encoding");
7326     return rex_prefix_and_encode(dst_enc, src_enc, pre, opc, attributes->is_rex_vex_w());
7327   }
7328 }
7329 
7330 void Assembler::cmppd(XMMRegister dst, XMMRegister nds, XMMRegister src, int cop, int vector_len) {
7331   assert(VM_Version::supports_avx(), "");
7332   assert(!VM_Version::supports_evex(), "");
7333   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
7334   int encode = simd_prefix_and_encode(dst, nds, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7335   emit_int8((unsigned char)0xC2);
7336   emit_int8((unsigned char)(0xC0 | encode));
7337   emit_int8((unsigned char)(0xF & cop));
7338 }
7339 
7340 void Assembler::vpblendd(XMMRegister dst, XMMRegister nds, XMMRegister src1, XMMRegister src2, int vector_len) {
7341   assert(VM_Version::supports_avx(), "");
7342   assert(!VM_Version::supports_evex(), "");
7343   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
7344   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src1->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
7345   emit_int8((unsigned char)0x4B);
7346   emit_int8((unsigned char)(0xC0 | encode));
7347   int src2_enc = src2->encoding();
7348   emit_int8((unsigned char)(0xF0 & src2_enc<<4));
7349 }
7350 
7351 void Assembler::shlxl(Register dst, Register src1, Register src2) {
7352   assert(VM_Version::supports_bmi2(), "");
7353   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
7354   int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src1->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
7355   emit_int8((unsigned char)0xF7);
7356   emit_int8((unsigned char)(0xC0 | encode));
7357 }
7358 
7359 void Assembler::shlxq(Register dst, Register src1, Register src2) {
7360   assert(VM_Version::supports_bmi2(), "");
7361   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
7362   int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src1->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
7363   emit_int8((unsigned char)0xF7);
7364   emit_int8((unsigned char)(0xC0 | encode));
7365 }
7366 
7367 #ifndef _LP64
7368 
7369 void Assembler::incl(Register dst) {
7370   // Don't use it directly. Use MacroAssembler::incrementl() instead.
7371   emit_int8(0x40 | dst->encoding());
7372 }
7373 
7374 void Assembler::lea(Register dst, Address src) {
7375   leal(dst, src);
7376 }
7377 
7378 void Assembler::mov_literal32(Address dst, int32_t imm32, RelocationHolder const& rspec) {
7379   InstructionMark im(this);
7380   emit_int8((unsigned char)0xC7);
7381   emit_operand(rax, dst);
7382   emit_data((int)imm32, rspec, 0);
7383 }
7384 
7385 void Assembler::mov_literal32(Register dst, int32_t imm32, RelocationHolder const& rspec) {
7386   InstructionMark im(this);
7387   int encode = prefix_and_encode(dst->encoding());
7388   emit_int8((unsigned char)(0xB8 | encode));
7389   emit_data((int)imm32, rspec, 0);
7390 }
7391 
7392 void Assembler::popa() { // 32bit
7393   emit_int8(0x61);
7394 }
7395 
7396 void Assembler::push_literal32(int32_t imm32, RelocationHolder const& rspec) {
7397   InstructionMark im(this);
7398   emit_int8(0x68);
7399   emit_data(imm32, rspec, 0);
7400 }
7401 
7402 void Assembler::pusha() { // 32bit
7403   emit_int8(0x60);
7404 }
7405 
7406 void Assembler::set_byte_if_not_zero(Register dst) {
7407   emit_int8(0x0F);
7408   emit_int8((unsigned char)0x95);
7409   emit_int8((unsigned char)(0xE0 | dst->encoding()));
7410 }
7411 
7412 void Assembler::shldl(Register dst, Register src) {
7413   emit_int8(0x0F);
7414   emit_int8((unsigned char)0xA5);
7415   emit_int8((unsigned char)(0xC0 | src->encoding() << 3 | dst->encoding()));
7416 }
7417 
7418 // 0F A4 / r ib
7419 void Assembler::shldl(Register dst, Register src, int8_t imm8) {
7420   emit_int8(0x0F);
7421   emit_int8((unsigned char)0xA4);
7422   emit_int8((unsigned char)(0xC0 | src->encoding() << 3 | dst->encoding()));
7423   emit_int8(imm8);
7424 }
7425 
7426 void Assembler::shrdl(Register dst, Register src) {
7427   emit_int8(0x0F);
7428   emit_int8((unsigned char)0xAD);
7429   emit_int8((unsigned char)(0xC0 | src->encoding() << 3 | dst->encoding()));
7430 }
7431 
7432 #else // LP64
7433 
7434 void Assembler::set_byte_if_not_zero(Register dst) {
7435   int enc = prefix_and_encode(dst->encoding(), true);
7436   emit_int8(0x0F);
7437   emit_int8((unsigned char)0x95);
7438   emit_int8((unsigned char)(0xE0 | enc));
7439 }
7440 
7441 // 64bit only pieces of the assembler
7442 // This should only be used by 64bit instructions that can use rip-relative
7443 // it cannot be used by instructions that want an immediate value.
7444 
7445 bool Assembler::reachable(AddressLiteral adr) {
7446   int64_t disp;
7447   // None will force a 64bit literal to the code stream. Likely a placeholder
7448   // for something that will be patched later and we need to certain it will
7449   // always be reachable.
7450   if (adr.reloc() == relocInfo::none) {
7451     return false;
7452   }
7453   if (adr.reloc() == relocInfo::internal_word_type) {
7454     // This should be rip relative and easily reachable.
7455     return true;
7456   }
7457   if (adr.reloc() == relocInfo::virtual_call_type ||
7458       adr.reloc() == relocInfo::opt_virtual_call_type ||
7459       adr.reloc() == relocInfo::static_call_type ||
7460       adr.reloc() == relocInfo::static_stub_type ) {
7461     // This should be rip relative within the code cache and easily
7462     // reachable until we get huge code caches. (At which point
7463     // ic code is going to have issues).
7464     return true;
7465   }
7466   if (adr.reloc() != relocInfo::external_word_type &&
7467       adr.reloc() != relocInfo::poll_return_type &&  // these are really external_word but need special
7468       adr.reloc() != relocInfo::poll_type &&         // relocs to identify them
7469       adr.reloc() != relocInfo::runtime_call_type ) {
7470     return false;
7471   }
7472 
7473   // Stress the correction code
7474   if (ForceUnreachable) {
7475     // Must be runtimecall reloc, see if it is in the codecache
7476     // Flipping stuff in the codecache to be unreachable causes issues
7477     // with things like inline caches where the additional instructions
7478     // are not handled.
7479     if (CodeCache::find_blob(adr._target) == NULL) {
7480       return false;
7481     }
7482   }
7483   // For external_word_type/runtime_call_type if it is reachable from where we
7484   // are now (possibly a temp buffer) and where we might end up
7485   // anywhere in the codeCache then we are always reachable.
7486   // This would have to change if we ever save/restore shared code
7487   // to be more pessimistic.
7488   disp = (int64_t)adr._target - ((int64_t)CodeCache::low_bound() + sizeof(int));
7489   if (!is_simm32(disp)) return false;
7490   disp = (int64_t)adr._target - ((int64_t)CodeCache::high_bound() + sizeof(int));
7491   if (!is_simm32(disp)) return false;
7492 
7493   disp = (int64_t)adr._target - ((int64_t)pc() + sizeof(int));
7494 
7495   // Because rip relative is a disp + address_of_next_instruction and we
7496   // don't know the value of address_of_next_instruction we apply a fudge factor
7497   // to make sure we will be ok no matter the size of the instruction we get placed into.
7498   // We don't have to fudge the checks above here because they are already worst case.
7499 
7500   // 12 == override/rex byte, opcode byte, rm byte, sib byte, a 4-byte disp , 4-byte literal
7501   // + 4 because better safe than sorry.
7502   const int fudge = 12 + 4;
7503   if (disp < 0) {
7504     disp -= fudge;
7505   } else {
7506     disp += fudge;
7507   }
7508   return is_simm32(disp);
7509 }
7510 
7511 // Check if the polling page is not reachable from the code cache using rip-relative
7512 // addressing.
7513 bool Assembler::is_polling_page_far() {
7514   intptr_t addr = (intptr_t)os::get_polling_page();
7515   return ForceUnreachable ||
7516          !is_simm32(addr - (intptr_t)CodeCache::low_bound()) ||
7517          !is_simm32(addr - (intptr_t)CodeCache::high_bound());
7518 }
7519 
7520 void Assembler::emit_data64(jlong data,
7521                             relocInfo::relocType rtype,
7522                             int format) {
7523   if (rtype == relocInfo::none) {
7524     emit_int64(data);
7525   } else {
7526     emit_data64(data, Relocation::spec_simple(rtype), format);
7527   }
7528 }
7529 
7530 void Assembler::emit_data64(jlong data,
7531                             RelocationHolder const& rspec,
7532                             int format) {
7533   assert(imm_operand == 0, "default format must be immediate in this file");
7534   assert(imm_operand == format, "must be immediate");
7535   assert(inst_mark() != NULL, "must be inside InstructionMark");
7536   // Do not use AbstractAssembler::relocate, which is not intended for
7537   // embedded words.  Instead, relocate to the enclosing instruction.
7538   code_section()->relocate(inst_mark(), rspec, format);
7539 #ifdef ASSERT
7540   check_relocation(rspec, format);
7541 #endif
7542   emit_int64(data);
7543 }
7544 
7545 int Assembler::prefix_and_encode(int reg_enc, bool byteinst) {
7546   if (reg_enc >= 8) {
7547     prefix(REX_B);
7548     reg_enc -= 8;
7549   } else if (byteinst && reg_enc >= 4) {
7550     prefix(REX);
7551   }
7552   return reg_enc;
7553 }
7554 
7555 int Assembler::prefixq_and_encode(int reg_enc) {
7556   if (reg_enc < 8) {
7557     prefix(REX_W);
7558   } else {
7559     prefix(REX_WB);
7560     reg_enc -= 8;
7561   }
7562   return reg_enc;
7563 }
7564 
7565 int Assembler::prefix_and_encode(int dst_enc, bool dst_is_byte, int src_enc, bool src_is_byte) {
7566   if (dst_enc < 8) {
7567     if (src_enc >= 8) {
7568       prefix(REX_B);
7569       src_enc -= 8;
7570     } else if ((src_is_byte && src_enc >= 4) || (dst_is_byte && dst_enc >= 4)) {
7571       prefix(REX);
7572     }
7573   } else {
7574     if (src_enc < 8) {
7575       prefix(REX_R);
7576     } else {
7577       prefix(REX_RB);
7578       src_enc -= 8;
7579     }
7580     dst_enc -= 8;
7581   }
7582   return dst_enc << 3 | src_enc;
7583 }
7584 
7585 int Assembler::prefixq_and_encode(int dst_enc, int src_enc) {
7586   if (dst_enc < 8) {
7587     if (src_enc < 8) {
7588       prefix(REX_W);
7589     } else {
7590       prefix(REX_WB);
7591       src_enc -= 8;
7592     }
7593   } else {
7594     if (src_enc < 8) {
7595       prefix(REX_WR);
7596     } else {
7597       prefix(REX_WRB);
7598       src_enc -= 8;
7599     }
7600     dst_enc -= 8;
7601   }
7602   return dst_enc << 3 | src_enc;
7603 }
7604 
7605 void Assembler::prefix(Register reg) {
7606   if (reg->encoding() >= 8) {
7607     prefix(REX_B);
7608   }
7609 }
7610 
7611 void Assembler::prefix(Register dst, Register src, Prefix p) {
7612   if (src->encoding() >= 8) {
7613     p = (Prefix)(p | REX_B);
7614   }
7615   if (dst->encoding() >= 8) {
7616     p = (Prefix)( p | REX_R);
7617   }
7618   if (p != Prefix_EMPTY) {
7619     // do not generate an empty prefix
7620     prefix(p);
7621   }
7622 }
7623 
7624 void Assembler::prefix(Register dst, Address adr, Prefix p) {
7625   if (adr.base_needs_rex()) {
7626     if (adr.index_needs_rex()) {
7627       assert(false, "prefix(Register dst, Address adr, Prefix p) does not support handling of an X");
7628     } else {
7629       prefix(REX_B);
7630     }
7631   } else {
7632     if (adr.index_needs_rex()) {
7633       assert(false, "prefix(Register dst, Address adr, Prefix p) does not support handling of an X");
7634     }
7635   }
7636   if (dst->encoding() >= 8) {
7637     p = (Prefix)(p | REX_R);
7638   }
7639   if (p != Prefix_EMPTY) {
7640     // do not generate an empty prefix
7641     prefix(p);
7642   }
7643 }
7644 
7645 void Assembler::prefix(Address adr) {
7646   if (adr.base_needs_rex()) {
7647     if (adr.index_needs_rex()) {
7648       prefix(REX_XB);
7649     } else {
7650       prefix(REX_B);
7651     }
7652   } else {
7653     if (adr.index_needs_rex()) {
7654       prefix(REX_X);
7655     }
7656   }
7657 }
7658 
7659 void Assembler::prefixq(Address adr) {
7660   if (adr.base_needs_rex()) {
7661     if (adr.index_needs_rex()) {
7662       prefix(REX_WXB);
7663     } else {
7664       prefix(REX_WB);
7665     }
7666   } else {
7667     if (adr.index_needs_rex()) {
7668       prefix(REX_WX);
7669     } else {
7670       prefix(REX_W);
7671     }
7672   }
7673 }
7674 
7675 
7676 void Assembler::prefix(Address adr, Register reg, bool byteinst) {
7677   if (reg->encoding() < 8) {
7678     if (adr.base_needs_rex()) {
7679       if (adr.index_needs_rex()) {
7680         prefix(REX_XB);
7681       } else {
7682         prefix(REX_B);
7683       }
7684     } else {
7685       if (adr.index_needs_rex()) {
7686         prefix(REX_X);
7687       } else if (byteinst && reg->encoding() >= 4 ) {
7688         prefix(REX);
7689       }
7690     }
7691   } else {
7692     if (adr.base_needs_rex()) {
7693       if (adr.index_needs_rex()) {
7694         prefix(REX_RXB);
7695       } else {
7696         prefix(REX_RB);
7697       }
7698     } else {
7699       if (adr.index_needs_rex()) {
7700         prefix(REX_RX);
7701       } else {
7702         prefix(REX_R);
7703       }
7704     }
7705   }
7706 }
7707 
7708 void Assembler::prefixq(Address adr, Register src) {
7709   if (src->encoding() < 8) {
7710     if (adr.base_needs_rex()) {
7711       if (adr.index_needs_rex()) {
7712         prefix(REX_WXB);
7713       } else {
7714         prefix(REX_WB);
7715       }
7716     } else {
7717       if (adr.index_needs_rex()) {
7718         prefix(REX_WX);
7719       } else {
7720         prefix(REX_W);
7721       }
7722     }
7723   } else {
7724     if (adr.base_needs_rex()) {
7725       if (adr.index_needs_rex()) {
7726         prefix(REX_WRXB);
7727       } else {
7728         prefix(REX_WRB);
7729       }
7730     } else {
7731       if (adr.index_needs_rex()) {
7732         prefix(REX_WRX);
7733       } else {
7734         prefix(REX_WR);
7735       }
7736     }
7737   }
7738 }
7739 
7740 void Assembler::prefix(Address adr, XMMRegister reg) {
7741   if (reg->encoding() < 8) {
7742     if (adr.base_needs_rex()) {
7743       if (adr.index_needs_rex()) {
7744         prefix(REX_XB);
7745       } else {
7746         prefix(REX_B);
7747       }
7748     } else {
7749       if (adr.index_needs_rex()) {
7750         prefix(REX_X);
7751       }
7752     }
7753   } else {
7754     if (adr.base_needs_rex()) {
7755       if (adr.index_needs_rex()) {
7756         prefix(REX_RXB);
7757       } else {
7758         prefix(REX_RB);
7759       }
7760     } else {
7761       if (adr.index_needs_rex()) {
7762         prefix(REX_RX);
7763       } else {
7764         prefix(REX_R);
7765       }
7766     }
7767   }
7768 }
7769 
7770 void Assembler::prefixq(Address adr, XMMRegister src) {
7771   if (src->encoding() < 8) {
7772     if (adr.base_needs_rex()) {
7773       if (adr.index_needs_rex()) {
7774         prefix(REX_WXB);
7775       } else {
7776         prefix(REX_WB);
7777       }
7778     } else {
7779       if (adr.index_needs_rex()) {
7780         prefix(REX_WX);
7781       } else {
7782         prefix(REX_W);
7783       }
7784     }
7785   } else {
7786     if (adr.base_needs_rex()) {
7787       if (adr.index_needs_rex()) {
7788         prefix(REX_WRXB);
7789       } else {
7790         prefix(REX_WRB);
7791       }
7792     } else {
7793       if (adr.index_needs_rex()) {
7794         prefix(REX_WRX);
7795       } else {
7796         prefix(REX_WR);
7797       }
7798     }
7799   }
7800 }
7801 
7802 void Assembler::adcq(Register dst, int32_t imm32) {
7803   (void) prefixq_and_encode(dst->encoding());
7804   emit_arith(0x81, 0xD0, dst, imm32);
7805 }
7806 
7807 void Assembler::adcq(Register dst, Address src) {
7808   InstructionMark im(this);
7809   prefixq(src, dst);
7810   emit_int8(0x13);
7811   emit_operand(dst, src);
7812 }
7813 
7814 void Assembler::adcq(Register dst, Register src) {
7815   (void) prefixq_and_encode(dst->encoding(), src->encoding());
7816   emit_arith(0x13, 0xC0, dst, src);
7817 }
7818 
7819 void Assembler::addq(Address dst, int32_t imm32) {
7820   InstructionMark im(this);
7821   prefixq(dst);
7822   emit_arith_operand(0x81, rax, dst,imm32);
7823 }
7824 
7825 void Assembler::addq(Address dst, Register src) {
7826   InstructionMark im(this);
7827   prefixq(dst, src);
7828   emit_int8(0x01);
7829   emit_operand(src, dst);
7830 }
7831 
7832 void Assembler::addq(Register dst, int32_t imm32) {
7833   (void) prefixq_and_encode(dst->encoding());
7834   emit_arith(0x81, 0xC0, dst, imm32);
7835 }
7836 
7837 void Assembler::addq(Register dst, Address src) {
7838   InstructionMark im(this);
7839   prefixq(src, dst);
7840   emit_int8(0x03);
7841   emit_operand(dst, src);
7842 }
7843 
7844 void Assembler::addq(Register dst, Register src) {
7845   (void) prefixq_and_encode(dst->encoding(), src->encoding());
7846   emit_arith(0x03, 0xC0, dst, src);
7847 }
7848 
7849 void Assembler::adcxq(Register dst, Register src) {
7850   //assert(VM_Version::supports_adx(), "adx instructions not supported");
7851   emit_int8((unsigned char)0x66);
7852   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
7853   emit_int8(0x0F);
7854   emit_int8(0x38);
7855   emit_int8((unsigned char)0xF6);
7856   emit_int8((unsigned char)(0xC0 | encode));
7857 }
7858 
7859 void Assembler::adoxq(Register dst, Register src) {
7860   //assert(VM_Version::supports_adx(), "adx instructions not supported");
7861   emit_int8((unsigned char)0xF3);
7862   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
7863   emit_int8(0x0F);
7864   emit_int8(0x38);
7865   emit_int8((unsigned char)0xF6);
7866   emit_int8((unsigned char)(0xC0 | encode));
7867 }
7868 
7869 void Assembler::andq(Address dst, int32_t imm32) {
7870   InstructionMark im(this);
7871   prefixq(dst);
7872   emit_int8((unsigned char)0x81);
7873   emit_operand(rsp, dst, 4);
7874   emit_int32(imm32);
7875 }
7876 
7877 void Assembler::andq(Register dst, int32_t imm32) {
7878   (void) prefixq_and_encode(dst->encoding());
7879   emit_arith(0x81, 0xE0, dst, imm32);
7880 }
7881 
7882 void Assembler::andq(Register dst, Address src) {
7883   InstructionMark im(this);
7884   prefixq(src, dst);
7885   emit_int8(0x23);
7886   emit_operand(dst, src);
7887 }
7888 
7889 void Assembler::andq(Register dst, Register src) {
7890   (void) prefixq_and_encode(dst->encoding(), src->encoding());
7891   emit_arith(0x23, 0xC0, dst, src);
7892 }
7893 
7894 void Assembler::andnq(Register dst, Register src1, Register src2) {
7895   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
7896   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
7897   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
7898   emit_int8((unsigned char)0xF2);
7899   emit_int8((unsigned char)(0xC0 | encode));
7900 }
7901 
7902 void Assembler::andnq(Register dst, Register src1, Address src2) {
7903   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
7904   InstructionMark im(this);
7905   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
7906   vex_prefix(src2, src1->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
7907   emit_int8((unsigned char)0xF2);
7908   emit_operand(dst, src2);
7909 }
7910 
7911 void Assembler::bsfq(Register dst, Register src) {
7912   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
7913   emit_int8(0x0F);
7914   emit_int8((unsigned char)0xBC);
7915   emit_int8((unsigned char)(0xC0 | encode));
7916 }
7917 
7918 void Assembler::bsrq(Register dst, Register src) {
7919   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
7920   emit_int8(0x0F);
7921   emit_int8((unsigned char)0xBD);
7922   emit_int8((unsigned char)(0xC0 | encode));
7923 }
7924 
7925 void Assembler::bswapq(Register reg) {
7926   int encode = prefixq_and_encode(reg->encoding());
7927   emit_int8(0x0F);
7928   emit_int8((unsigned char)(0xC8 | encode));
7929 }
7930 
7931 void Assembler::blsiq(Register dst, Register src) {
7932   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
7933   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
7934   int encode = vex_prefix_and_encode(rbx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
7935   emit_int8((unsigned char)0xF3);
7936   emit_int8((unsigned char)(0xC0 | encode));
7937 }
7938 
7939 void Assembler::blsiq(Register dst, Address src) {
7940   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
7941   InstructionMark im(this);
7942   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
7943   vex_prefix(src, dst->encoding(), rbx->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
7944   emit_int8((unsigned char)0xF3);
7945   emit_operand(rbx, src);
7946 }
7947 
7948 void Assembler::blsmskq(Register dst, Register src) {
7949   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
7950   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
7951   int encode = vex_prefix_and_encode(rdx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
7952   emit_int8((unsigned char)0xF3);
7953   emit_int8((unsigned char)(0xC0 | encode));
7954 }
7955 
7956 void Assembler::blsmskq(Register dst, Address src) {
7957   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
7958   InstructionMark im(this);
7959   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
7960   vex_prefix(src, dst->encoding(), rdx->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
7961   emit_int8((unsigned char)0xF3);
7962   emit_operand(rdx, src);
7963 }
7964 
7965 void Assembler::blsrq(Register dst, Register src) {
7966   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
7967   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
7968   int encode = vex_prefix_and_encode(rcx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
7969   emit_int8((unsigned char)0xF3);
7970   emit_int8((unsigned char)(0xC0 | encode));
7971 }
7972 
7973 void Assembler::blsrq(Register dst, Address src) {
7974   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
7975   InstructionMark im(this);
7976   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
7977   vex_prefix(src, dst->encoding(), rcx->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
7978   emit_int8((unsigned char)0xF3);
7979   emit_operand(rcx, src);
7980 }
7981 
7982 void Assembler::cdqq() {
7983   prefix(REX_W);
7984   emit_int8((unsigned char)0x99);
7985 }
7986 
7987 void Assembler::clflush(Address adr) {
7988   prefix(adr);
7989   emit_int8(0x0F);
7990   emit_int8((unsigned char)0xAE);
7991   emit_operand(rdi, adr);
7992 }
7993 
7994 void Assembler::cmovq(Condition cc, Register dst, Register src) {
7995   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
7996   emit_int8(0x0F);
7997   emit_int8(0x40 | cc);
7998   emit_int8((unsigned char)(0xC0 | encode));
7999 }
8000 
8001 void Assembler::cmovq(Condition cc, Register dst, Address src) {
8002   InstructionMark im(this);
8003   prefixq(src, dst);
8004   emit_int8(0x0F);
8005   emit_int8(0x40 | cc);
8006   emit_operand(dst, src);
8007 }
8008 
8009 void Assembler::cmpq(Address dst, int32_t imm32) {
8010   InstructionMark im(this);
8011   prefixq(dst);
8012   emit_int8((unsigned char)0x81);
8013   emit_operand(rdi, dst, 4);
8014   emit_int32(imm32);
8015 }
8016 
8017 void Assembler::cmpq(Register dst, int32_t imm32) {
8018   (void) prefixq_and_encode(dst->encoding());
8019   emit_arith(0x81, 0xF8, dst, imm32);
8020 }
8021 
8022 void Assembler::cmpq(Address dst, Register src) {
8023   InstructionMark im(this);
8024   prefixq(dst, src);
8025   emit_int8(0x3B);
8026   emit_operand(src, dst);
8027 }
8028 
8029 void Assembler::cmpq(Register dst, Register src) {
8030   (void) prefixq_and_encode(dst->encoding(), src->encoding());
8031   emit_arith(0x3B, 0xC0, dst, src);
8032 }
8033 
8034 void Assembler::cmpq(Register dst, Address  src) {
8035   InstructionMark im(this);
8036   prefixq(src, dst);
8037   emit_int8(0x3B);
8038   emit_operand(dst, src);
8039 }
8040 
8041 void Assembler::cmpxchgq(Register reg, Address adr) {
8042   InstructionMark im(this);
8043   prefixq(adr, reg);
8044   emit_int8(0x0F);
8045   emit_int8((unsigned char)0xB1);
8046   emit_operand(reg, adr);
8047 }
8048 
8049 void Assembler::cvtsi2sdq(XMMRegister dst, Register src) {
8050   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
8051   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
8052   int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
8053   emit_int8(0x2A);
8054   emit_int8((unsigned char)(0xC0 | encode));
8055 }
8056 
8057 void Assembler::cvtsi2sdq(XMMRegister dst, Address src) {
8058   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
8059   InstructionMark im(this);
8060   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
8061   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
8062   simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
8063   emit_int8(0x2A);
8064   emit_operand(dst, src);
8065 }
8066 
8067 void Assembler::cvtsi2ssq(XMMRegister dst, Address src) {
8068   NOT_LP64(assert(VM_Version::supports_sse(), ""));
8069   InstructionMark im(this);
8070   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
8071   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
8072   simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
8073   emit_int8(0x2A);
8074   emit_operand(dst, src);
8075 }
8076 
8077 void Assembler::cvttsd2siq(Register dst, XMMRegister src) {
8078   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
8079   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
8080   int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
8081   emit_int8(0x2C);
8082   emit_int8((unsigned char)(0xC0 | encode));
8083 }
8084 
8085 void Assembler::cvttss2siq(Register dst, XMMRegister src) {
8086   NOT_LP64(assert(VM_Version::supports_sse(), ""));
8087   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
8088   int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
8089   emit_int8(0x2C);
8090   emit_int8((unsigned char)(0xC0 | encode));
8091 }
8092 
8093 void Assembler::decl(Register dst) {
8094   // Don't use it directly. Use MacroAssembler::decrementl() instead.
8095   // Use two-byte form (one-byte form is a REX prefix in 64-bit mode)
8096   int encode = prefix_and_encode(dst->encoding());
8097   emit_int8((unsigned char)0xFF);
8098   emit_int8((unsigned char)(0xC8 | encode));
8099 }
8100 
8101 void Assembler::decq(Register dst) {
8102   // Don't use it directly. Use MacroAssembler::decrementq() instead.
8103   // Use two-byte form (one-byte from is a REX prefix in 64-bit mode)
8104   int encode = prefixq_and_encode(dst->encoding());
8105   emit_int8((unsigned char)0xFF);
8106   emit_int8(0xC8 | encode);
8107 }
8108 
8109 void Assembler::decq(Address dst) {
8110   // Don't use it directly. Use MacroAssembler::decrementq() instead.
8111   InstructionMark im(this);
8112   prefixq(dst);
8113   emit_int8((unsigned char)0xFF);
8114   emit_operand(rcx, dst);
8115 }
8116 
8117 void Assembler::fxrstor(Address src) {
8118   prefixq(src);
8119   emit_int8(0x0F);
8120   emit_int8((unsigned char)0xAE);
8121   emit_operand(as_Register(1), src);
8122 }
8123 
8124 void Assembler::xrstor(Address src) {
8125   prefixq(src);
8126   emit_int8(0x0F);
8127   emit_int8((unsigned char)0xAE);
8128   emit_operand(as_Register(5), src);
8129 }
8130 
8131 void Assembler::fxsave(Address dst) {
8132   prefixq(dst);
8133   emit_int8(0x0F);
8134   emit_int8((unsigned char)0xAE);
8135   emit_operand(as_Register(0), dst);
8136 }
8137 
8138 void Assembler::xsave(Address dst) {
8139   prefixq(dst);
8140   emit_int8(0x0F);
8141   emit_int8((unsigned char)0xAE);
8142   emit_operand(as_Register(4), dst);
8143 }
8144 
8145 void Assembler::idivq(Register src) {
8146   int encode = prefixq_and_encode(src->encoding());
8147   emit_int8((unsigned char)0xF7);
8148   emit_int8((unsigned char)(0xF8 | encode));
8149 }
8150 
8151 void Assembler::imulq(Register dst, Register src) {
8152   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
8153   emit_int8(0x0F);
8154   emit_int8((unsigned char)0xAF);
8155   emit_int8((unsigned char)(0xC0 | encode));
8156 }
8157 
8158 void Assembler::imulq(Register dst, Register src, int value) {
8159   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
8160   if (is8bit(value)) {
8161     emit_int8(0x6B);
8162     emit_int8((unsigned char)(0xC0 | encode));
8163     emit_int8(value & 0xFF);
8164   } else {
8165     emit_int8(0x69);
8166     emit_int8((unsigned char)(0xC0 | encode));
8167     emit_int32(value);
8168   }
8169 }
8170 
8171 void Assembler::imulq(Register dst, Address src) {
8172   InstructionMark im(this);
8173   prefixq(src, dst);
8174   emit_int8(0x0F);
8175   emit_int8((unsigned char) 0xAF);
8176   emit_operand(dst, src);
8177 }
8178 
8179 void Assembler::incl(Register dst) {
8180   // Don't use it directly. Use MacroAssembler::incrementl() instead.
8181   // Use two-byte form (one-byte from is a REX prefix in 64-bit mode)
8182   int encode = prefix_and_encode(dst->encoding());
8183   emit_int8((unsigned char)0xFF);
8184   emit_int8((unsigned char)(0xC0 | encode));
8185 }
8186 
8187 void Assembler::incq(Register dst) {
8188   // Don't use it directly. Use MacroAssembler::incrementq() instead.
8189   // Use two-byte form (one-byte from is a REX prefix in 64-bit mode)
8190   int encode = prefixq_and_encode(dst->encoding());
8191   emit_int8((unsigned char)0xFF);
8192   emit_int8((unsigned char)(0xC0 | encode));
8193 }
8194 
8195 void Assembler::incq(Address dst) {
8196   // Don't use it directly. Use MacroAssembler::incrementq() instead.
8197   InstructionMark im(this);
8198   prefixq(dst);
8199   emit_int8((unsigned char)0xFF);
8200   emit_operand(rax, dst);
8201 }
8202 
8203 void Assembler::lea(Register dst, Address src) {
8204   leaq(dst, src);
8205 }
8206 
8207 void Assembler::leaq(Register dst, Address src) {
8208   InstructionMark im(this);
8209   prefixq(src, dst);
8210   emit_int8((unsigned char)0x8D);
8211   emit_operand(dst, src);
8212 }
8213 
8214 void Assembler::mov64(Register dst, int64_t imm64) {
8215   InstructionMark im(this);
8216   int encode = prefixq_and_encode(dst->encoding());
8217   emit_int8((unsigned char)(0xB8 | encode));
8218   emit_int64(imm64);
8219 }
8220 
8221 void Assembler::mov_literal64(Register dst, intptr_t imm64, RelocationHolder const& rspec) {
8222   InstructionMark im(this);
8223   int encode = prefixq_and_encode(dst->encoding());
8224   emit_int8(0xB8 | encode);
8225   emit_data64(imm64, rspec);
8226 }
8227 
8228 void Assembler::mov_narrow_oop(Register dst, int32_t imm32, RelocationHolder const& rspec) {
8229   InstructionMark im(this);
8230   int encode = prefix_and_encode(dst->encoding());
8231   emit_int8((unsigned char)(0xB8 | encode));
8232   emit_data((int)imm32, rspec, narrow_oop_operand);
8233 }
8234 
8235 void Assembler::mov_narrow_oop(Address dst, int32_t imm32,  RelocationHolder const& rspec) {
8236   InstructionMark im(this);
8237   prefix(dst);
8238   emit_int8((unsigned char)0xC7);
8239   emit_operand(rax, dst, 4);
8240   emit_data((int)imm32, rspec, narrow_oop_operand);
8241 }
8242 
8243 void Assembler::cmp_narrow_oop(Register src1, int32_t imm32, RelocationHolder const& rspec) {
8244   InstructionMark im(this);
8245   int encode = prefix_and_encode(src1->encoding());
8246   emit_int8((unsigned char)0x81);
8247   emit_int8((unsigned char)(0xF8 | encode));
8248   emit_data((int)imm32, rspec, narrow_oop_operand);
8249 }
8250 
8251 void Assembler::cmp_narrow_oop(Address src1, int32_t imm32, RelocationHolder const& rspec) {
8252   InstructionMark im(this);
8253   prefix(src1);
8254   emit_int8((unsigned char)0x81);
8255   emit_operand(rax, src1, 4);
8256   emit_data((int)imm32, rspec, narrow_oop_operand);
8257 }
8258 
8259 void Assembler::lzcntq(Register dst, Register src) {
8260   assert(VM_Version::supports_lzcnt(), "encoding is treated as BSR");
8261   emit_int8((unsigned char)0xF3);
8262   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
8263   emit_int8(0x0F);
8264   emit_int8((unsigned char)0xBD);
8265   emit_int8((unsigned char)(0xC0 | encode));
8266 }
8267 
8268 void Assembler::movdq(XMMRegister dst, Register src) {
8269   // table D-1 says MMX/SSE2
8270   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
8271   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
8272   int encode = simd_prefix_and_encode(dst, xnoreg, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
8273   emit_int8(0x6E);
8274   emit_int8((unsigned char)(0xC0 | encode));
8275 }
8276 
8277 void Assembler::movdq(Register dst, XMMRegister src) {
8278   // table D-1 says MMX/SSE2
8279   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
8280   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
8281   // swap src/dst to get correct prefix
8282   int encode = simd_prefix_and_encode(src, xnoreg, as_XMMRegister(dst->encoding()), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
8283   emit_int8(0x7E);
8284   emit_int8((unsigned char)(0xC0 | encode));
8285 }
8286 
8287 void Assembler::movq(Register dst, Register src) {
8288   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
8289   emit_int8((unsigned char)0x8B);
8290   emit_int8((unsigned char)(0xC0 | encode));
8291 }
8292 
8293 void Assembler::movq(Register dst, Address src) {
8294   InstructionMark im(this);
8295   prefixq(src, dst);
8296   emit_int8((unsigned char)0x8B);
8297   emit_operand(dst, src);
8298 }
8299 
8300 void Assembler::movq(Address dst, Register src) {
8301   InstructionMark im(this);
8302   prefixq(dst, src);
8303   emit_int8((unsigned char)0x89);
8304   emit_operand(src, dst);
8305 }
8306 
8307 void Assembler::movsbq(Register dst, Address src) {
8308   InstructionMark im(this);
8309   prefixq(src, dst);
8310   emit_int8(0x0F);
8311   emit_int8((unsigned char)0xBE);
8312   emit_operand(dst, src);
8313 }
8314 
8315 void Assembler::movsbq(Register dst, Register src) {
8316   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
8317   emit_int8(0x0F);
8318   emit_int8((unsigned char)0xBE);
8319   emit_int8((unsigned char)(0xC0 | encode));
8320 }
8321 
8322 void Assembler::movslq(Register dst, int32_t imm32) {
8323   // dbx shows movslq(rcx, 3) as movq     $0x0000000049000000,(%rbx)
8324   // and movslq(r8, 3); as movl     $0x0000000048000000,(%rbx)
8325   // as a result we shouldn't use until tested at runtime...
8326   ShouldNotReachHere();
8327   InstructionMark im(this);
8328   int encode = prefixq_and_encode(dst->encoding());
8329   emit_int8((unsigned char)(0xC7 | encode));
8330   emit_int32(imm32);
8331 }
8332 
8333 void Assembler::movslq(Address dst, int32_t imm32) {
8334   assert(is_simm32(imm32), "lost bits");
8335   InstructionMark im(this);
8336   prefixq(dst);
8337   emit_int8((unsigned char)0xC7);
8338   emit_operand(rax, dst, 4);
8339   emit_int32(imm32);
8340 }
8341 
8342 void Assembler::movslq(Register dst, Address src) {
8343   InstructionMark im(this);
8344   prefixq(src, dst);
8345   emit_int8(0x63);
8346   emit_operand(dst, src);
8347 }
8348 
8349 void Assembler::movslq(Register dst, Register src) {
8350   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
8351   emit_int8(0x63);
8352   emit_int8((unsigned char)(0xC0 | encode));
8353 }
8354 
8355 void Assembler::movswq(Register dst, Address src) {
8356   InstructionMark im(this);
8357   prefixq(src, dst);
8358   emit_int8(0x0F);
8359   emit_int8((unsigned char)0xBF);
8360   emit_operand(dst, src);
8361 }
8362 
8363 void Assembler::movswq(Register dst, Register src) {
8364   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
8365   emit_int8((unsigned char)0x0F);
8366   emit_int8((unsigned char)0xBF);
8367   emit_int8((unsigned char)(0xC0 | encode));
8368 }
8369 
8370 void Assembler::movzbq(Register dst, Address src) {
8371   InstructionMark im(this);
8372   prefixq(src, dst);
8373   emit_int8((unsigned char)0x0F);
8374   emit_int8((unsigned char)0xB6);
8375   emit_operand(dst, src);
8376 }
8377 
8378 void Assembler::movzbq(Register dst, Register src) {
8379   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
8380   emit_int8(0x0F);
8381   emit_int8((unsigned char)0xB6);
8382   emit_int8(0xC0 | encode);
8383 }
8384 
8385 void Assembler::movzwq(Register dst, Address src) {
8386   InstructionMark im(this);
8387   prefixq(src, dst);
8388   emit_int8((unsigned char)0x0F);
8389   emit_int8((unsigned char)0xB7);
8390   emit_operand(dst, src);
8391 }
8392 
8393 void Assembler::movzwq(Register dst, Register src) {
8394   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
8395   emit_int8((unsigned char)0x0F);
8396   emit_int8((unsigned char)0xB7);
8397   emit_int8((unsigned char)(0xC0 | encode));
8398 }
8399 
8400 void Assembler::mulq(Address src) {
8401   InstructionMark im(this);
8402   prefixq(src);
8403   emit_int8((unsigned char)0xF7);
8404   emit_operand(rsp, src);
8405 }
8406 
8407 void Assembler::mulq(Register src) {
8408   int encode = prefixq_and_encode(src->encoding());
8409   emit_int8((unsigned char)0xF7);
8410   emit_int8((unsigned char)(0xE0 | encode));
8411 }
8412 
8413 void Assembler::mulxq(Register dst1, Register dst2, Register src) {
8414   assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
8415   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
8416   int encode = vex_prefix_and_encode(dst1->encoding(), dst2->encoding(), src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_38, &attributes);
8417   emit_int8((unsigned char)0xF6);
8418   emit_int8((unsigned char)(0xC0 | encode));
8419 }
8420 
8421 void Assembler::negq(Register dst) {
8422   int encode = prefixq_and_encode(dst->encoding());
8423   emit_int8((unsigned char)0xF7);
8424   emit_int8((unsigned char)(0xD8 | encode));
8425 }
8426 
8427 void Assembler::notq(Register dst) {
8428   int encode = prefixq_and_encode(dst->encoding());
8429   emit_int8((unsigned char)0xF7);
8430   emit_int8((unsigned char)(0xD0 | encode));
8431 }
8432 
8433 void Assembler::orq(Address dst, int32_t imm32) {
8434   InstructionMark im(this);
8435   prefixq(dst);
8436   emit_int8((unsigned char)0x81);
8437   emit_operand(rcx, dst, 4);
8438   emit_int32(imm32);
8439 }
8440 
8441 void Assembler::orq(Register dst, int32_t imm32) {
8442   (void) prefixq_and_encode(dst->encoding());
8443   emit_arith(0x81, 0xC8, dst, imm32);
8444 }
8445 
8446 void Assembler::orq(Register dst, Address src) {
8447   InstructionMark im(this);
8448   prefixq(src, dst);
8449   emit_int8(0x0B);
8450   emit_operand(dst, src);
8451 }
8452 
8453 void Assembler::orq(Register dst, Register src) {
8454   (void) prefixq_and_encode(dst->encoding(), src->encoding());
8455   emit_arith(0x0B, 0xC0, dst, src);
8456 }
8457 
8458 void Assembler::popa() { // 64bit
8459   movq(r15, Address(rsp, 0));
8460   movq(r14, Address(rsp, wordSize));
8461   movq(r13, Address(rsp, 2 * wordSize));
8462   movq(r12, Address(rsp, 3 * wordSize));
8463   movq(r11, Address(rsp, 4 * wordSize));
8464   movq(r10, Address(rsp, 5 * wordSize));
8465   movq(r9,  Address(rsp, 6 * wordSize));
8466   movq(r8,  Address(rsp, 7 * wordSize));
8467   movq(rdi, Address(rsp, 8 * wordSize));
8468   movq(rsi, Address(rsp, 9 * wordSize));
8469   movq(rbp, Address(rsp, 10 * wordSize));
8470   // skip rsp
8471   movq(rbx, Address(rsp, 12 * wordSize));
8472   movq(rdx, Address(rsp, 13 * wordSize));
8473   movq(rcx, Address(rsp, 14 * wordSize));
8474   movq(rax, Address(rsp, 15 * wordSize));
8475 
8476   addq(rsp, 16 * wordSize);
8477 }
8478 
8479 void Assembler::popcntq(Register dst, Address src) {
8480   assert(VM_Version::supports_popcnt(), "must support");
8481   InstructionMark im(this);
8482   emit_int8((unsigned char)0xF3);
8483   prefixq(src, dst);
8484   emit_int8((unsigned char)0x0F);
8485   emit_int8((unsigned char)0xB8);
8486   emit_operand(dst, src);
8487 }
8488 
8489 void Assembler::popcntq(Register dst, Register src) {
8490   assert(VM_Version::supports_popcnt(), "must support");
8491   emit_int8((unsigned char)0xF3);
8492   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
8493   emit_int8((unsigned char)0x0F);
8494   emit_int8((unsigned char)0xB8);
8495   emit_int8((unsigned char)(0xC0 | encode));
8496 }
8497 
8498 void Assembler::popq(Address dst) {
8499   InstructionMark im(this);
8500   prefixq(dst);
8501   emit_int8((unsigned char)0x8F);
8502   emit_operand(rax, dst);
8503 }
8504 
8505 void Assembler::pusha() { // 64bit
8506   // we have to store original rsp.  ABI says that 128 bytes
8507   // below rsp are local scratch.
8508   movq(Address(rsp, -5 * wordSize), rsp);
8509 
8510   subq(rsp, 16 * wordSize);
8511 
8512   movq(Address(rsp, 15 * wordSize), rax);
8513   movq(Address(rsp, 14 * wordSize), rcx);
8514   movq(Address(rsp, 13 * wordSize), rdx);
8515   movq(Address(rsp, 12 * wordSize), rbx);
8516   // skip rsp
8517   movq(Address(rsp, 10 * wordSize), rbp);
8518   movq(Address(rsp, 9 * wordSize), rsi);
8519   movq(Address(rsp, 8 * wordSize), rdi);
8520   movq(Address(rsp, 7 * wordSize), r8);
8521   movq(Address(rsp, 6 * wordSize), r9);
8522   movq(Address(rsp, 5 * wordSize), r10);
8523   movq(Address(rsp, 4 * wordSize), r11);
8524   movq(Address(rsp, 3 * wordSize), r12);
8525   movq(Address(rsp, 2 * wordSize), r13);
8526   movq(Address(rsp, wordSize), r14);
8527   movq(Address(rsp, 0), r15);
8528 }
8529 
8530 void Assembler::pushq(Address src) {
8531   InstructionMark im(this);
8532   prefixq(src);
8533   emit_int8((unsigned char)0xFF);
8534   emit_operand(rsi, src);
8535 }
8536 
8537 void Assembler::rclq(Register dst, int imm8) {
8538   assert(isShiftCount(imm8 >> 1), "illegal shift count");
8539   int encode = prefixq_and_encode(dst->encoding());
8540   if (imm8 == 1) {
8541     emit_int8((unsigned char)0xD1);
8542     emit_int8((unsigned char)(0xD0 | encode));
8543   } else {
8544     emit_int8((unsigned char)0xC1);
8545     emit_int8((unsigned char)(0xD0 | encode));
8546     emit_int8(imm8);
8547   }
8548 }
8549 
8550 void Assembler::rcrq(Register dst, int imm8) {
8551   assert(isShiftCount(imm8 >> 1), "illegal shift count");
8552   int encode = prefixq_and_encode(dst->encoding());
8553   if (imm8 == 1) {
8554     emit_int8((unsigned char)0xD1);
8555     emit_int8((unsigned char)(0xD8 | encode));
8556   } else {
8557     emit_int8((unsigned char)0xC1);
8558     emit_int8((unsigned char)(0xD8 | encode));
8559     emit_int8(imm8);
8560   }
8561 }
8562 
8563 void Assembler::rorq(Register dst, int imm8) {
8564   assert(isShiftCount(imm8 >> 1), "illegal shift count");
8565   int encode = prefixq_and_encode(dst->encoding());
8566   if (imm8 == 1) {
8567     emit_int8((unsigned char)0xD1);
8568     emit_int8((unsigned char)(0xC8 | encode));
8569   } else {
8570     emit_int8((unsigned char)0xC1);
8571     emit_int8((unsigned char)(0xc8 | encode));
8572     emit_int8(imm8);
8573   }
8574 }
8575 
8576 void Assembler::rorxq(Register dst, Register src, int imm8) {
8577   assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
8578   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
8579   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_3A, &attributes);
8580   emit_int8((unsigned char)0xF0);
8581   emit_int8((unsigned char)(0xC0 | encode));
8582   emit_int8(imm8);
8583 }
8584 
8585 void Assembler::rorxd(Register dst, Register src, int imm8) {
8586   assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
8587   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
8588   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_3A, &attributes);
8589   emit_int8((unsigned char)0xF0);
8590   emit_int8((unsigned char)(0xC0 | encode));
8591   emit_int8(imm8);
8592 }
8593 
8594 void Assembler::sarq(Register dst, int imm8) {
8595   assert(isShiftCount(imm8 >> 1), "illegal shift count");
8596   int encode = prefixq_and_encode(dst->encoding());
8597   if (imm8 == 1) {
8598     emit_int8((unsigned char)0xD1);
8599     emit_int8((unsigned char)(0xF8 | encode));
8600   } else {
8601     emit_int8((unsigned char)0xC1);
8602     emit_int8((unsigned char)(0xF8 | encode));
8603     emit_int8(imm8);
8604   }
8605 }
8606 
8607 void Assembler::sarq(Register dst) {
8608   int encode = prefixq_and_encode(dst->encoding());
8609   emit_int8((unsigned char)0xD3);
8610   emit_int8((unsigned char)(0xF8 | encode));
8611 }
8612 
8613 void Assembler::sbbq(Address dst, int32_t imm32) {
8614   InstructionMark im(this);
8615   prefixq(dst);
8616   emit_arith_operand(0x81, rbx, dst, imm32);
8617 }
8618 
8619 void Assembler::sbbq(Register dst, int32_t imm32) {
8620   (void) prefixq_and_encode(dst->encoding());
8621   emit_arith(0x81, 0xD8, dst, imm32);
8622 }
8623 
8624 void Assembler::sbbq(Register dst, Address src) {
8625   InstructionMark im(this);
8626   prefixq(src, dst);
8627   emit_int8(0x1B);
8628   emit_operand(dst, src);
8629 }
8630 
8631 void Assembler::sbbq(Register dst, Register src) {
8632   (void) prefixq_and_encode(dst->encoding(), src->encoding());
8633   emit_arith(0x1B, 0xC0, dst, src);
8634 }
8635 
8636 void Assembler::shlq(Register dst, int imm8) {
8637   assert(isShiftCount(imm8 >> 1), "illegal shift count");
8638   int encode = prefixq_and_encode(dst->encoding());
8639   if (imm8 == 1) {
8640     emit_int8((unsigned char)0xD1);
8641     emit_int8((unsigned char)(0xE0 | encode));
8642   } else {
8643     emit_int8((unsigned char)0xC1);
8644     emit_int8((unsigned char)(0xE0 | encode));
8645     emit_int8(imm8);
8646   }
8647 }
8648 
8649 void Assembler::shlq(Register dst) {
8650   int encode = prefixq_and_encode(dst->encoding());
8651   emit_int8((unsigned char)0xD3);
8652   emit_int8((unsigned char)(0xE0 | encode));
8653 }
8654 
8655 void Assembler::shrq(Register dst, int imm8) {
8656   assert(isShiftCount(imm8 >> 1), "illegal shift count");
8657   int encode = prefixq_and_encode(dst->encoding());
8658   emit_int8((unsigned char)0xC1);
8659   emit_int8((unsigned char)(0xE8 | encode));
8660   emit_int8(imm8);
8661 }
8662 
8663 void Assembler::shrq(Register dst) {
8664   int encode = prefixq_and_encode(dst->encoding());
8665   emit_int8((unsigned char)0xD3);
8666   emit_int8(0xE8 | encode);
8667 }
8668 
8669 void Assembler::subq(Address dst, int32_t imm32) {
8670   InstructionMark im(this);
8671   prefixq(dst);
8672   emit_arith_operand(0x81, rbp, dst, imm32);
8673 }
8674 
8675 void Assembler::subq(Address dst, Register src) {
8676   InstructionMark im(this);
8677   prefixq(dst, src);
8678   emit_int8(0x29);
8679   emit_operand(src, dst);
8680 }
8681 
8682 void Assembler::subq(Register dst, int32_t imm32) {
8683   (void) prefixq_and_encode(dst->encoding());
8684   emit_arith(0x81, 0xE8, dst, imm32);
8685 }
8686 
8687 // Force generation of a 4 byte immediate value even if it fits into 8bit
8688 void Assembler::subq_imm32(Register dst, int32_t imm32) {
8689   (void) prefixq_and_encode(dst->encoding());
8690   emit_arith_imm32(0x81, 0xE8, dst, imm32);
8691 }
8692 
8693 void Assembler::subq(Register dst, Address src) {
8694   InstructionMark im(this);
8695   prefixq(src, dst);
8696   emit_int8(0x2B);
8697   emit_operand(dst, src);
8698 }
8699 
8700 void Assembler::subq(Register dst, Register src) {
8701   (void) prefixq_and_encode(dst->encoding(), src->encoding());
8702   emit_arith(0x2B, 0xC0, dst, src);
8703 }
8704 
8705 void Assembler::testq(Register dst, int32_t imm32) {
8706   // not using emit_arith because test
8707   // doesn't support sign-extension of
8708   // 8bit operands
8709   int encode = dst->encoding();
8710   if (encode == 0) {
8711     prefix(REX_W);
8712     emit_int8((unsigned char)0xA9);
8713   } else {
8714     encode = prefixq_and_encode(encode);
8715     emit_int8((unsigned char)0xF7);
8716     emit_int8((unsigned char)(0xC0 | encode));
8717   }
8718   emit_int32(imm32);
8719 }
8720 
8721 void Assembler::testq(Register dst, Register src) {
8722   (void) prefixq_and_encode(dst->encoding(), src->encoding());
8723   emit_arith(0x85, 0xC0, dst, src);
8724 }
8725 
8726 void Assembler::xaddq(Address dst, Register src) {
8727   InstructionMark im(this);
8728   prefixq(dst, src);
8729   emit_int8(0x0F);
8730   emit_int8((unsigned char)0xC1);
8731   emit_operand(src, dst);
8732 }
8733 
8734 void Assembler::xchgq(Register dst, Address src) {
8735   InstructionMark im(this);
8736   prefixq(src, dst);
8737   emit_int8((unsigned char)0x87);
8738   emit_operand(dst, src);
8739 }
8740 
8741 void Assembler::xchgq(Register dst, Register src) {
8742   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
8743   emit_int8((unsigned char)0x87);
8744   emit_int8((unsigned char)(0xc0 | encode));
8745 }
8746 
8747 void Assembler::xorq(Register dst, Register src) {
8748   (void) prefixq_and_encode(dst->encoding(), src->encoding());
8749   emit_arith(0x33, 0xC0, dst, src);
8750 }
8751 
8752 void Assembler::xorq(Register dst, Address src) {
8753   InstructionMark im(this);
8754   prefixq(src, dst);
8755   emit_int8(0x33);
8756   emit_operand(dst, src);
8757 }
8758 
8759 #endif // !LP64