1 /*
   2  * Copyright (c) 1997, 2016, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #include "precompiled.hpp"
  26 #include "asm/assembler.hpp"
  27 #include "asm/assembler.inline.hpp"
  28 #include "gc/shared/cardTableModRefBS.hpp"
  29 #include "gc/shared/collectedHeap.inline.hpp"
  30 #include "interpreter/interpreter.hpp"
  31 #include "memory/resourceArea.hpp"
  32 #include "prims/methodHandles.hpp"
  33 #include "runtime/biasedLocking.hpp"
  34 #include "runtime/interfaceSupport.hpp"
  35 #include "runtime/objectMonitor.hpp"
  36 #include "runtime/os.hpp"
  37 #include "runtime/sharedRuntime.hpp"
  38 #include "runtime/stubRoutines.hpp"
  39 #include "utilities/macros.hpp"
  40 #if INCLUDE_ALL_GCS
  41 #include "gc/g1/g1CollectedHeap.inline.hpp"
  42 #include "gc/g1/g1SATBCardTableModRefBS.hpp"
  43 #include "gc/g1/heapRegion.hpp"
  44 #endif // INCLUDE_ALL_GCS
  45 
  46 #ifdef PRODUCT
  47 #define BLOCK_COMMENT(str) /* nothing */
  48 #define STOP(error) stop(error)
  49 #else
  50 #define BLOCK_COMMENT(str) block_comment(str)
  51 #define STOP(error) block_comment(error); stop(error)
  52 #endif
  53 
  54 #define BIND(label) bind(label); BLOCK_COMMENT(#label ":")
  55 // Implementation of AddressLiteral
  56 
  57 // A 2-D table for managing compressed displacement(disp8) on EVEX enabled platforms.
  58 unsigned char tuple_table[Assembler::EVEX_ETUP + 1][Assembler::AVX_512bit + 1] = {
  59   // -----------------Table 4.5 -------------------- //
  60   16, 32, 64,  // EVEX_FV(0)
  61   4,  4,  4,   // EVEX_FV(1) - with Evex.b
  62   16, 32, 64,  // EVEX_FV(2) - with Evex.w
  63   8,  8,  8,   // EVEX_FV(3) - with Evex.w and Evex.b
  64   8,  16, 32,  // EVEX_HV(0)
  65   4,  4,  4,   // EVEX_HV(1) - with Evex.b
  66   // -----------------Table 4.6 -------------------- //
  67   16, 32, 64,  // EVEX_FVM(0)
  68   1,  1,  1,   // EVEX_T1S(0)
  69   2,  2,  2,   // EVEX_T1S(1)
  70   4,  4,  4,   // EVEX_T1S(2)
  71   8,  8,  8,   // EVEX_T1S(3)
  72   4,  4,  4,   // EVEX_T1F(0)
  73   8,  8,  8,   // EVEX_T1F(1)
  74   8,  8,  8,   // EVEX_T2(0)
  75   0,  16, 16,  // EVEX_T2(1)
  76   0,  16, 16,  // EVEX_T4(0)
  77   0,  0,  32,  // EVEX_T4(1)
  78   0,  0,  32,  // EVEX_T8(0)
  79   8,  16, 32,  // EVEX_HVM(0)
  80   4,  8,  16,  // EVEX_QVM(0)
  81   2,  4,  8,   // EVEX_OVM(0)
  82   16, 16, 16,  // EVEX_M128(0)
  83   8,  32, 64,  // EVEX_DUP(0)
  84   0,  0,  0    // EVEX_NTUP
  85 };
  86 
  87 AddressLiteral::AddressLiteral(address target, relocInfo::relocType rtype) {
  88   _is_lval = false;
  89   _target = target;
  90   switch (rtype) {
  91   case relocInfo::oop_type:
  92   case relocInfo::metadata_type:
  93     // Oops are a special case. Normally they would be their own section
  94     // but in cases like icBuffer they are literals in the code stream that
  95     // we don't have a section for. We use none so that we get a literal address
  96     // which is always patchable.
  97     break;
  98   case relocInfo::external_word_type:
  99     _rspec = external_word_Relocation::spec(target);
 100     break;
 101   case relocInfo::internal_word_type:
 102     _rspec = internal_word_Relocation::spec(target);
 103     break;
 104   case relocInfo::opt_virtual_call_type:
 105     _rspec = opt_virtual_call_Relocation::spec();
 106     break;
 107   case relocInfo::static_call_type:
 108     _rspec = static_call_Relocation::spec();
 109     break;
 110   case relocInfo::runtime_call_type:
 111     _rspec = runtime_call_Relocation::spec();
 112     break;
 113   case relocInfo::poll_type:
 114   case relocInfo::poll_return_type:
 115     _rspec = Relocation::spec_simple(rtype);
 116     break;
 117   case relocInfo::none:
 118     break;
 119   default:
 120     ShouldNotReachHere();
 121     break;
 122   }
 123 }
 124 
 125 // Implementation of Address
 126 
 127 #ifdef _LP64
 128 
 129 Address Address::make_array(ArrayAddress adr) {
 130   // Not implementable on 64bit machines
 131   // Should have been handled higher up the call chain.
 132   ShouldNotReachHere();
 133   return Address();
 134 }
 135 
 136 // exceedingly dangerous constructor
 137 Address::Address(int disp, address loc, relocInfo::relocType rtype) {
 138   _base  = noreg;
 139   _index = noreg;
 140   _scale = no_scale;
 141   _disp  = disp;
 142   switch (rtype) {
 143     case relocInfo::external_word_type:
 144       _rspec = external_word_Relocation::spec(loc);
 145       break;
 146     case relocInfo::internal_word_type:
 147       _rspec = internal_word_Relocation::spec(loc);
 148       break;
 149     case relocInfo::runtime_call_type:
 150       // HMM
 151       _rspec = runtime_call_Relocation::spec();
 152       break;
 153     case relocInfo::poll_type:
 154     case relocInfo::poll_return_type:
 155       _rspec = Relocation::spec_simple(rtype);
 156       break;
 157     case relocInfo::none:
 158       break;
 159     default:
 160       ShouldNotReachHere();
 161   }
 162 }
 163 #else // LP64
 164 
 165 Address Address::make_array(ArrayAddress adr) {
 166   AddressLiteral base = adr.base();
 167   Address index = adr.index();
 168   assert(index._disp == 0, "must not have disp"); // maybe it can?
 169   Address array(index._base, index._index, index._scale, (intptr_t) base.target());
 170   array._rspec = base._rspec;
 171   return array;
 172 }
 173 
 174 // exceedingly dangerous constructor
 175 Address::Address(address loc, RelocationHolder spec) {
 176   _base  = noreg;
 177   _index = noreg;
 178   _scale = no_scale;
 179   _disp  = (intptr_t) loc;
 180   _rspec = spec;
 181 }
 182 
 183 #endif // _LP64
 184 
 185 
 186 
 187 // Convert the raw encoding form into the form expected by the constructor for
 188 // Address.  An index of 4 (rsp) corresponds to having no index, so convert
 189 // that to noreg for the Address constructor.
 190 Address Address::make_raw(int base, int index, int scale, int disp, relocInfo::relocType disp_reloc) {
 191   RelocationHolder rspec;
 192   if (disp_reloc != relocInfo::none) {
 193     rspec = Relocation::spec_simple(disp_reloc);
 194   }
 195   bool valid_index = index != rsp->encoding();
 196   if (valid_index) {
 197     Address madr(as_Register(base), as_Register(index), (Address::ScaleFactor)scale, in_ByteSize(disp));
 198     madr._rspec = rspec;
 199     return madr;
 200   } else {
 201     Address madr(as_Register(base), noreg, Address::no_scale, in_ByteSize(disp));
 202     madr._rspec = rspec;
 203     return madr;
 204   }
 205 }
 206 
 207 // Implementation of Assembler
 208 
 209 int AbstractAssembler::code_fill_byte() {
 210   return (u_char)'\xF4'; // hlt
 211 }
 212 
 213 // make this go away someday
 214 void Assembler::emit_data(jint data, relocInfo::relocType rtype, int format) {
 215   if (rtype == relocInfo::none)
 216     emit_int32(data);
 217   else
 218     emit_data(data, Relocation::spec_simple(rtype), format);
 219 }
 220 
 221 void Assembler::emit_data(jint data, RelocationHolder const& rspec, int format) {
 222   assert(imm_operand == 0, "default format must be immediate in this file");
 223   assert(inst_mark() != NULL, "must be inside InstructionMark");
 224   if (rspec.type() !=  relocInfo::none) {
 225     #ifdef ASSERT
 226       check_relocation(rspec, format);
 227     #endif
 228     // Do not use AbstractAssembler::relocate, which is not intended for
 229     // embedded words.  Instead, relocate to the enclosing instruction.
 230 
 231     // hack. call32 is too wide for mask so use disp32
 232     if (format == call32_operand)
 233       code_section()->relocate(inst_mark(), rspec, disp32_operand);
 234     else
 235       code_section()->relocate(inst_mark(), rspec, format);
 236   }
 237   emit_int32(data);
 238 }
 239 
 240 static int encode(Register r) {
 241   int enc = r->encoding();
 242   if (enc >= 8) {
 243     enc -= 8;
 244   }
 245   return enc;
 246 }
 247 
 248 void Assembler::emit_arith_b(int op1, int op2, Register dst, int imm8) {
 249   assert(dst->has_byte_register(), "must have byte register");
 250   assert(isByte(op1) && isByte(op2), "wrong opcode");
 251   assert(isByte(imm8), "not a byte");
 252   assert((op1 & 0x01) == 0, "should be 8bit operation");
 253   emit_int8(op1);
 254   emit_int8(op2 | encode(dst));
 255   emit_int8(imm8);
 256 }
 257 
 258 
 259 void Assembler::emit_arith(int op1, int op2, Register dst, int32_t imm32) {
 260   assert(isByte(op1) && isByte(op2), "wrong opcode");
 261   assert((op1 & 0x01) == 1, "should be 32bit operation");
 262   assert((op1 & 0x02) == 0, "sign-extension bit should not be set");
 263   if (is8bit(imm32)) {
 264     emit_int8(op1 | 0x02); // set sign bit
 265     emit_int8(op2 | encode(dst));
 266     emit_int8(imm32 & 0xFF);
 267   } else {
 268     emit_int8(op1);
 269     emit_int8(op2 | encode(dst));
 270     emit_int32(imm32);
 271   }
 272 }
 273 
 274 // Force generation of a 4 byte immediate value even if it fits into 8bit
 275 void Assembler::emit_arith_imm32(int op1, int op2, Register dst, int32_t imm32) {
 276   assert(isByte(op1) && isByte(op2), "wrong opcode");
 277   assert((op1 & 0x01) == 1, "should be 32bit operation");
 278   assert((op1 & 0x02) == 0, "sign-extension bit should not be set");
 279   emit_int8(op1);
 280   emit_int8(op2 | encode(dst));
 281   emit_int32(imm32);
 282 }
 283 
 284 // immediate-to-memory forms
 285 void Assembler::emit_arith_operand(int op1, Register rm, Address adr, int32_t imm32) {
 286   assert((op1 & 0x01) == 1, "should be 32bit operation");
 287   assert((op1 & 0x02) == 0, "sign-extension bit should not be set");
 288   if (is8bit(imm32)) {
 289     emit_int8(op1 | 0x02); // set sign bit
 290     emit_operand(rm, adr, 1);
 291     emit_int8(imm32 & 0xFF);
 292   } else {
 293     emit_int8(op1);
 294     emit_operand(rm, adr, 4);
 295     emit_int32(imm32);
 296   }
 297 }
 298 
 299 
 300 void Assembler::emit_arith(int op1, int op2, Register dst, Register src) {
 301   assert(isByte(op1) && isByte(op2), "wrong opcode");
 302   emit_int8(op1);
 303   emit_int8(op2 | encode(dst) << 3 | encode(src));
 304 }
 305 
 306 
 307 bool Assembler::query_compressed_disp_byte(int disp, bool is_evex_inst, int vector_len,
 308                                            int cur_tuple_type, int in_size_in_bits, int cur_encoding) {
 309   int mod_idx = 0;
 310   // We will test if the displacement fits the compressed format and if so
 311   // apply the compression to the displacment iff the result is8bit.
 312   if (VM_Version::supports_evex() && is_evex_inst) {
 313     switch (cur_tuple_type) {
 314     case EVEX_FV:
 315       if ((cur_encoding & VEX_W) == VEX_W) {
 316         mod_idx = ((cur_encoding & EVEX_Rb) == EVEX_Rb) ? 3 : 2;
 317       } else {
 318         mod_idx = ((cur_encoding & EVEX_Rb) == EVEX_Rb) ? 1 : 0;
 319       }
 320       break;
 321 
 322     case EVEX_HV:
 323       mod_idx = ((cur_encoding & EVEX_Rb) == EVEX_Rb) ? 1 : 0;
 324       break;
 325 
 326     case EVEX_FVM:
 327       break;
 328 
 329     case EVEX_T1S:
 330       switch (in_size_in_bits) {
 331       case EVEX_8bit:
 332         break;
 333 
 334       case EVEX_16bit:
 335         mod_idx = 1;
 336         break;
 337 
 338       case EVEX_32bit:
 339         mod_idx = 2;
 340         break;
 341 
 342       case EVEX_64bit:
 343         mod_idx = 3;
 344         break;
 345       }
 346       break;
 347 
 348     case EVEX_T1F:
 349     case EVEX_T2:
 350     case EVEX_T4:
 351       mod_idx = (in_size_in_bits == EVEX_64bit) ? 1 : 0;
 352       break;
 353 
 354     case EVEX_T8:
 355       break;
 356 
 357     case EVEX_HVM:
 358       break;
 359 
 360     case EVEX_QVM:
 361       break;
 362 
 363     case EVEX_OVM:
 364       break;
 365 
 366     case EVEX_M128:
 367       break;
 368 
 369     case EVEX_DUP:
 370       break;
 371 
 372     default:
 373       assert(0, "no valid evex tuple_table entry");
 374       break;
 375     }
 376 
 377     if (vector_len >= AVX_128bit && vector_len <= AVX_512bit) {
 378       int disp_factor = tuple_table[cur_tuple_type + mod_idx][vector_len];
 379       if ((disp % disp_factor) == 0) {
 380         int new_disp = disp / disp_factor;
 381         if ((-0x80 <= new_disp && new_disp < 0x80)) {
 382           disp = new_disp;
 383         }
 384       } else {
 385         return false;
 386       }
 387     }
 388   }
 389   return (-0x80 <= disp && disp < 0x80);
 390 }
 391 
 392 
 393 bool Assembler::emit_compressed_disp_byte(int &disp) {
 394   int mod_idx = 0;
 395   // We will test if the displacement fits the compressed format and if so
 396   // apply the compression to the displacment iff the result is8bit.
 397   if (VM_Version::supports_evex() && _attributes && _attributes->is_evex_instruction()) {
 398     int evex_encoding = _attributes->get_evex_encoding();
 399     int tuple_type = _attributes->get_tuple_type();
 400     switch (tuple_type) {
 401     case EVEX_FV:
 402       if ((evex_encoding & VEX_W) == VEX_W) {
 403         mod_idx = ((evex_encoding & EVEX_Rb) == EVEX_Rb) ? 3 : 2;
 404       } else {
 405         mod_idx = ((evex_encoding & EVEX_Rb) == EVEX_Rb) ? 1 : 0;
 406       }
 407       break;
 408 
 409     case EVEX_HV:
 410       mod_idx = ((evex_encoding & EVEX_Rb) == EVEX_Rb) ? 1 : 0;
 411       break;
 412 
 413     case EVEX_FVM:
 414       break;
 415 
 416     case EVEX_T1S:
 417       switch (_attributes->get_input_size()) {
 418       case EVEX_8bit:
 419         break;
 420 
 421       case EVEX_16bit:
 422         mod_idx = 1;
 423         break;
 424 
 425       case EVEX_32bit:
 426         mod_idx = 2;
 427         break;
 428 
 429       case EVEX_64bit:
 430         mod_idx = 3;
 431         break;
 432       }
 433       break;
 434 
 435     case EVEX_T1F:
 436     case EVEX_T2:
 437     case EVEX_T4:
 438       mod_idx = (_attributes->get_input_size() == EVEX_64bit) ? 1 : 0;
 439       break;
 440 
 441     case EVEX_T8:
 442       break;
 443 
 444     case EVEX_HVM:
 445       break;
 446 
 447     case EVEX_QVM:
 448       break;
 449 
 450     case EVEX_OVM:
 451       break;
 452 
 453     case EVEX_M128:
 454       break;
 455 
 456     case EVEX_DUP:
 457       break;
 458 
 459     default:
 460       assert(0, "no valid evex tuple_table entry");
 461       break;
 462     }
 463 
 464     int vector_len = _attributes->get_vector_len();
 465     if (vector_len >= AVX_128bit && vector_len <= AVX_512bit) {
 466       int disp_factor = tuple_table[tuple_type + mod_idx][vector_len];
 467       if ((disp % disp_factor) == 0) {
 468         int new_disp = disp / disp_factor;
 469         if (is8bit(new_disp)) {
 470           disp = new_disp;
 471         }
 472       } else {
 473         return false;
 474       }
 475     }
 476   }
 477   return is8bit(disp);
 478 }
 479 
 480 
 481 void Assembler::emit_operand(Register reg, Register base, Register index,
 482                              Address::ScaleFactor scale, int disp,
 483                              RelocationHolder const& rspec,
 484                              int rip_relative_correction) {
 485   relocInfo::relocType rtype = (relocInfo::relocType) rspec.type();
 486 
 487   // Encode the registers as needed in the fields they are used in
 488 
 489   int regenc = encode(reg) << 3;
 490   int indexenc = index->is_valid() ? encode(index) << 3 : 0;
 491   int baseenc = base->is_valid() ? encode(base) : 0;
 492 
 493   if (base->is_valid()) {
 494     if (index->is_valid()) {
 495       assert(scale != Address::no_scale, "inconsistent address");
 496       // [base + index*scale + disp]
 497       if (disp == 0 && rtype == relocInfo::none  &&
 498           base != rbp LP64_ONLY(&& base != r13)) {
 499         // [base + index*scale]
 500         // [00 reg 100][ss index base]
 501         assert(index != rsp, "illegal addressing mode");
 502         emit_int8(0x04 | regenc);
 503         emit_int8(scale << 6 | indexenc | baseenc);
 504       } else if (emit_compressed_disp_byte(disp) && rtype == relocInfo::none) {
 505         // [base + index*scale + imm8]
 506         // [01 reg 100][ss index base] imm8
 507         assert(index != rsp, "illegal addressing mode");
 508         emit_int8(0x44 | regenc);
 509         emit_int8(scale << 6 | indexenc | baseenc);
 510         emit_int8(disp & 0xFF);
 511       } else {
 512         // [base + index*scale + disp32]
 513         // [10 reg 100][ss index base] disp32
 514         assert(index != rsp, "illegal addressing mode");
 515         emit_int8(0x84 | regenc);
 516         emit_int8(scale << 6 | indexenc | baseenc);
 517         emit_data(disp, rspec, disp32_operand);
 518       }
 519     } else if (base == rsp LP64_ONLY(|| base == r12)) {
 520       // [rsp + disp]
 521       if (disp == 0 && rtype == relocInfo::none) {
 522         // [rsp]
 523         // [00 reg 100][00 100 100]
 524         emit_int8(0x04 | regenc);
 525         emit_int8(0x24);
 526       } else if (emit_compressed_disp_byte(disp) && rtype == relocInfo::none) {
 527         // [rsp + imm8]
 528         // [01 reg 100][00 100 100] disp8
 529         emit_int8(0x44 | regenc);
 530         emit_int8(0x24);
 531         emit_int8(disp & 0xFF);
 532       } else {
 533         // [rsp + imm32]
 534         // [10 reg 100][00 100 100] disp32
 535         emit_int8(0x84 | regenc);
 536         emit_int8(0x24);
 537         emit_data(disp, rspec, disp32_operand);
 538       }
 539     } else {
 540       // [base + disp]
 541       assert(base != rsp LP64_ONLY(&& base != r12), "illegal addressing mode");
 542       if (disp == 0 && rtype == relocInfo::none &&
 543           base != rbp LP64_ONLY(&& base != r13)) {
 544         // [base]
 545         // [00 reg base]
 546         emit_int8(0x00 | regenc | baseenc);
 547       } else if (emit_compressed_disp_byte(disp) && rtype == relocInfo::none) {
 548         // [base + disp8]
 549         // [01 reg base] disp8
 550         emit_int8(0x40 | regenc | baseenc);
 551         emit_int8(disp & 0xFF);
 552       } else {
 553         // [base + disp32]
 554         // [10 reg base] disp32
 555         emit_int8(0x80 | regenc | baseenc);
 556         emit_data(disp, rspec, disp32_operand);
 557       }
 558     }
 559   } else {
 560     if (index->is_valid()) {
 561       assert(scale != Address::no_scale, "inconsistent address");
 562       // [index*scale + disp]
 563       // [00 reg 100][ss index 101] disp32
 564       assert(index != rsp, "illegal addressing mode");
 565       emit_int8(0x04 | regenc);
 566       emit_int8(scale << 6 | indexenc | 0x05);
 567       emit_data(disp, rspec, disp32_operand);
 568     } else if (rtype != relocInfo::none ) {
 569       // [disp] (64bit) RIP-RELATIVE (32bit) abs
 570       // [00 000 101] disp32
 571 
 572       emit_int8(0x05 | regenc);
 573       // Note that the RIP-rel. correction applies to the generated
 574       // disp field, but _not_ to the target address in the rspec.
 575 
 576       // disp was created by converting the target address minus the pc
 577       // at the start of the instruction. That needs more correction here.
 578       // intptr_t disp = target - next_ip;
 579       assert(inst_mark() != NULL, "must be inside InstructionMark");
 580       address next_ip = pc() + sizeof(int32_t) + rip_relative_correction;
 581       int64_t adjusted = disp;
 582       // Do rip-rel adjustment for 64bit
 583       LP64_ONLY(adjusted -=  (next_ip - inst_mark()));
 584       assert(is_simm32(adjusted),
 585              "must be 32bit offset (RIP relative address)");
 586       emit_data((int32_t) adjusted, rspec, disp32_operand);
 587 
 588     } else {
 589       // 32bit never did this, did everything as the rip-rel/disp code above
 590       // [disp] ABSOLUTE
 591       // [00 reg 100][00 100 101] disp32
 592       emit_int8(0x04 | regenc);
 593       emit_int8(0x25);
 594       emit_data(disp, rspec, disp32_operand);
 595     }
 596   }
 597 }
 598 
 599 void Assembler::emit_operand(XMMRegister reg, Register base, Register index,
 600                              Address::ScaleFactor scale, int disp,
 601                              RelocationHolder const& rspec) {
 602   if (UseAVX > 2) {
 603     int xreg_enc = reg->encoding();
 604     if (xreg_enc > 15) {
 605       XMMRegister new_reg = as_XMMRegister(xreg_enc & 0xf);
 606       emit_operand((Register)new_reg, base, index, scale, disp, rspec);
 607       return;
 608     }
 609   }
 610   emit_operand((Register)reg, base, index, scale, disp, rspec);
 611 }
 612 
 613 // Secret local extension to Assembler::WhichOperand:
 614 #define end_pc_operand (_WhichOperand_limit)
 615 
 616 address Assembler::locate_operand(address inst, WhichOperand which) {
 617   // Decode the given instruction, and return the address of
 618   // an embedded 32-bit operand word.
 619 
 620   // If "which" is disp32_operand, selects the displacement portion
 621   // of an effective address specifier.
 622   // If "which" is imm64_operand, selects the trailing immediate constant.
 623   // If "which" is call32_operand, selects the displacement of a call or jump.
 624   // Caller is responsible for ensuring that there is such an operand,
 625   // and that it is 32/64 bits wide.
 626 
 627   // If "which" is end_pc_operand, find the end of the instruction.
 628 
 629   address ip = inst;
 630   bool is_64bit = false;
 631 
 632   debug_only(bool has_disp32 = false);
 633   int tail_size = 0; // other random bytes (#32, #16, etc.) at end of insn
 634 
 635   again_after_prefix:
 636   switch (0xFF & *ip++) {
 637 
 638   // These convenience macros generate groups of "case" labels for the switch.
 639 #define REP4(x) (x)+0: case (x)+1: case (x)+2: case (x)+3
 640 #define REP8(x) (x)+0: case (x)+1: case (x)+2: case (x)+3: \
 641              case (x)+4: case (x)+5: case (x)+6: case (x)+7
 642 #define REP16(x) REP8((x)+0): \
 643               case REP8((x)+8)
 644 
 645   case CS_segment:
 646   case SS_segment:
 647   case DS_segment:
 648   case ES_segment:
 649   case FS_segment:
 650   case GS_segment:
 651     // Seems dubious
 652     LP64_ONLY(assert(false, "shouldn't have that prefix"));
 653     assert(ip == inst+1, "only one prefix allowed");
 654     goto again_after_prefix;
 655 
 656   case 0x67:
 657   case REX:
 658   case REX_B:
 659   case REX_X:
 660   case REX_XB:
 661   case REX_R:
 662   case REX_RB:
 663   case REX_RX:
 664   case REX_RXB:
 665     NOT_LP64(assert(false, "64bit prefixes"));
 666     goto again_after_prefix;
 667 
 668   case REX_W:
 669   case REX_WB:
 670   case REX_WX:
 671   case REX_WXB:
 672   case REX_WR:
 673   case REX_WRB:
 674   case REX_WRX:
 675   case REX_WRXB:
 676     NOT_LP64(assert(false, "64bit prefixes"));
 677     is_64bit = true;
 678     goto again_after_prefix;
 679 
 680   case 0xFF: // pushq a; decl a; incl a; call a; jmp a
 681   case 0x88: // movb a, r
 682   case 0x89: // movl a, r
 683   case 0x8A: // movb r, a
 684   case 0x8B: // movl r, a
 685   case 0x8F: // popl a
 686     debug_only(has_disp32 = true);
 687     break;
 688 
 689   case 0x68: // pushq #32
 690     if (which == end_pc_operand) {
 691       return ip + 4;
 692     }
 693     assert(which == imm_operand && !is_64bit, "pushl has no disp32 or 64bit immediate");
 694     return ip;                  // not produced by emit_operand
 695 
 696   case 0x66: // movw ... (size prefix)
 697     again_after_size_prefix2:
 698     switch (0xFF & *ip++) {
 699     case REX:
 700     case REX_B:
 701     case REX_X:
 702     case REX_XB:
 703     case REX_R:
 704     case REX_RB:
 705     case REX_RX:
 706     case REX_RXB:
 707     case REX_W:
 708     case REX_WB:
 709     case REX_WX:
 710     case REX_WXB:
 711     case REX_WR:
 712     case REX_WRB:
 713     case REX_WRX:
 714     case REX_WRXB:
 715       NOT_LP64(assert(false, "64bit prefix found"));
 716       goto again_after_size_prefix2;
 717     case 0x8B: // movw r, a
 718     case 0x89: // movw a, r
 719       debug_only(has_disp32 = true);
 720       break;
 721     case 0xC7: // movw a, #16
 722       debug_only(has_disp32 = true);
 723       tail_size = 2;  // the imm16
 724       break;
 725     case 0x0F: // several SSE/SSE2 variants
 726       ip--;    // reparse the 0x0F
 727       goto again_after_prefix;
 728     default:
 729       ShouldNotReachHere();
 730     }
 731     break;
 732 
 733   case REP8(0xB8): // movl/q r, #32/#64(oop?)
 734     if (which == end_pc_operand)  return ip + (is_64bit ? 8 : 4);
 735     // these asserts are somewhat nonsensical
 736 #ifndef _LP64
 737     assert(which == imm_operand || which == disp32_operand,
 738            "which %d is_64_bit %d ip " INTPTR_FORMAT, which, is_64bit, p2i(ip));
 739 #else
 740     assert((which == call32_operand || which == imm_operand) && is_64bit ||
 741            which == narrow_oop_operand && !is_64bit,
 742            "which %d is_64_bit %d ip " INTPTR_FORMAT, which, is_64bit, p2i(ip));
 743 #endif // _LP64
 744     return ip;
 745 
 746   case 0x69: // imul r, a, #32
 747   case 0xC7: // movl a, #32(oop?)
 748     tail_size = 4;
 749     debug_only(has_disp32 = true); // has both kinds of operands!
 750     break;
 751 
 752   case 0x0F: // movx..., etc.
 753     switch (0xFF & *ip++) {
 754     case 0x3A: // pcmpestri
 755       tail_size = 1;
 756     case 0x38: // ptest, pmovzxbw
 757       ip++; // skip opcode
 758       debug_only(has_disp32 = true); // has both kinds of operands!
 759       break;
 760 
 761     case 0x70: // pshufd r, r/a, #8
 762       debug_only(has_disp32 = true); // has both kinds of operands!
 763     case 0x73: // psrldq r, #8
 764       tail_size = 1;
 765       break;
 766 
 767     case 0x12: // movlps
 768     case 0x28: // movaps
 769     case 0x2E: // ucomiss
 770     case 0x2F: // comiss
 771     case 0x54: // andps
 772     case 0x55: // andnps
 773     case 0x56: // orps
 774     case 0x57: // xorps
 775     case 0x58: // addpd
 776     case 0x59: // mulpd
 777     case 0x6E: // movd
 778     case 0x7E: // movd
 779     case 0xAE: // ldmxcsr, stmxcsr, fxrstor, fxsave, clflush
 780     case 0xFE: // paddd
 781       debug_only(has_disp32 = true);
 782       break;
 783 
 784     case 0xAD: // shrd r, a, %cl
 785     case 0xAF: // imul r, a
 786     case 0xBE: // movsbl r, a (movsxb)
 787     case 0xBF: // movswl r, a (movsxw)
 788     case 0xB6: // movzbl r, a (movzxb)
 789     case 0xB7: // movzwl r, a (movzxw)
 790     case REP16(0x40): // cmovl cc, r, a
 791     case 0xB0: // cmpxchgb
 792     case 0xB1: // cmpxchg
 793     case 0xC1: // xaddl
 794     case 0xC7: // cmpxchg8
 795     case REP16(0x90): // setcc a
 796       debug_only(has_disp32 = true);
 797       // fall out of the switch to decode the address
 798       break;
 799 
 800     case 0xC4: // pinsrw r, a, #8
 801       debug_only(has_disp32 = true);
 802     case 0xC5: // pextrw r, r, #8
 803       tail_size = 1;  // the imm8
 804       break;
 805 
 806     case 0xAC: // shrd r, a, #8
 807       debug_only(has_disp32 = true);
 808       tail_size = 1;  // the imm8
 809       break;
 810 
 811     case REP16(0x80): // jcc rdisp32
 812       if (which == end_pc_operand)  return ip + 4;
 813       assert(which == call32_operand, "jcc has no disp32 or imm");
 814       return ip;
 815     default:
 816       ShouldNotReachHere();
 817     }
 818     break;
 819 
 820   case 0x81: // addl a, #32; addl r, #32
 821     // also: orl, adcl, sbbl, andl, subl, xorl, cmpl
 822     // on 32bit in the case of cmpl, the imm might be an oop
 823     tail_size = 4;
 824     debug_only(has_disp32 = true); // has both kinds of operands!
 825     break;
 826 
 827   case 0x83: // addl a, #8; addl r, #8
 828     // also: orl, adcl, sbbl, andl, subl, xorl, cmpl
 829     debug_only(has_disp32 = true); // has both kinds of operands!
 830     tail_size = 1;
 831     break;
 832 
 833   case 0x9B:
 834     switch (0xFF & *ip++) {
 835     case 0xD9: // fnstcw a
 836       debug_only(has_disp32 = true);
 837       break;
 838     default:
 839       ShouldNotReachHere();
 840     }
 841     break;
 842 
 843   case REP4(0x00): // addb a, r; addl a, r; addb r, a; addl r, a
 844   case REP4(0x10): // adc...
 845   case REP4(0x20): // and...
 846   case REP4(0x30): // xor...
 847   case REP4(0x08): // or...
 848   case REP4(0x18): // sbb...
 849   case REP4(0x28): // sub...
 850   case 0xF7: // mull a
 851   case 0x8D: // lea r, a
 852   case 0x87: // xchg r, a
 853   case REP4(0x38): // cmp...
 854   case 0x85: // test r, a
 855     debug_only(has_disp32 = true); // has both kinds of operands!
 856     break;
 857 
 858   case 0xC1: // sal a, #8; sar a, #8; shl a, #8; shr a, #8
 859   case 0xC6: // movb a, #8
 860   case 0x80: // cmpb a, #8
 861   case 0x6B: // imul r, a, #8
 862     debug_only(has_disp32 = true); // has both kinds of operands!
 863     tail_size = 1; // the imm8
 864     break;
 865 
 866   case 0xC4: // VEX_3bytes
 867   case 0xC5: // VEX_2bytes
 868     assert((UseAVX > 0), "shouldn't have VEX prefix");
 869     assert(ip == inst+1, "no prefixes allowed");
 870     // C4 and C5 are also used as opcodes for PINSRW and PEXTRW instructions
 871     // but they have prefix 0x0F and processed when 0x0F processed above.
 872     //
 873     // In 32-bit mode the VEX first byte C4 and C5 alias onto LDS and LES
 874     // instructions (these instructions are not supported in 64-bit mode).
 875     // To distinguish them bits [7:6] are set in the VEX second byte since
 876     // ModRM byte can not be of the form 11xxxxxx in 32-bit mode. To set
 877     // those VEX bits REX and vvvv bits are inverted.
 878     //
 879     // Fortunately C2 doesn't generate these instructions so we don't need
 880     // to check for them in product version.
 881 
 882     // Check second byte
 883     NOT_LP64(assert((0xC0 & *ip) == 0xC0, "shouldn't have LDS and LES instructions"));
 884 
 885     int vex_opcode;
 886     // First byte
 887     if ((0xFF & *inst) == VEX_3bytes) {
 888       vex_opcode = VEX_OPCODE_MASK & *ip;
 889       ip++; // third byte
 890       is_64bit = ((VEX_W & *ip) == VEX_W);
 891     } else {
 892       vex_opcode = VEX_OPCODE_0F;
 893     }
 894     ip++; // opcode
 895     // To find the end of instruction (which == end_pc_operand).
 896     switch (vex_opcode) {
 897       case VEX_OPCODE_0F:
 898         switch (0xFF & *ip) {
 899         case 0x70: // pshufd r, r/a, #8
 900         case 0x71: // ps[rl|ra|ll]w r, #8
 901         case 0x72: // ps[rl|ra|ll]d r, #8
 902         case 0x73: // ps[rl|ra|ll]q r, #8
 903         case 0xC2: // cmp[ps|pd|ss|sd] r, r, r/a, #8
 904         case 0xC4: // pinsrw r, r, r/a, #8
 905         case 0xC5: // pextrw r/a, r, #8
 906         case 0xC6: // shufp[s|d] r, r, r/a, #8
 907           tail_size = 1;  // the imm8
 908           break;
 909         }
 910         break;
 911       case VEX_OPCODE_0F_3A:
 912         tail_size = 1;
 913         break;
 914     }
 915     ip++; // skip opcode
 916     debug_only(has_disp32 = true); // has both kinds of operands!
 917     break;
 918 
 919   case 0x62: // EVEX_4bytes
 920     assert((UseAVX > 0), "shouldn't have EVEX prefix");
 921     assert(ip == inst+1, "no prefixes allowed");
 922     // no EVEX collisions, all instructions that have 0x62 opcodes
 923     // have EVEX versions and are subopcodes of 0x66
 924     ip++; // skip P0 and exmaine W in P1
 925     is_64bit = ((VEX_W & *ip) == VEX_W);
 926     ip++; // move to P2
 927     ip++; // skip P2, move to opcode
 928     // To find the end of instruction (which == end_pc_operand).
 929     switch (0xFF & *ip) {
 930     case 0x22: // pinsrd r, r/a, #8
 931     case 0x61: // pcmpestri r, r/a, #8
 932     case 0x70: // pshufd r, r/a, #8
 933     case 0x73: // psrldq r, #8
 934       tail_size = 1;  // the imm8
 935       break;
 936     default:
 937       break;
 938     }
 939     ip++; // skip opcode
 940     debug_only(has_disp32 = true); // has both kinds of operands!
 941     break;
 942 
 943   case 0xD1: // sal a, 1; sar a, 1; shl a, 1; shr a, 1
 944   case 0xD3: // sal a, %cl; sar a, %cl; shl a, %cl; shr a, %cl
 945   case 0xD9: // fld_s a; fst_s a; fstp_s a; fldcw a
 946   case 0xDD: // fld_d a; fst_d a; fstp_d a
 947   case 0xDB: // fild_s a; fistp_s a; fld_x a; fstp_x a
 948   case 0xDF: // fild_d a; fistp_d a
 949   case 0xD8: // fadd_s a; fsubr_s a; fmul_s a; fdivr_s a; fcomp_s a
 950   case 0xDC: // fadd_d a; fsubr_d a; fmul_d a; fdivr_d a; fcomp_d a
 951   case 0xDE: // faddp_d a; fsubrp_d a; fmulp_d a; fdivrp_d a; fcompp_d a
 952     debug_only(has_disp32 = true);
 953     break;
 954 
 955   case 0xE8: // call rdisp32
 956   case 0xE9: // jmp  rdisp32
 957     if (which == end_pc_operand)  return ip + 4;
 958     assert(which == call32_operand, "call has no disp32 or imm");
 959     return ip;
 960 
 961   case 0xF0:                    // Lock
 962     assert(os::is_MP(), "only on MP");
 963     goto again_after_prefix;
 964 
 965   case 0xF3:                    // For SSE
 966   case 0xF2:                    // For SSE2
 967     switch (0xFF & *ip++) {
 968     case REX:
 969     case REX_B:
 970     case REX_X:
 971     case REX_XB:
 972     case REX_R:
 973     case REX_RB:
 974     case REX_RX:
 975     case REX_RXB:
 976     case REX_W:
 977     case REX_WB:
 978     case REX_WX:
 979     case REX_WXB:
 980     case REX_WR:
 981     case REX_WRB:
 982     case REX_WRX:
 983     case REX_WRXB:
 984       NOT_LP64(assert(false, "found 64bit prefix"));
 985       ip++;
 986     default:
 987       ip++;
 988     }
 989     debug_only(has_disp32 = true); // has both kinds of operands!
 990     break;
 991 
 992   default:
 993     ShouldNotReachHere();
 994 
 995 #undef REP8
 996 #undef REP16
 997   }
 998 
 999   assert(which != call32_operand, "instruction is not a call, jmp, or jcc");
1000 #ifdef _LP64
1001   assert(which != imm_operand, "instruction is not a movq reg, imm64");
1002 #else
1003   // assert(which != imm_operand || has_imm32, "instruction has no imm32 field");
1004   assert(which != imm_operand || has_disp32, "instruction has no imm32 field");
1005 #endif // LP64
1006   assert(which != disp32_operand || has_disp32, "instruction has no disp32 field");
1007 
1008   // parse the output of emit_operand
1009   int op2 = 0xFF & *ip++;
1010   int base = op2 & 0x07;
1011   int op3 = -1;
1012   const int b100 = 4;
1013   const int b101 = 5;
1014   if (base == b100 && (op2 >> 6) != 3) {
1015     op3 = 0xFF & *ip++;
1016     base = op3 & 0x07;   // refetch the base
1017   }
1018   // now ip points at the disp (if any)
1019 
1020   switch (op2 >> 6) {
1021   case 0:
1022     // [00 reg  100][ss index base]
1023     // [00 reg  100][00   100  esp]
1024     // [00 reg base]
1025     // [00 reg  100][ss index  101][disp32]
1026     // [00 reg  101]               [disp32]
1027 
1028     if (base == b101) {
1029       if (which == disp32_operand)
1030         return ip;              // caller wants the disp32
1031       ip += 4;                  // skip the disp32
1032     }
1033     break;
1034 
1035   case 1:
1036     // [01 reg  100][ss index base][disp8]
1037     // [01 reg  100][00   100  esp][disp8]
1038     // [01 reg base]               [disp8]
1039     ip += 1;                    // skip the disp8
1040     break;
1041 
1042   case 2:
1043     // [10 reg  100][ss index base][disp32]
1044     // [10 reg  100][00   100  esp][disp32]
1045     // [10 reg base]               [disp32]
1046     if (which == disp32_operand)
1047       return ip;                // caller wants the disp32
1048     ip += 4;                    // skip the disp32
1049     break;
1050 
1051   case 3:
1052     // [11 reg base]  (not a memory addressing mode)
1053     break;
1054   }
1055 
1056   if (which == end_pc_operand) {
1057     return ip + tail_size;
1058   }
1059 
1060 #ifdef _LP64
1061   assert(which == narrow_oop_operand && !is_64bit, "instruction is not a movl adr, imm32");
1062 #else
1063   assert(which == imm_operand, "instruction has only an imm field");
1064 #endif // LP64
1065   return ip;
1066 }
1067 
1068 address Assembler::locate_next_instruction(address inst) {
1069   // Secretly share code with locate_operand:
1070   return locate_operand(inst, end_pc_operand);
1071 }
1072 
1073 
1074 #ifdef ASSERT
1075 void Assembler::check_relocation(RelocationHolder const& rspec, int format) {
1076   address inst = inst_mark();
1077   assert(inst != NULL && inst < pc(), "must point to beginning of instruction");
1078   address opnd;
1079 
1080   Relocation* r = rspec.reloc();
1081   if (r->type() == relocInfo::none) {
1082     return;
1083   } else if (r->is_call() || format == call32_operand) {
1084     // assert(format == imm32_operand, "cannot specify a nonzero format");
1085     opnd = locate_operand(inst, call32_operand);
1086   } else if (r->is_data()) {
1087     assert(format == imm_operand || format == disp32_operand
1088            LP64_ONLY(|| format == narrow_oop_operand), "format ok");
1089     opnd = locate_operand(inst, (WhichOperand)format);
1090   } else {
1091     assert(format == imm_operand, "cannot specify a format");
1092     return;
1093   }
1094   assert(opnd == pc(), "must put operand where relocs can find it");
1095 }
1096 #endif // ASSERT
1097 
1098 void Assembler::emit_operand32(Register reg, Address adr) {
1099   assert(reg->encoding() < 8, "no extended registers");
1100   assert(!adr.base_needs_rex() && !adr.index_needs_rex(), "no extended registers");
1101   emit_operand(reg, adr._base, adr._index, adr._scale, adr._disp,
1102                adr._rspec);
1103 }
1104 
1105 void Assembler::emit_operand(Register reg, Address adr,
1106                              int rip_relative_correction) {
1107   emit_operand(reg, adr._base, adr._index, adr._scale, adr._disp,
1108                adr._rspec,
1109                rip_relative_correction);
1110 }
1111 
1112 void Assembler::emit_operand(XMMRegister reg, Address adr) {
1113   emit_operand(reg, adr._base, adr._index, adr._scale, adr._disp,
1114                adr._rspec);
1115 }
1116 
1117 // MMX operations
1118 void Assembler::emit_operand(MMXRegister reg, Address adr) {
1119   assert(!adr.base_needs_rex() && !adr.index_needs_rex(), "no extended registers");
1120   emit_operand((Register)reg, adr._base, adr._index, adr._scale, adr._disp, adr._rspec);
1121 }
1122 
1123 // work around gcc (3.2.1-7a) bug
1124 void Assembler::emit_operand(Address adr, MMXRegister reg) {
1125   assert(!adr.base_needs_rex() && !adr.index_needs_rex(), "no extended registers");
1126   emit_operand((Register)reg, adr._base, adr._index, adr._scale, adr._disp, adr._rspec);
1127 }
1128 
1129 
1130 void Assembler::emit_farith(int b1, int b2, int i) {
1131   assert(isByte(b1) && isByte(b2), "wrong opcode");
1132   assert(0 <= i &&  i < 8, "illegal stack offset");
1133   emit_int8(b1);
1134   emit_int8(b2 + i);
1135 }
1136 
1137 
1138 // Now the Assembler instructions (identical for 32/64 bits)
1139 
1140 void Assembler::adcl(Address dst, int32_t imm32) {
1141   InstructionMark im(this);
1142   prefix(dst);
1143   emit_arith_operand(0x81, rdx, dst, imm32);
1144 }
1145 
1146 void Assembler::adcl(Address dst, Register src) {
1147   InstructionMark im(this);
1148   prefix(dst, src);
1149   emit_int8(0x11);
1150   emit_operand(src, dst);
1151 }
1152 
1153 void Assembler::adcl(Register dst, int32_t imm32) {
1154   prefix(dst);
1155   emit_arith(0x81, 0xD0, dst, imm32);
1156 }
1157 
1158 void Assembler::adcl(Register dst, Address src) {
1159   InstructionMark im(this);
1160   prefix(src, dst);
1161   emit_int8(0x13);
1162   emit_operand(dst, src);
1163 }
1164 
1165 void Assembler::adcl(Register dst, Register src) {
1166   (void) prefix_and_encode(dst->encoding(), src->encoding());
1167   emit_arith(0x13, 0xC0, dst, src);
1168 }
1169 
1170 void Assembler::addl(Address dst, int32_t imm32) {
1171   InstructionMark im(this);
1172   prefix(dst);
1173   emit_arith_operand(0x81, rax, dst, imm32);
1174 }
1175 
1176 void Assembler::addl(Address dst, Register src) {
1177   InstructionMark im(this);
1178   prefix(dst, src);
1179   emit_int8(0x01);
1180   emit_operand(src, dst);
1181 }
1182 
1183 void Assembler::addl(Register dst, int32_t imm32) {
1184   prefix(dst);
1185   emit_arith(0x81, 0xC0, dst, imm32);
1186 }
1187 
1188 void Assembler::addl(Register dst, Address src) {
1189   InstructionMark im(this);
1190   prefix(src, dst);
1191   emit_int8(0x03);
1192   emit_operand(dst, src);
1193 }
1194 
1195 void Assembler::addl(Register dst, Register src) {
1196   (void) prefix_and_encode(dst->encoding(), src->encoding());
1197   emit_arith(0x03, 0xC0, dst, src);
1198 }
1199 
1200 void Assembler::addr_nop_4() {
1201   assert(UseAddressNop, "no CPU support");
1202   // 4 bytes: NOP DWORD PTR [EAX+0]
1203   emit_int8(0x0F);
1204   emit_int8(0x1F);
1205   emit_int8(0x40); // emit_rm(cbuf, 0x1, EAX_enc, EAX_enc);
1206   emit_int8(0);    // 8-bits offset (1 byte)
1207 }
1208 
1209 void Assembler::addr_nop_5() {
1210   assert(UseAddressNop, "no CPU support");
1211   // 5 bytes: NOP DWORD PTR [EAX+EAX*0+0] 8-bits offset
1212   emit_int8(0x0F);
1213   emit_int8(0x1F);
1214   emit_int8(0x44); // emit_rm(cbuf, 0x1, EAX_enc, 0x4);
1215   emit_int8(0x00); // emit_rm(cbuf, 0x0, EAX_enc, EAX_enc);
1216   emit_int8(0);    // 8-bits offset (1 byte)
1217 }
1218 
1219 void Assembler::addr_nop_7() {
1220   assert(UseAddressNop, "no CPU support");
1221   // 7 bytes: NOP DWORD PTR [EAX+0] 32-bits offset
1222   emit_int8(0x0F);
1223   emit_int8(0x1F);
1224   emit_int8((unsigned char)0x80);
1225                    // emit_rm(cbuf, 0x2, EAX_enc, EAX_enc);
1226   emit_int32(0);   // 32-bits offset (4 bytes)
1227 }
1228 
1229 void Assembler::addr_nop_8() {
1230   assert(UseAddressNop, "no CPU support");
1231   // 8 bytes: NOP DWORD PTR [EAX+EAX*0+0] 32-bits offset
1232   emit_int8(0x0F);
1233   emit_int8(0x1F);
1234   emit_int8((unsigned char)0x84);
1235                    // emit_rm(cbuf, 0x2, EAX_enc, 0x4);
1236   emit_int8(0x00); // emit_rm(cbuf, 0x0, EAX_enc, EAX_enc);
1237   emit_int32(0);   // 32-bits offset (4 bytes)
1238 }
1239 
1240 void Assembler::addsd(XMMRegister dst, XMMRegister src) {
1241   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1242   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
1243   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
1244   emit_int8(0x58);
1245   emit_int8((unsigned char)(0xC0 | encode));
1246 }
1247 
1248 void Assembler::addsd(XMMRegister dst, Address src) {
1249   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1250   InstructionMark im(this);
1251   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
1252   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
1253   simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
1254   emit_int8(0x58);
1255   emit_operand(dst, src);
1256 }
1257 
1258 void Assembler::addss(XMMRegister dst, XMMRegister src) {
1259   NOT_LP64(assert(VM_Version::supports_sse(), ""));
1260   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
1261   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
1262   emit_int8(0x58);
1263   emit_int8((unsigned char)(0xC0 | encode));
1264 }
1265 
1266 void Assembler::addss(XMMRegister dst, Address src) {
1267   NOT_LP64(assert(VM_Version::supports_sse(), ""));
1268   InstructionMark im(this);
1269   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
1270   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
1271   simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
1272   emit_int8(0x58);
1273   emit_operand(dst, src);
1274 }
1275 
1276 void Assembler::aesdec(XMMRegister dst, Address src) {
1277   assert(VM_Version::supports_aes(), "");
1278   InstructionMark im(this);
1279   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
1280   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
1281   emit_int8((unsigned char)0xDE);
1282   emit_operand(dst, src);
1283 }
1284 
1285 void Assembler::aesdec(XMMRegister dst, XMMRegister src) {
1286   assert(VM_Version::supports_aes(), "");
1287   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
1288   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
1289   emit_int8((unsigned char)0xDE);
1290   emit_int8(0xC0 | encode);
1291 }
1292 
1293 void Assembler::aesdeclast(XMMRegister dst, Address src) {
1294   assert(VM_Version::supports_aes(), "");
1295   InstructionMark im(this);
1296   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
1297   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
1298   emit_int8((unsigned char)0xDF);
1299   emit_operand(dst, src);
1300 }
1301 
1302 void Assembler::aesdeclast(XMMRegister dst, XMMRegister src) {
1303   assert(VM_Version::supports_aes(), "");
1304   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
1305   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
1306   emit_int8((unsigned char)0xDF);
1307   emit_int8((unsigned char)(0xC0 | encode));
1308 }
1309 
1310 void Assembler::aesenc(XMMRegister dst, Address src) {
1311   assert(VM_Version::supports_aes(), "");
1312   InstructionMark im(this);
1313   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
1314   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
1315   emit_int8((unsigned char)0xDC);
1316   emit_operand(dst, src);
1317 }
1318 
1319 void Assembler::aesenc(XMMRegister dst, XMMRegister src) {
1320   assert(VM_Version::supports_aes(), "");
1321   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
1322   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
1323   emit_int8((unsigned char)0xDC);
1324   emit_int8(0xC0 | encode);
1325 }
1326 
1327 void Assembler::aesenclast(XMMRegister dst, Address src) {
1328   assert(VM_Version::supports_aes(), "");
1329   InstructionMark im(this);
1330   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
1331   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
1332   emit_int8((unsigned char)0xDD);
1333   emit_operand(dst, src);
1334 }
1335 
1336 void Assembler::aesenclast(XMMRegister dst, XMMRegister src) {
1337   assert(VM_Version::supports_aes(), "");
1338   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
1339   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
1340   emit_int8((unsigned char)0xDD);
1341   emit_int8((unsigned char)(0xC0 | encode));
1342 }
1343 
1344 void Assembler::andl(Address dst, int32_t imm32) {
1345   InstructionMark im(this);
1346   prefix(dst);
1347   emit_int8((unsigned char)0x81);
1348   emit_operand(rsp, dst, 4);
1349   emit_int32(imm32);
1350 }
1351 
1352 void Assembler::andl(Register dst, int32_t imm32) {
1353   prefix(dst);
1354   emit_arith(0x81, 0xE0, dst, imm32);
1355 }
1356 
1357 void Assembler::andl(Register dst, Address src) {
1358   InstructionMark im(this);
1359   prefix(src, dst);
1360   emit_int8(0x23);
1361   emit_operand(dst, src);
1362 }
1363 
1364 void Assembler::andl(Register dst, Register src) {
1365   (void) prefix_and_encode(dst->encoding(), src->encoding());
1366   emit_arith(0x23, 0xC0, dst, src);
1367 }
1368 
1369 void Assembler::andnl(Register dst, Register src1, Register src2) {
1370   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1371   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
1372   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
1373   emit_int8((unsigned char)0xF2);
1374   emit_int8((unsigned char)(0xC0 | encode));
1375 }
1376 
1377 void Assembler::andnl(Register dst, Register src1, Address src2) {
1378   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1379   InstructionMark im(this);
1380   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
1381   vex_prefix(src2, src1->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
1382   emit_int8((unsigned char)0xF2);
1383   emit_operand(dst, src2);
1384 }
1385 
1386 void Assembler::bsfl(Register dst, Register src) {
1387   int encode = prefix_and_encode(dst->encoding(), src->encoding());
1388   emit_int8(0x0F);
1389   emit_int8((unsigned char)0xBC);
1390   emit_int8((unsigned char)(0xC0 | encode));
1391 }
1392 
1393 void Assembler::bsrl(Register dst, Register src) {
1394   int encode = prefix_and_encode(dst->encoding(), src->encoding());
1395   emit_int8(0x0F);
1396   emit_int8((unsigned char)0xBD);
1397   emit_int8((unsigned char)(0xC0 | encode));
1398 }
1399 
1400 void Assembler::bswapl(Register reg) { // bswap
1401   int encode = prefix_and_encode(reg->encoding());
1402   emit_int8(0x0F);
1403   emit_int8((unsigned char)(0xC8 | encode));
1404 }
1405 
1406 void Assembler::blsil(Register dst, Register src) {
1407   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1408   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
1409   int encode = vex_prefix_and_encode(rbx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
1410   emit_int8((unsigned char)0xF3);
1411   emit_int8((unsigned char)(0xC0 | encode));
1412 }
1413 
1414 void Assembler::blsil(Register dst, Address src) {
1415   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1416   InstructionMark im(this);
1417   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
1418   vex_prefix(src, dst->encoding(), rbx->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
1419   emit_int8((unsigned char)0xF3);
1420   emit_operand(rbx, src);
1421 }
1422 
1423 void Assembler::blsmskl(Register dst, Register src) {
1424   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1425   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
1426   int encode = vex_prefix_and_encode(rdx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
1427   emit_int8((unsigned char)0xF3);
1428   emit_int8((unsigned char)(0xC0 | encode));
1429 }
1430 
1431 void Assembler::blsmskl(Register dst, Address src) {
1432   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1433   InstructionMark im(this);
1434   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
1435   vex_prefix(src, dst->encoding(), rdx->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
1436   emit_int8((unsigned char)0xF3);
1437   emit_operand(rdx, src);
1438 }
1439 
1440 void Assembler::blsrl(Register dst, Register src) {
1441   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1442   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
1443   int encode = vex_prefix_and_encode(rcx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
1444   emit_int8((unsigned char)0xF3);
1445   emit_int8((unsigned char)(0xC0 | encode));
1446 }
1447 
1448 void Assembler::blsrl(Register dst, Address src) {
1449   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1450   InstructionMark im(this);
1451   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
1452   vex_prefix(src, dst->encoding(), rcx->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
1453   emit_int8((unsigned char)0xF3);
1454   emit_operand(rcx, src);
1455 }
1456 
1457 void Assembler::call(Label& L, relocInfo::relocType rtype) {
1458   // suspect disp32 is always good
1459   int operand = LP64_ONLY(disp32_operand) NOT_LP64(imm_operand);
1460 
1461   if (L.is_bound()) {
1462     const int long_size = 5;
1463     int offs = (int)( target(L) - pc() );
1464     assert(offs <= 0, "assembler error");
1465     InstructionMark im(this);
1466     // 1110 1000 #32-bit disp
1467     emit_int8((unsigned char)0xE8);
1468     emit_data(offs - long_size, rtype, operand);
1469   } else {
1470     InstructionMark im(this);
1471     // 1110 1000 #32-bit disp
1472     L.add_patch_at(code(), locator());
1473 
1474     emit_int8((unsigned char)0xE8);
1475     emit_data(int(0), rtype, operand);
1476   }
1477 }
1478 
1479 void Assembler::call(Register dst) {
1480   int encode = prefix_and_encode(dst->encoding());
1481   emit_int8((unsigned char)0xFF);
1482   emit_int8((unsigned char)(0xD0 | encode));
1483 }
1484 
1485 
1486 void Assembler::call(Address adr) {
1487   InstructionMark im(this);
1488   prefix(adr);
1489   emit_int8((unsigned char)0xFF);
1490   emit_operand(rdx, adr);
1491 }
1492 
1493 void Assembler::call_literal(address entry, RelocationHolder const& rspec) {
1494   assert(entry != NULL, "call most probably wrong");
1495   InstructionMark im(this);
1496   emit_int8((unsigned char)0xE8);
1497   intptr_t disp = entry - (pc() + sizeof(int32_t));
1498   assert(is_simm32(disp), "must be 32bit offset (call2)");
1499   // Technically, should use call32_operand, but this format is
1500   // implied by the fact that we're emitting a call instruction.
1501 
1502   int operand = LP64_ONLY(disp32_operand) NOT_LP64(call32_operand);
1503   emit_data((int) disp, rspec, operand);
1504 }
1505 
1506 void Assembler::cdql() {
1507   emit_int8((unsigned char)0x99);
1508 }
1509 
1510 void Assembler::cld() {
1511   emit_int8((unsigned char)0xFC);
1512 }
1513 
1514 void Assembler::cmovl(Condition cc, Register dst, Register src) {
1515   NOT_LP64(guarantee(VM_Version::supports_cmov(), "illegal instruction"));
1516   int encode = prefix_and_encode(dst->encoding(), src->encoding());
1517   emit_int8(0x0F);
1518   emit_int8(0x40 | cc);
1519   emit_int8((unsigned char)(0xC0 | encode));
1520 }
1521 
1522 
1523 void Assembler::cmovl(Condition cc, Register dst, Address src) {
1524   NOT_LP64(guarantee(VM_Version::supports_cmov(), "illegal instruction"));
1525   prefix(src, dst);
1526   emit_int8(0x0F);
1527   emit_int8(0x40 | cc);
1528   emit_operand(dst, src);
1529 }
1530 
1531 void Assembler::cmpb(Address dst, int imm8) {
1532   InstructionMark im(this);
1533   prefix(dst);
1534   emit_int8((unsigned char)0x80);
1535   emit_operand(rdi, dst, 1);
1536   emit_int8(imm8);
1537 }
1538 
1539 void Assembler::cmpl(Address dst, int32_t imm32) {
1540   InstructionMark im(this);
1541   prefix(dst);
1542   emit_int8((unsigned char)0x81);
1543   emit_operand(rdi, dst, 4);
1544   emit_int32(imm32);
1545 }
1546 
1547 void Assembler::cmpl(Register dst, int32_t imm32) {
1548   prefix(dst);
1549   emit_arith(0x81, 0xF8, dst, imm32);
1550 }
1551 
1552 void Assembler::cmpl(Register dst, Register src) {
1553   (void) prefix_and_encode(dst->encoding(), src->encoding());
1554   emit_arith(0x3B, 0xC0, dst, src);
1555 }
1556 
1557 void Assembler::cmpl(Register dst, Address  src) {
1558   InstructionMark im(this);
1559   prefix(src, dst);
1560   emit_int8((unsigned char)0x3B);
1561   emit_operand(dst, src);
1562 }
1563 
1564 void Assembler::cmpw(Address dst, int imm16) {
1565   InstructionMark im(this);
1566   assert(!dst.base_needs_rex() && !dst.index_needs_rex(), "no extended registers");
1567   emit_int8(0x66);
1568   emit_int8((unsigned char)0x81);
1569   emit_operand(rdi, dst, 2);
1570   emit_int16(imm16);
1571 }
1572 
1573 // The 32-bit cmpxchg compares the value at adr with the contents of rax,
1574 // and stores reg into adr if so; otherwise, the value at adr is loaded into rax,.
1575 // The ZF is set if the compared values were equal, and cleared otherwise.
1576 void Assembler::cmpxchgl(Register reg, Address adr) { // cmpxchg
1577   InstructionMark im(this);
1578   prefix(adr, reg);
1579   emit_int8(0x0F);
1580   emit_int8((unsigned char)0xB1);
1581   emit_operand(reg, adr);
1582 }
1583 
1584 // The 8-bit cmpxchg compares the value at adr with the contents of rax,
1585 // and stores reg into adr if so; otherwise, the value at adr is loaded into rax,.
1586 // The ZF is set if the compared values were equal, and cleared otherwise.
1587 void Assembler::cmpxchgb(Register reg, Address adr) { // cmpxchg
1588   InstructionMark im(this);
1589   prefix(adr, reg, true);
1590   emit_int8(0x0F);
1591   emit_int8((unsigned char)0xB0);
1592   emit_operand(reg, adr);
1593 }
1594 
1595 void Assembler::comisd(XMMRegister dst, Address src) {
1596   // NOTE: dbx seems to decode this as comiss even though the
1597   // 0x66 is there. Strangly ucomisd comes out correct
1598   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1599   InstructionMark im(this);
1600   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);;
1601   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
1602   simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
1603   emit_int8(0x2F);
1604   emit_operand(dst, src);
1605 }
1606 
1607 void Assembler::comisd(XMMRegister dst, XMMRegister src) {
1608   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1609   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1610   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
1611   emit_int8(0x2F);
1612   emit_int8((unsigned char)(0xC0 | encode));
1613 }
1614 
1615 void Assembler::comiss(XMMRegister dst, Address src) {
1616   NOT_LP64(assert(VM_Version::supports_sse(), ""));
1617   InstructionMark im(this);
1618   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1619   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
1620   simd_prefix(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
1621   emit_int8(0x2F);
1622   emit_operand(dst, src);
1623 }
1624 
1625 void Assembler::comiss(XMMRegister dst, XMMRegister src) {
1626   NOT_LP64(assert(VM_Version::supports_sse(), ""));
1627   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1628   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
1629   emit_int8(0x2F);
1630   emit_int8((unsigned char)(0xC0 | encode));
1631 }
1632 
1633 void Assembler::cpuid() {
1634   emit_int8(0x0F);
1635   emit_int8((unsigned char)0xA2);
1636 }
1637 
1638 // Opcode / Instruction                      Op /  En  64 - Bit Mode     Compat / Leg Mode Description                  Implemented
1639 // F2 0F 38 F0 / r       CRC32 r32, r / m8   RM        Valid             Valid             Accumulate CRC32 on r / m8.  v
1640 // F2 REX 0F 38 F0 / r   CRC32 r32, r / m8*  RM        Valid             N.E.              Accumulate CRC32 on r / m8.  -
1641 // F2 REX.W 0F 38 F0 / r CRC32 r64, r / m8   RM        Valid             N.E.              Accumulate CRC32 on r / m8.  -
1642 //
1643 // F2 0F 38 F1 / r       CRC32 r32, r / m16  RM        Valid             Valid             Accumulate CRC32 on r / m16. v
1644 //
1645 // F2 0F 38 F1 / r       CRC32 r32, r / m32  RM        Valid             Valid             Accumulate CRC32 on r / m32. v
1646 //
1647 // F2 REX.W 0F 38 F1 / r CRC32 r64, r / m64  RM        Valid             N.E.              Accumulate CRC32 on r / m64. v
1648 void Assembler::crc32(Register crc, Register v, int8_t sizeInBytes) {
1649   assert(VM_Version::supports_sse4_2(), "");
1650   int8_t w = 0x01;
1651   Prefix p = Prefix_EMPTY;
1652 
1653   emit_int8((int8_t)0xF2);
1654   switch (sizeInBytes) {
1655   case 1:
1656     w = 0;
1657     break;
1658   case 2:
1659   case 4:
1660     break;
1661   LP64_ONLY(case 8:)
1662     // This instruction is not valid in 32 bits
1663     // Note:
1664     // http://www.intel.com/content/dam/www/public/us/en/documents/manuals/64-ia-32-architectures-software-developer-instruction-set-reference-manual-325383.pdf
1665     //
1666     // Page B - 72   Vol. 2C says
1667     // qwreg2 to qwreg            1111 0010 : 0100 1R0B : 0000 1111 : 0011 1000 : 1111 0000 : 11 qwreg1 qwreg2
1668     // mem64 to qwreg             1111 0010 : 0100 1R0B : 0000 1111 : 0011 1000 : 1111 0000 : mod qwreg r / m
1669     //                                                                            F0!!!
1670     // while 3 - 208 Vol. 2A
1671     // F2 REX.W 0F 38 F1 / r       CRC32 r64, r / m64             RM         Valid      N.E.Accumulate CRC32 on r / m64.
1672     //
1673     // the 0 on a last bit is reserved for a different flavor of this instruction :
1674     // F2 REX.W 0F 38 F0 / r       CRC32 r64, r / m8              RM         Valid      N.E.Accumulate CRC32 on r / m8.
1675     p = REX_W;
1676     break;
1677   default:
1678     assert(0, "Unsupported value for a sizeInBytes argument");
1679     break;
1680   }
1681   LP64_ONLY(prefix(crc, v, p);)
1682   emit_int8((int8_t)0x0F);
1683   emit_int8(0x38);
1684   emit_int8((int8_t)(0xF0 | w));
1685   emit_int8(0xC0 | ((crc->encoding() & 0x7) << 3) | (v->encoding() & 7));
1686 }
1687 
1688 void Assembler::crc32(Register crc, Address adr, int8_t sizeInBytes) {
1689   assert(VM_Version::supports_sse4_2(), "");
1690   InstructionMark im(this);
1691   int8_t w = 0x01;
1692   Prefix p = Prefix_EMPTY;
1693 
1694   emit_int8((int8_t)0xF2);
1695   switch (sizeInBytes) {
1696   case 1:
1697     w = 0;
1698     break;
1699   case 2:
1700   case 4:
1701     break;
1702   LP64_ONLY(case 8:)
1703     // This instruction is not valid in 32 bits
1704     p = REX_W;
1705     break;
1706   default:
1707     assert(0, "Unsupported value for a sizeInBytes argument");
1708     break;
1709   }
1710   LP64_ONLY(prefix(crc, adr, p);)
1711   emit_int8((int8_t)0x0F);
1712   emit_int8(0x38);
1713   emit_int8((int8_t)(0xF0 | w));
1714   emit_operand(crc, adr);
1715 }
1716 
1717 void Assembler::cvtdq2pd(XMMRegister dst, XMMRegister src) {
1718   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1719   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
1720   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
1721   emit_int8((unsigned char)0xE6);
1722   emit_int8((unsigned char)(0xC0 | encode));
1723 }
1724 
1725 void Assembler::cvtdq2ps(XMMRegister dst, XMMRegister src) {
1726   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1727   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
1728   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
1729   emit_int8(0x5B);
1730   emit_int8((unsigned char)(0xC0 | encode));
1731 }
1732 
1733 void Assembler::cvtsd2ss(XMMRegister dst, XMMRegister src) {
1734   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1735   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
1736   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
1737   emit_int8(0x5A);
1738   emit_int8((unsigned char)(0xC0 | encode));
1739 }
1740 
1741 void Assembler::cvtsd2ss(XMMRegister dst, Address src) {
1742   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1743   InstructionMark im(this);
1744   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
1745   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
1746   simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
1747   emit_int8(0x5A);
1748   emit_operand(dst, src);
1749 }
1750 
1751 void Assembler::cvtsi2sdl(XMMRegister dst, Register src) {
1752   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1753   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1754   int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
1755   emit_int8(0x2A);
1756   emit_int8((unsigned char)(0xC0 | encode));
1757 }
1758 
1759 void Assembler::cvtsi2sdl(XMMRegister dst, Address src) {
1760   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1761   InstructionMark im(this);
1762   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1763   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
1764   simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
1765   emit_int8(0x2A);
1766   emit_operand(dst, src);
1767 }
1768 
1769 void Assembler::cvtsi2ssl(XMMRegister dst, Register src) {
1770   NOT_LP64(assert(VM_Version::supports_sse(), ""));
1771   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1772   int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
1773   emit_int8(0x2A);
1774   emit_int8((unsigned char)(0xC0 | encode));
1775 }
1776 
1777 void Assembler::cvtsi2ssl(XMMRegister dst, Address src) {
1778   NOT_LP64(assert(VM_Version::supports_sse(), ""));
1779   InstructionMark im(this);
1780   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1781   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
1782   simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
1783   emit_int8(0x2A);
1784   emit_operand(dst, src);
1785 }
1786 
1787 void Assembler::cvtsi2ssq(XMMRegister dst, Register src) {
1788   NOT_LP64(assert(VM_Version::supports_sse(), ""));
1789   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1790   int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
1791   emit_int8(0x2A);
1792   emit_int8((unsigned char)(0xC0 | encode));
1793 }
1794 
1795 void Assembler::cvtss2sd(XMMRegister dst, XMMRegister src) {
1796   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1797   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
1798   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
1799   emit_int8(0x5A);
1800   emit_int8((unsigned char)(0xC0 | encode));
1801 }
1802 
1803 void Assembler::cvtss2sd(XMMRegister dst, Address src) {
1804   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1805   InstructionMark im(this);
1806   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
1807   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
1808   simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
1809   emit_int8(0x5A);
1810   emit_operand(dst, src);
1811 }
1812 
1813 
1814 void Assembler::cvttsd2sil(Register dst, XMMRegister src) {
1815   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1816   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1817   int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
1818   emit_int8(0x2C);
1819   emit_int8((unsigned char)(0xC0 | encode));
1820 }
1821 
1822 void Assembler::cvttss2sil(Register dst, XMMRegister src) {
1823   NOT_LP64(assert(VM_Version::supports_sse(), ""));
1824   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1825   int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
1826   emit_int8(0x2C);
1827   emit_int8((unsigned char)(0xC0 | encode));
1828 }
1829 
1830 void Assembler::cvttpd2dq(XMMRegister dst, XMMRegister src) {
1831   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1832   int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_128bit;
1833   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
1834   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
1835   emit_int8((unsigned char)0xE6);
1836   emit_int8((unsigned char)(0xC0 | encode));
1837 }
1838 
1839 void Assembler::decl(Address dst) {
1840   // Don't use it directly. Use MacroAssembler::decrement() instead.
1841   InstructionMark im(this);
1842   prefix(dst);
1843   emit_int8((unsigned char)0xFF);
1844   emit_operand(rcx, dst);
1845 }
1846 
1847 void Assembler::divsd(XMMRegister dst, Address src) {
1848   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1849   InstructionMark im(this);
1850   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
1851   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
1852   simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
1853   emit_int8(0x5E);
1854   emit_operand(dst, src);
1855 }
1856 
1857 void Assembler::divsd(XMMRegister dst, XMMRegister src) {
1858   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1859   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
1860   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
1861   emit_int8(0x5E);
1862   emit_int8((unsigned char)(0xC0 | encode));
1863 }
1864 
1865 void Assembler::divss(XMMRegister dst, Address src) {
1866   NOT_LP64(assert(VM_Version::supports_sse(), ""));
1867   InstructionMark im(this);
1868   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
1869   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
1870   simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
1871   emit_int8(0x5E);
1872   emit_operand(dst, src);
1873 }
1874 
1875 void Assembler::divss(XMMRegister dst, XMMRegister src) {
1876   NOT_LP64(assert(VM_Version::supports_sse(), ""));
1877   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
1878   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
1879   emit_int8(0x5E);
1880   emit_int8((unsigned char)(0xC0 | encode));
1881 }
1882 
1883 void Assembler::emms() {
1884   NOT_LP64(assert(VM_Version::supports_mmx(), ""));
1885   emit_int8(0x0F);
1886   emit_int8(0x77);
1887 }
1888 
1889 void Assembler::hlt() {
1890   emit_int8((unsigned char)0xF4);
1891 }
1892 
1893 void Assembler::idivl(Register src) {
1894   int encode = prefix_and_encode(src->encoding());
1895   emit_int8((unsigned char)0xF7);
1896   emit_int8((unsigned char)(0xF8 | encode));
1897 }
1898 
1899 void Assembler::divl(Register src) { // Unsigned
1900   int encode = prefix_and_encode(src->encoding());
1901   emit_int8((unsigned char)0xF7);
1902   emit_int8((unsigned char)(0xF0 | encode));
1903 }
1904 
1905 void Assembler::imull(Register src) {
1906   int encode = prefix_and_encode(src->encoding());
1907   emit_int8((unsigned char)0xF7);
1908   emit_int8((unsigned char)(0xE8 | encode));
1909 }
1910 
1911 void Assembler::imull(Register dst, Register src) {
1912   int encode = prefix_and_encode(dst->encoding(), src->encoding());
1913   emit_int8(0x0F);
1914   emit_int8((unsigned char)0xAF);
1915   emit_int8((unsigned char)(0xC0 | encode));
1916 }
1917 
1918 
1919 void Assembler::imull(Register dst, Register src, int value) {
1920   int encode = prefix_and_encode(dst->encoding(), src->encoding());
1921   if (is8bit(value)) {
1922     emit_int8(0x6B);
1923     emit_int8((unsigned char)(0xC0 | encode));
1924     emit_int8(value & 0xFF);
1925   } else {
1926     emit_int8(0x69);
1927     emit_int8((unsigned char)(0xC0 | encode));
1928     emit_int32(value);
1929   }
1930 }
1931 
1932 void Assembler::imull(Register dst, Address src) {
1933   InstructionMark im(this);
1934   prefix(src, dst);
1935   emit_int8(0x0F);
1936   emit_int8((unsigned char) 0xAF);
1937   emit_operand(dst, src);
1938 }
1939 
1940 
1941 void Assembler::incl(Address dst) {
1942   // Don't use it directly. Use MacroAssembler::increment() instead.
1943   InstructionMark im(this);
1944   prefix(dst);
1945   emit_int8((unsigned char)0xFF);
1946   emit_operand(rax, dst);
1947 }
1948 
1949 void Assembler::jcc(Condition cc, Label& L, bool maybe_short) {
1950   InstructionMark im(this);
1951   assert((0 <= cc) && (cc < 16), "illegal cc");
1952   if (L.is_bound()) {
1953     address dst = target(L);
1954     assert(dst != NULL, "jcc most probably wrong");
1955 
1956     const int short_size = 2;
1957     const int long_size = 6;
1958     intptr_t offs = (intptr_t)dst - (intptr_t)pc();
1959     if (maybe_short && is8bit(offs - short_size)) {
1960       // 0111 tttn #8-bit disp
1961       emit_int8(0x70 | cc);
1962       emit_int8((offs - short_size) & 0xFF);
1963     } else {
1964       // 0000 1111 1000 tttn #32-bit disp
1965       assert(is_simm32(offs - long_size),
1966              "must be 32bit offset (call4)");
1967       emit_int8(0x0F);
1968       emit_int8((unsigned char)(0x80 | cc));
1969       emit_int32(offs - long_size);
1970     }
1971   } else {
1972     // Note: could eliminate cond. jumps to this jump if condition
1973     //       is the same however, seems to be rather unlikely case.
1974     // Note: use jccb() if label to be bound is very close to get
1975     //       an 8-bit displacement
1976     L.add_patch_at(code(), locator());
1977     emit_int8(0x0F);
1978     emit_int8((unsigned char)(0x80 | cc));
1979     emit_int32(0);
1980   }
1981 }
1982 
1983 void Assembler::jccb(Condition cc, Label& L) {
1984   if (L.is_bound()) {
1985     const int short_size = 2;
1986     address entry = target(L);
1987 #ifdef ASSERT
1988     intptr_t dist = (intptr_t)entry - ((intptr_t)pc() + short_size);
1989     intptr_t delta = short_branch_delta();
1990     if (delta != 0) {
1991       dist += (dist < 0 ? (-delta) :delta);
1992     }
1993     assert(is8bit(dist), "Dispacement too large for a short jmp");
1994 #endif
1995     intptr_t offs = (intptr_t)entry - (intptr_t)pc();
1996     // 0111 tttn #8-bit disp
1997     emit_int8(0x70 | cc);
1998     emit_int8((offs - short_size) & 0xFF);
1999   } else {
2000     InstructionMark im(this);
2001     L.add_patch_at(code(), locator());
2002     emit_int8(0x70 | cc);
2003     emit_int8(0);
2004   }
2005 }
2006 
2007 void Assembler::jmp(Address adr) {
2008   InstructionMark im(this);
2009   prefix(adr);
2010   emit_int8((unsigned char)0xFF);
2011   emit_operand(rsp, adr);
2012 }
2013 
2014 void Assembler::jmp(Label& L, bool maybe_short) {
2015   if (L.is_bound()) {
2016     address entry = target(L);
2017     assert(entry != NULL, "jmp most probably wrong");
2018     InstructionMark im(this);
2019     const int short_size = 2;
2020     const int long_size = 5;
2021     intptr_t offs = entry - pc();
2022     if (maybe_short && is8bit(offs - short_size)) {
2023       emit_int8((unsigned char)0xEB);
2024       emit_int8((offs - short_size) & 0xFF);
2025     } else {
2026       emit_int8((unsigned char)0xE9);
2027       emit_int32(offs - long_size);
2028     }
2029   } else {
2030     // By default, forward jumps are always 32-bit displacements, since
2031     // we can't yet know where the label will be bound.  If you're sure that
2032     // the forward jump will not run beyond 256 bytes, use jmpb to
2033     // force an 8-bit displacement.
2034     InstructionMark im(this);
2035     L.add_patch_at(code(), locator());
2036     emit_int8((unsigned char)0xE9);
2037     emit_int32(0);
2038   }
2039 }
2040 
2041 void Assembler::jmp(Register entry) {
2042   int encode = prefix_and_encode(entry->encoding());
2043   emit_int8((unsigned char)0xFF);
2044   emit_int8((unsigned char)(0xE0 | encode));
2045 }
2046 
2047 void Assembler::jmp_literal(address dest, RelocationHolder const& rspec) {
2048   InstructionMark im(this);
2049   emit_int8((unsigned char)0xE9);
2050   assert(dest != NULL, "must have a target");
2051   intptr_t disp = dest - (pc() + sizeof(int32_t));
2052   assert(is_simm32(disp), "must be 32bit offset (jmp)");
2053   emit_data(disp, rspec.reloc(), call32_operand);
2054 }
2055 
2056 void Assembler::jmpb(Label& L) {
2057   if (L.is_bound()) {
2058     const int short_size = 2;
2059     address entry = target(L);
2060     assert(entry != NULL, "jmp most probably wrong");
2061 #ifdef ASSERT
2062     intptr_t dist = (intptr_t)entry - ((intptr_t)pc() + short_size);
2063     intptr_t delta = short_branch_delta();
2064     if (delta != 0) {
2065       dist += (dist < 0 ? (-delta) :delta);
2066     }
2067     assert(is8bit(dist), "Dispacement too large for a short jmp");
2068 #endif
2069     intptr_t offs = entry - pc();
2070     emit_int8((unsigned char)0xEB);
2071     emit_int8((offs - short_size) & 0xFF);
2072   } else {
2073     InstructionMark im(this);
2074     L.add_patch_at(code(), locator());
2075     emit_int8((unsigned char)0xEB);
2076     emit_int8(0);
2077   }
2078 }
2079 
2080 void Assembler::ldmxcsr( Address src) {
2081   NOT_LP64(assert(VM_Version::supports_sse(), ""));
2082   InstructionMark im(this);
2083   prefix(src);
2084   emit_int8(0x0F);
2085   emit_int8((unsigned char)0xAE);
2086   emit_operand(as_Register(2), src);
2087 }
2088 
2089 void Assembler::leal(Register dst, Address src) {
2090   InstructionMark im(this);
2091 #ifdef _LP64
2092   emit_int8(0x67); // addr32
2093   prefix(src, dst);
2094 #endif // LP64
2095   emit_int8((unsigned char)0x8D);
2096   emit_operand(dst, src);
2097 }
2098 
2099 void Assembler::lfence() {
2100   emit_int8(0x0F);
2101   emit_int8((unsigned char)0xAE);
2102   emit_int8((unsigned char)0xE8);
2103 }
2104 
2105 void Assembler::lock() {
2106   emit_int8((unsigned char)0xF0);
2107 }
2108 
2109 void Assembler::lzcntl(Register dst, Register src) {
2110   assert(VM_Version::supports_lzcnt(), "encoding is treated as BSR");
2111   emit_int8((unsigned char)0xF3);
2112   int encode = prefix_and_encode(dst->encoding(), src->encoding());
2113   emit_int8(0x0F);
2114   emit_int8((unsigned char)0xBD);
2115   emit_int8((unsigned char)(0xC0 | encode));
2116 }
2117 
2118 // Emit mfence instruction
2119 void Assembler::mfence() {
2120   NOT_LP64(assert(VM_Version::supports_sse2(), "unsupported");)
2121   emit_int8(0x0F);
2122   emit_int8((unsigned char)0xAE);
2123   emit_int8((unsigned char)0xF0);
2124 }
2125 
2126 void Assembler::mov(Register dst, Register src) {
2127   LP64_ONLY(movq(dst, src)) NOT_LP64(movl(dst, src));
2128 }
2129 
2130 void Assembler::movapd(XMMRegister dst, XMMRegister src) {
2131   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2132   int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_128bit;
2133   InstructionAttr attributes(vector_len, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
2134   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2135   emit_int8(0x28);
2136   emit_int8((unsigned char)(0xC0 | encode));
2137 }
2138 
2139 void Assembler::movaps(XMMRegister dst, XMMRegister src) {
2140   NOT_LP64(assert(VM_Version::supports_sse(), ""));
2141   int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_128bit;
2142   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
2143   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2144   emit_int8(0x28);
2145   emit_int8((unsigned char)(0xC0 | encode));
2146 }
2147 
2148 void Assembler::movlhps(XMMRegister dst, XMMRegister src) {
2149   NOT_LP64(assert(VM_Version::supports_sse(), ""));
2150   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2151   int encode = simd_prefix_and_encode(dst, src, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2152   emit_int8(0x16);
2153   emit_int8((unsigned char)(0xC0 | encode));
2154 }
2155 
2156 void Assembler::movb(Register dst, Address src) {
2157   NOT_LP64(assert(dst->has_byte_register(), "must have byte register"));
2158   InstructionMark im(this);
2159   prefix(src, dst, true);
2160   emit_int8((unsigned char)0x8A);
2161   emit_operand(dst, src);
2162 }
2163 
2164 void Assembler::movddup(XMMRegister dst, XMMRegister src) {
2165   NOT_LP64(assert(VM_Version::supports_sse3(), ""));
2166   int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_128bit;
2167   InstructionAttr attributes(vector_len, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2168   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2169   emit_int8(0x12);
2170   emit_int8(0xC0 | encode);
2171 }
2172 
2173 void Assembler::kmovbl(KRegister dst, Register src) {
2174   assert(VM_Version::supports_avx512dq(), "");
2175   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2176   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2177   emit_int8((unsigned char)0x92);
2178   emit_int8((unsigned char)(0xC0 | encode));
2179 }
2180 
2181 void Assembler::kmovbl(Register dst, KRegister src) {
2182   assert(VM_Version::supports_avx512dq(), "");
2183   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2184   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2185   emit_int8((unsigned char)0x93);
2186   emit_int8((unsigned char)(0xC0 | encode));
2187 }
2188 
2189 void Assembler::kmovwl(KRegister dst, Register src) {
2190   assert(VM_Version::supports_evex(), "");
2191   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2192   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2193   emit_int8((unsigned char)0x92);
2194   emit_int8((unsigned char)(0xC0 | encode));
2195 }
2196 
2197 void Assembler::kmovwl(Register dst, KRegister src) {
2198   assert(VM_Version::supports_evex(), "");
2199   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2200   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2201   emit_int8((unsigned char)0x93);
2202   emit_int8((unsigned char)(0xC0 | encode));
2203 }
2204 
2205 void Assembler::kmovdl(KRegister dst, Register src) {
2206   assert(VM_Version::supports_avx512bw(), "");
2207   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2208   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2209   emit_int8((unsigned char)0x92);
2210   emit_int8((unsigned char)(0xC0 | encode));
2211 }
2212 
2213 void Assembler::kmovdl(Register dst, KRegister src) {
2214   assert(VM_Version::supports_avx512bw(), "");
2215   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2216   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2217   emit_int8((unsigned char)0x93);
2218   emit_int8((unsigned char)(0xC0 | encode));
2219 }
2220 
2221 void Assembler::kmovql(KRegister dst, KRegister src) {
2222   assert(VM_Version::supports_avx512bw(), "");
2223   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2224   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2225   emit_int8((unsigned char)0x90);
2226   emit_int8((unsigned char)(0xC0 | encode));
2227 }
2228 
2229 void Assembler::kmovql(KRegister dst, Address src) {
2230   assert(VM_Version::supports_avx512bw(), "");
2231   InstructionMark im(this);
2232   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2233   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2234   emit_int8((unsigned char)0x90);
2235   emit_operand((Register)dst, src);
2236 }
2237 
2238 void Assembler::kmovql(Address dst, KRegister src) {
2239   assert(VM_Version::supports_avx512bw(), "");
2240   InstructionMark im(this);
2241   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2242   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2243   emit_int8((unsigned char)0x90);
2244   emit_operand((Register)src, dst);
2245 }
2246 
2247 void Assembler::kmovql(KRegister dst, Register src) {
2248   assert(VM_Version::supports_avx512bw(), "");
2249   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2250   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2251   emit_int8((unsigned char)0x92);
2252   emit_int8((unsigned char)(0xC0 | encode));
2253 }
2254 
2255 void Assembler::kmovql(Register dst, KRegister src) {
2256   assert(VM_Version::supports_avx512bw(), "");
2257   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2258   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2259   emit_int8((unsigned char)0x93);
2260   emit_int8((unsigned char)(0xC0 | encode));
2261 }
2262 
2263 // This instruction produces ZF or CF flags
2264 void Assembler::kortestbl(KRegister src1, KRegister src2) {
2265   assert(VM_Version::supports_avx512dq(), "");
2266   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2267   int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2268   emit_int8((unsigned char)0x98);
2269   emit_int8((unsigned char)(0xC0 | encode));
2270 }
2271 
2272 // This instruction produces ZF or CF flags
2273 void Assembler::kortestwl(KRegister src1, KRegister src2) {
2274   assert(VM_Version::supports_evex(), "");
2275   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2276   int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2277   emit_int8((unsigned char)0x98);
2278   emit_int8((unsigned char)(0xC0 | encode));
2279 }
2280 
2281 // This instruction produces ZF or CF flags
2282 void Assembler::kortestdl(KRegister src1, KRegister src2) {
2283   assert(VM_Version::supports_avx512bw(), "");
2284   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2285   int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2286   emit_int8((unsigned char)0x98);
2287   emit_int8((unsigned char)(0xC0 | encode));
2288 }
2289 
2290 // This instruction produces ZF or CF flags
2291 void Assembler::kortestql(KRegister src1, KRegister src2) {
2292   assert(VM_Version::supports_avx512bw(), "");
2293   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2294   int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2295   emit_int8((unsigned char)0x98);
2296   emit_int8((unsigned char)(0xC0 | encode));
2297 }
2298 
2299 void Assembler::movb(Address dst, int imm8) {
2300   InstructionMark im(this);
2301    prefix(dst);
2302   emit_int8((unsigned char)0xC6);
2303   emit_operand(rax, dst, 1);
2304   emit_int8(imm8);
2305 }
2306 
2307 
2308 void Assembler::movb(Address dst, Register src) {
2309   assert(src->has_byte_register(), "must have byte register");
2310   InstructionMark im(this);
2311   prefix(dst, src, true);
2312   emit_int8((unsigned char)0x88);
2313   emit_operand(src, dst);
2314 }
2315 
2316 void Assembler::movdl(XMMRegister dst, Register src) {
2317   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2318   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2319   int encode = simd_prefix_and_encode(dst, xnoreg, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2320   emit_int8(0x6E);
2321   emit_int8((unsigned char)(0xC0 | encode));
2322 }
2323 
2324 void Assembler::movdl(Register dst, XMMRegister src) {
2325   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2326   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2327   // swap src/dst to get correct prefix
2328   int encode = simd_prefix_and_encode(src, xnoreg, as_XMMRegister(dst->encoding()), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2329   emit_int8(0x7E);
2330   emit_int8((unsigned char)(0xC0 | encode));
2331 }
2332 
2333 void Assembler::movdl(XMMRegister dst, Address src) {
2334   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2335   InstructionMark im(this);
2336   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2337   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
2338   simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2339   emit_int8(0x6E);
2340   emit_operand(dst, src);
2341 }
2342 
2343 void Assembler::movdl(Address dst, XMMRegister src) {
2344   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2345   InstructionMark im(this);
2346   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2347   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
2348   simd_prefix(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2349   emit_int8(0x7E);
2350   emit_operand(src, dst);
2351 }
2352 
2353 void Assembler::movdqa(XMMRegister dst, XMMRegister src) {
2354   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2355   int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_128bit;
2356   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2357   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2358   emit_int8(0x6F);
2359   emit_int8((unsigned char)(0xC0 | encode));
2360 }
2361 
2362 void Assembler::movdqa(XMMRegister dst, Address src) {
2363   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2364   InstructionMark im(this);
2365   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2366   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2367   simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2368   emit_int8(0x6F);
2369   emit_operand(dst, src);
2370 }
2371 
2372 void Assembler::movdqa(Address dst, XMMRegister src) {
2373   assert(VM_Version::supports_sse2(), "");
2374   InstructionMark im(this);
2375   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2376   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2377   simd_prefix(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2378   emit_int8(0x7F);
2379   emit_operand(src, dst);
2380 }
2381 
2382 void Assembler::vmovdqa(XMMRegister dst, Address src) {
2383   assert(UseAVX > 0, "");
2384   InstructionMark im(this);
2385   InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2386   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2387   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2388   emit_int8(0x6F);
2389   emit_operand(dst, src);
2390 }
2391 
2392 void Assembler::vmovdqa(Address dst, XMMRegister src) {
2393   assert(UseAVX > 0, "");
2394   InstructionMark im(this);
2395   InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2396   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2397   // swap src<->dst for encoding
2398   assert(src != xnoreg, "sanity");
2399   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2400   emit_int8(0x7F);
2401   emit_operand(src, dst);
2402 }
2403 
2404 
2405 void Assembler::movdqu(XMMRegister dst, Address src) {
2406   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2407   InstructionMark im(this);
2408   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2409   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2410   simd_prefix(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2411   emit_int8(0x6F);
2412   emit_operand(dst, src);
2413 }
2414 
2415 void Assembler::movdqu(XMMRegister dst, XMMRegister src) {
2416   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2417   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2418   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2419   emit_int8(0x6F);
2420   emit_int8((unsigned char)(0xC0 | encode));
2421 }
2422 
2423 void Assembler::movdqu(Address dst, XMMRegister src) {
2424   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2425   InstructionMark im(this);
2426   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2427   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2428   simd_prefix(src, xnoreg, dst, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2429   emit_int8(0x7F);
2430   emit_operand(src, dst);
2431 }
2432 
2433 // Move Unaligned 256bit Vector
2434 void Assembler::vmovdqu(XMMRegister dst, XMMRegister src) {
2435   assert(UseAVX > 0, "");
2436   InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2437   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2438   emit_int8(0x6F);
2439   emit_int8((unsigned char)(0xC0 | encode));
2440 }
2441 
2442 void Assembler::vmovdqu(XMMRegister dst, Address src) {
2443   assert(UseAVX > 0, "");
2444   InstructionMark im(this);
2445   InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2446   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2447   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2448   emit_int8(0x6F);
2449   emit_operand(dst, src);
2450 }
2451 
2452 void Assembler::vmovdqu(Address dst, XMMRegister src) {
2453   assert(UseAVX > 0, "");
2454   InstructionMark im(this);
2455   InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2456   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2457   // swap src<->dst for encoding
2458   assert(src != xnoreg, "sanity");
2459   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2460   emit_int8(0x7F);
2461   emit_operand(src, dst);
2462 }
2463 
2464 // Move Unaligned EVEX enabled Vector (programmable : 8,16,32,64)
2465 void Assembler::evmovdqub(XMMRegister dst, XMMRegister src, int vector_len) {
2466   assert(VM_Version::supports_evex(), "");
2467   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
2468   int prefix = (_legacy_mode_bw) ? VEX_SIMD_F2 : VEX_SIMD_F3;
2469   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), (Assembler::VexSimdPrefix)prefix, VEX_OPCODE_0F, &attributes);
2470   emit_int8(0x6F);
2471   emit_int8((unsigned char)(0xC0 | encode));
2472 }
2473 
2474 void Assembler::evmovdqub(XMMRegister dst, Address src, int vector_len) {
2475   assert(VM_Version::supports_evex(), "");
2476   InstructionMark im(this);
2477   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
2478   int prefix = (_legacy_mode_bw) ? VEX_SIMD_F2 : VEX_SIMD_F3;
2479   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2480   vex_prefix(src, 0, dst->encoding(), (Assembler::VexSimdPrefix)prefix, VEX_OPCODE_0F, &attributes);
2481   emit_int8(0x6F);
2482   emit_operand(dst, src);
2483 }
2484 
2485 void Assembler::evmovdqub(Address dst, XMMRegister src, int vector_len) {
2486   assert(VM_Version::supports_evex(), "");
2487   assert(src != xnoreg, "sanity");
2488   InstructionMark im(this);
2489   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
2490   int prefix = (_legacy_mode_bw) ? VEX_SIMD_F2 : VEX_SIMD_F3;
2491   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2492   vex_prefix(dst, 0, src->encoding(), (Assembler::VexSimdPrefix)prefix, VEX_OPCODE_0F, &attributes);
2493   emit_int8(0x7F);
2494   emit_operand(src, dst);
2495 }
2496 
2497 void Assembler::evmovdquw(XMMRegister dst, XMMRegister src, int vector_len) {
2498   assert(VM_Version::supports_evex(), "");
2499   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
2500   int prefix = (_legacy_mode_bw) ? VEX_SIMD_F2 : VEX_SIMD_F3;
2501   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), (Assembler::VexSimdPrefix)prefix, VEX_OPCODE_0F, &attributes);
2502   emit_int8(0x6F);
2503   emit_int8((unsigned char)(0xC0 | encode));
2504 }
2505 
2506 void Assembler::evmovdquw(XMMRegister dst, Address src, int vector_len) {
2507   assert(VM_Version::supports_evex(), "");
2508   InstructionMark im(this);
2509   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
2510   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2511   int prefix = (_legacy_mode_bw) ? VEX_SIMD_F2 : VEX_SIMD_F3;
2512   vex_prefix(src, 0, dst->encoding(), (Assembler::VexSimdPrefix)prefix, VEX_OPCODE_0F, &attributes);
2513   emit_int8(0x6F);
2514   emit_operand(dst, src);
2515 }
2516 
2517 void Assembler::evmovdquw(Address dst, XMMRegister src, int vector_len) {
2518   assert(VM_Version::supports_evex(), "");
2519   assert(src != xnoreg, "sanity");
2520   InstructionMark im(this);
2521   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
2522   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2523   int prefix = (_legacy_mode_bw) ? VEX_SIMD_F2 : VEX_SIMD_F3;
2524   vex_prefix(dst, 0, src->encoding(), (Assembler::VexSimdPrefix)prefix, VEX_OPCODE_0F, &attributes);
2525   emit_int8(0x7F);
2526   emit_operand(src, dst);
2527 }
2528 
2529 void Assembler::evmovdqul(XMMRegister dst, XMMRegister src, int vector_len) {
2530   assert(VM_Version::supports_evex(), "");
2531   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2532   attributes.set_is_evex_instruction();
2533   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2534   emit_int8(0x6F);
2535   emit_int8((unsigned char)(0xC0 | encode));
2536 }
2537 
2538 void Assembler::evmovdqul(XMMRegister dst, Address src, int vector_len) {
2539   assert(VM_Version::supports_evex(), "");
2540   InstructionMark im(this);
2541   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false , /* uses_vl */ true);
2542   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2543   attributes.set_is_evex_instruction();
2544   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2545   emit_int8(0x6F);
2546   emit_operand(dst, src);
2547 }
2548 
2549 void Assembler::evmovdqul(Address dst, XMMRegister src, int vector_len) {
2550   assert(VM_Version::supports_evex(), "");
2551   assert(src != xnoreg, "sanity");
2552   InstructionMark im(this);
2553   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2554   attributes.set_is_evex_instruction();
2555   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2556   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2557   emit_int8(0x7F);
2558   emit_operand(src, dst);
2559 }
2560 
2561 void Assembler::evmovdquq(XMMRegister dst, XMMRegister src, int vector_len) {
2562   assert(VM_Version::supports_evex(), "");
2563   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2564   attributes.set_is_evex_instruction();
2565   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2566   emit_int8(0x6F);
2567   emit_int8((unsigned char)(0xC0 | encode));
2568 }
2569 
2570 void Assembler::evmovdquq(XMMRegister dst, Address src, int vector_len) {
2571   assert(VM_Version::supports_evex(), "");
2572   InstructionMark im(this);
2573   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2574   attributes.set_is_evex_instruction();
2575   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2576   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2577   emit_int8(0x6F);
2578   emit_operand(dst, src);
2579 }
2580 
2581 void Assembler::evmovdquq(Address dst, XMMRegister src, int vector_len) {
2582   assert(VM_Version::supports_evex(), "");
2583   assert(src != xnoreg, "sanity");
2584   InstructionMark im(this);
2585   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2586   attributes.set_is_evex_instruction();
2587   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2588   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2589   emit_int8(0x7F);
2590   emit_operand(src, dst);
2591 }
2592 
2593 // Uses zero extension on 64bit
2594 
2595 void Assembler::movl(Register dst, int32_t imm32) {
2596   int encode = prefix_and_encode(dst->encoding());
2597   emit_int8((unsigned char)(0xB8 | encode));
2598   emit_int32(imm32);
2599 }
2600 
2601 void Assembler::movl(Register dst, Register src) {
2602   int encode = prefix_and_encode(dst->encoding(), src->encoding());
2603   emit_int8((unsigned char)0x8B);
2604   emit_int8((unsigned char)(0xC0 | encode));
2605 }
2606 
2607 void Assembler::movl(Register dst, Address src) {
2608   InstructionMark im(this);
2609   prefix(src, dst);
2610   emit_int8((unsigned char)0x8B);
2611   emit_operand(dst, src);
2612 }
2613 
2614 void Assembler::movl(Address dst, int32_t imm32) {
2615   InstructionMark im(this);
2616   prefix(dst);
2617   emit_int8((unsigned char)0xC7);
2618   emit_operand(rax, dst, 4);
2619   emit_int32(imm32);
2620 }
2621 
2622 void Assembler::movl(Address dst, Register src) {
2623   InstructionMark im(this);
2624   prefix(dst, src);
2625   emit_int8((unsigned char)0x89);
2626   emit_operand(src, dst);
2627 }
2628 
2629 // New cpus require to use movsd and movss to avoid partial register stall
2630 // when loading from memory. But for old Opteron use movlpd instead of movsd.
2631 // The selection is done in MacroAssembler::movdbl() and movflt().
2632 void Assembler::movlpd(XMMRegister dst, Address src) {
2633   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2634   InstructionMark im(this);
2635   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2636   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
2637   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2638   emit_int8(0x12);
2639   emit_operand(dst, src);
2640 }
2641 
2642 void Assembler::movq( MMXRegister dst, Address src ) {
2643   assert( VM_Version::supports_mmx(), "" );
2644   emit_int8(0x0F);
2645   emit_int8(0x6F);
2646   emit_operand(dst, src);
2647 }
2648 
2649 void Assembler::movq( Address dst, MMXRegister src ) {
2650   assert( VM_Version::supports_mmx(), "" );
2651   emit_int8(0x0F);
2652   emit_int8(0x7F);
2653   // workaround gcc (3.2.1-7a) bug
2654   // In that version of gcc with only an emit_operand(MMX, Address)
2655   // gcc will tail jump and try and reverse the parameters completely
2656   // obliterating dst in the process. By having a version available
2657   // that doesn't need to swap the args at the tail jump the bug is
2658   // avoided.
2659   emit_operand(dst, src);
2660 }
2661 
2662 void Assembler::movq(XMMRegister dst, Address src) {
2663   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2664   InstructionMark im(this);
2665   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2666   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
2667   simd_prefix(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2668   emit_int8(0x7E);
2669   emit_operand(dst, src);
2670 }
2671 
2672 void Assembler::movq(Address dst, XMMRegister src) {
2673   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2674   InstructionMark im(this);
2675   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2676   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
2677   simd_prefix(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2678   emit_int8((unsigned char)0xD6);
2679   emit_operand(src, dst);
2680 }
2681 
2682 void Assembler::movsbl(Register dst, Address src) { // movsxb
2683   InstructionMark im(this);
2684   prefix(src, dst);
2685   emit_int8(0x0F);
2686   emit_int8((unsigned char)0xBE);
2687   emit_operand(dst, src);
2688 }
2689 
2690 void Assembler::movsbl(Register dst, Register src) { // movsxb
2691   NOT_LP64(assert(src->has_byte_register(), "must have byte register"));
2692   int encode = prefix_and_encode(dst->encoding(), false, src->encoding(), true);
2693   emit_int8(0x0F);
2694   emit_int8((unsigned char)0xBE);
2695   emit_int8((unsigned char)(0xC0 | encode));
2696 }
2697 
2698 void Assembler::movsd(XMMRegister dst, XMMRegister src) {
2699   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2700   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
2701   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2702   emit_int8(0x10);
2703   emit_int8((unsigned char)(0xC0 | encode));
2704 }
2705 
2706 void Assembler::movsd(XMMRegister dst, Address src) {
2707   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2708   InstructionMark im(this);
2709   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
2710   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
2711   simd_prefix(dst, xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2712   emit_int8(0x10);
2713   emit_operand(dst, src);
2714 }
2715 
2716 void Assembler::movsd(Address dst, XMMRegister src) {
2717   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2718   InstructionMark im(this);
2719   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
2720   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
2721   simd_prefix(src, xnoreg, dst, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2722   emit_int8(0x11);
2723   emit_operand(src, dst);
2724 }
2725 
2726 void Assembler::movss(XMMRegister dst, XMMRegister src) {
2727   NOT_LP64(assert(VM_Version::supports_sse(), ""));
2728   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
2729   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2730   emit_int8(0x10);
2731   emit_int8((unsigned char)(0xC0 | encode));
2732 }
2733 
2734 void Assembler::movss(XMMRegister dst, Address src) {
2735   NOT_LP64(assert(VM_Version::supports_sse(), ""));
2736   InstructionMark im(this);
2737   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
2738   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
2739   simd_prefix(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2740   emit_int8(0x10);
2741   emit_operand(dst, src);
2742 }
2743 
2744 void Assembler::movss(Address dst, XMMRegister src) {
2745   NOT_LP64(assert(VM_Version::supports_sse(), ""));
2746   InstructionMark im(this);
2747   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
2748   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
2749   simd_prefix(src, xnoreg, dst, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2750   emit_int8(0x11);
2751   emit_operand(src, dst);
2752 }
2753 
2754 void Assembler::movswl(Register dst, Address src) { // movsxw
2755   InstructionMark im(this);
2756   prefix(src, dst);
2757   emit_int8(0x0F);
2758   emit_int8((unsigned char)0xBF);
2759   emit_operand(dst, src);
2760 }
2761 
2762 void Assembler::movswl(Register dst, Register src) { // movsxw
2763   int encode = prefix_and_encode(dst->encoding(), src->encoding());
2764   emit_int8(0x0F);
2765   emit_int8((unsigned char)0xBF);
2766   emit_int8((unsigned char)(0xC0 | encode));
2767 }
2768 
2769 void Assembler::movw(Address dst, int imm16) {
2770   InstructionMark im(this);
2771 
2772   emit_int8(0x66); // switch to 16-bit mode
2773   prefix(dst);
2774   emit_int8((unsigned char)0xC7);
2775   emit_operand(rax, dst, 2);
2776   emit_int16(imm16);
2777 }
2778 
2779 void Assembler::movw(Register dst, Address src) {
2780   InstructionMark im(this);
2781   emit_int8(0x66);
2782   prefix(src, dst);
2783   emit_int8((unsigned char)0x8B);
2784   emit_operand(dst, src);
2785 }
2786 
2787 void Assembler::movw(Address dst, Register src) {
2788   InstructionMark im(this);
2789   emit_int8(0x66);
2790   prefix(dst, src);
2791   emit_int8((unsigned char)0x89);
2792   emit_operand(src, dst);
2793 }
2794 
2795 void Assembler::movzbl(Register dst, Address src) { // movzxb
2796   InstructionMark im(this);
2797   prefix(src, dst);
2798   emit_int8(0x0F);
2799   emit_int8((unsigned char)0xB6);
2800   emit_operand(dst, src);
2801 }
2802 
2803 void Assembler::movzbl(Register dst, Register src) { // movzxb
2804   NOT_LP64(assert(src->has_byte_register(), "must have byte register"));
2805   int encode = prefix_and_encode(dst->encoding(), false, src->encoding(), true);
2806   emit_int8(0x0F);
2807   emit_int8((unsigned char)0xB6);
2808   emit_int8(0xC0 | encode);
2809 }
2810 
2811 void Assembler::movzwl(Register dst, Address src) { // movzxw
2812   InstructionMark im(this);
2813   prefix(src, dst);
2814   emit_int8(0x0F);
2815   emit_int8((unsigned char)0xB7);
2816   emit_operand(dst, src);
2817 }
2818 
2819 void Assembler::movzwl(Register dst, Register src) { // movzxw
2820   int encode = prefix_and_encode(dst->encoding(), src->encoding());
2821   emit_int8(0x0F);
2822   emit_int8((unsigned char)0xB7);
2823   emit_int8(0xC0 | encode);
2824 }
2825 
2826 void Assembler::mull(Address src) {
2827   InstructionMark im(this);
2828   prefix(src);
2829   emit_int8((unsigned char)0xF7);
2830   emit_operand(rsp, src);
2831 }
2832 
2833 void Assembler::mull(Register src) {
2834   int encode = prefix_and_encode(src->encoding());
2835   emit_int8((unsigned char)0xF7);
2836   emit_int8((unsigned char)(0xE0 | encode));
2837 }
2838 
2839 void Assembler::mulsd(XMMRegister dst, Address src) {
2840   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2841   InstructionMark im(this);
2842   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
2843   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
2844   simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2845   emit_int8(0x59);
2846   emit_operand(dst, src);
2847 }
2848 
2849 void Assembler::mulsd(XMMRegister dst, XMMRegister src) {
2850   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2851   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
2852   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2853   emit_int8(0x59);
2854   emit_int8((unsigned char)(0xC0 | encode));
2855 }
2856 
2857 void Assembler::mulss(XMMRegister dst, Address src) {
2858   NOT_LP64(assert(VM_Version::supports_sse(), ""));
2859   InstructionMark im(this);
2860   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
2861   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
2862   simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2863   emit_int8(0x59);
2864   emit_operand(dst, src);
2865 }
2866 
2867 void Assembler::mulss(XMMRegister dst, XMMRegister src) {
2868   NOT_LP64(assert(VM_Version::supports_sse(), ""));
2869   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
2870   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2871   emit_int8(0x59);
2872   emit_int8((unsigned char)(0xC0 | encode));
2873 }
2874 
2875 void Assembler::negl(Register dst) {
2876   int encode = prefix_and_encode(dst->encoding());
2877   emit_int8((unsigned char)0xF7);
2878   emit_int8((unsigned char)(0xD8 | encode));
2879 }
2880 
2881 void Assembler::nop(int i) {
2882 #ifdef ASSERT
2883   assert(i > 0, " ");
2884   // The fancy nops aren't currently recognized by debuggers making it a
2885   // pain to disassemble code while debugging. If asserts are on clearly
2886   // speed is not an issue so simply use the single byte traditional nop
2887   // to do alignment.
2888 
2889   for (; i > 0 ; i--) emit_int8((unsigned char)0x90);
2890   return;
2891 
2892 #endif // ASSERT
2893 
2894   if (UseAddressNop && VM_Version::is_intel()) {
2895     //
2896     // Using multi-bytes nops "0x0F 0x1F [address]" for Intel
2897     //  1: 0x90
2898     //  2: 0x66 0x90
2899     //  3: 0x66 0x66 0x90 (don't use "0x0F 0x1F 0x00" - need patching safe padding)
2900     //  4: 0x0F 0x1F 0x40 0x00
2901     //  5: 0x0F 0x1F 0x44 0x00 0x00
2902     //  6: 0x66 0x0F 0x1F 0x44 0x00 0x00
2903     //  7: 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00
2904     //  8: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
2905     //  9: 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
2906     // 10: 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
2907     // 11: 0x66 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
2908 
2909     // The rest coding is Intel specific - don't use consecutive address nops
2910 
2911     // 12: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
2912     // 13: 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
2913     // 14: 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
2914     // 15: 0x66 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
2915 
2916     while(i >= 15) {
2917       // For Intel don't generate consecutive addess nops (mix with regular nops)
2918       i -= 15;
2919       emit_int8(0x66);   // size prefix
2920       emit_int8(0x66);   // size prefix
2921       emit_int8(0x66);   // size prefix
2922       addr_nop_8();
2923       emit_int8(0x66);   // size prefix
2924       emit_int8(0x66);   // size prefix
2925       emit_int8(0x66);   // size prefix
2926       emit_int8((unsigned char)0x90);
2927                          // nop
2928     }
2929     switch (i) {
2930       case 14:
2931         emit_int8(0x66); // size prefix
2932       case 13:
2933         emit_int8(0x66); // size prefix
2934       case 12:
2935         addr_nop_8();
2936         emit_int8(0x66); // size prefix
2937         emit_int8(0x66); // size prefix
2938         emit_int8(0x66); // size prefix
2939         emit_int8((unsigned char)0x90);
2940                          // nop
2941         break;
2942       case 11:
2943         emit_int8(0x66); // size prefix
2944       case 10:
2945         emit_int8(0x66); // size prefix
2946       case 9:
2947         emit_int8(0x66); // size prefix
2948       case 8:
2949         addr_nop_8();
2950         break;
2951       case 7:
2952         addr_nop_7();
2953         break;
2954       case 6:
2955         emit_int8(0x66); // size prefix
2956       case 5:
2957         addr_nop_5();
2958         break;
2959       case 4:
2960         addr_nop_4();
2961         break;
2962       case 3:
2963         // Don't use "0x0F 0x1F 0x00" - need patching safe padding
2964         emit_int8(0x66); // size prefix
2965       case 2:
2966         emit_int8(0x66); // size prefix
2967       case 1:
2968         emit_int8((unsigned char)0x90);
2969                          // nop
2970         break;
2971       default:
2972         assert(i == 0, " ");
2973     }
2974     return;
2975   }
2976   if (UseAddressNop && VM_Version::is_amd()) {
2977     //
2978     // Using multi-bytes nops "0x0F 0x1F [address]" for AMD.
2979     //  1: 0x90
2980     //  2: 0x66 0x90
2981     //  3: 0x66 0x66 0x90 (don't use "0x0F 0x1F 0x00" - need patching safe padding)
2982     //  4: 0x0F 0x1F 0x40 0x00
2983     //  5: 0x0F 0x1F 0x44 0x00 0x00
2984     //  6: 0x66 0x0F 0x1F 0x44 0x00 0x00
2985     //  7: 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00
2986     //  8: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
2987     //  9: 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
2988     // 10: 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
2989     // 11: 0x66 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
2990 
2991     // The rest coding is AMD specific - use consecutive address nops
2992 
2993     // 12: 0x66 0x0F 0x1F 0x44 0x00 0x00 0x66 0x0F 0x1F 0x44 0x00 0x00
2994     // 13: 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00 0x66 0x0F 0x1F 0x44 0x00 0x00
2995     // 14: 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00
2996     // 15: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00
2997     // 16: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
2998     //     Size prefixes (0x66) are added for larger sizes
2999 
3000     while(i >= 22) {
3001       i -= 11;
3002       emit_int8(0x66); // size prefix
3003       emit_int8(0x66); // size prefix
3004       emit_int8(0x66); // size prefix
3005       addr_nop_8();
3006     }
3007     // Generate first nop for size between 21-12
3008     switch (i) {
3009       case 21:
3010         i -= 1;
3011         emit_int8(0x66); // size prefix
3012       case 20:
3013       case 19:
3014         i -= 1;
3015         emit_int8(0x66); // size prefix
3016       case 18:
3017       case 17:
3018         i -= 1;
3019         emit_int8(0x66); // size prefix
3020       case 16:
3021       case 15:
3022         i -= 8;
3023         addr_nop_8();
3024         break;
3025       case 14:
3026       case 13:
3027         i -= 7;
3028         addr_nop_7();
3029         break;
3030       case 12:
3031         i -= 6;
3032         emit_int8(0x66); // size prefix
3033         addr_nop_5();
3034         break;
3035       default:
3036         assert(i < 12, " ");
3037     }
3038 
3039     // Generate second nop for size between 11-1
3040     switch (i) {
3041       case 11:
3042         emit_int8(0x66); // size prefix
3043       case 10:
3044         emit_int8(0x66); // size prefix
3045       case 9:
3046         emit_int8(0x66); // size prefix
3047       case 8:
3048         addr_nop_8();
3049         break;
3050       case 7:
3051         addr_nop_7();
3052         break;
3053       case 6:
3054         emit_int8(0x66); // size prefix
3055       case 5:
3056         addr_nop_5();
3057         break;
3058       case 4:
3059         addr_nop_4();
3060         break;
3061       case 3:
3062         // Don't use "0x0F 0x1F 0x00" - need patching safe padding
3063         emit_int8(0x66); // size prefix
3064       case 2:
3065         emit_int8(0x66); // size prefix
3066       case 1:
3067         emit_int8((unsigned char)0x90);
3068                          // nop
3069         break;
3070       default:
3071         assert(i == 0, " ");
3072     }
3073     return;
3074   }
3075 
3076   // Using nops with size prefixes "0x66 0x90".
3077   // From AMD Optimization Guide:
3078   //  1: 0x90
3079   //  2: 0x66 0x90
3080   //  3: 0x66 0x66 0x90
3081   //  4: 0x66 0x66 0x66 0x90
3082   //  5: 0x66 0x66 0x90 0x66 0x90
3083   //  6: 0x66 0x66 0x90 0x66 0x66 0x90
3084   //  7: 0x66 0x66 0x66 0x90 0x66 0x66 0x90
3085   //  8: 0x66 0x66 0x66 0x90 0x66 0x66 0x66 0x90
3086   //  9: 0x66 0x66 0x90 0x66 0x66 0x90 0x66 0x66 0x90
3087   // 10: 0x66 0x66 0x66 0x90 0x66 0x66 0x90 0x66 0x66 0x90
3088   //
3089   while(i > 12) {
3090     i -= 4;
3091     emit_int8(0x66); // size prefix
3092     emit_int8(0x66);
3093     emit_int8(0x66);
3094     emit_int8((unsigned char)0x90);
3095                      // nop
3096   }
3097   // 1 - 12 nops
3098   if(i > 8) {
3099     if(i > 9) {
3100       i -= 1;
3101       emit_int8(0x66);
3102     }
3103     i -= 3;
3104     emit_int8(0x66);
3105     emit_int8(0x66);
3106     emit_int8((unsigned char)0x90);
3107   }
3108   // 1 - 8 nops
3109   if(i > 4) {
3110     if(i > 6) {
3111       i -= 1;
3112       emit_int8(0x66);
3113     }
3114     i -= 3;
3115     emit_int8(0x66);
3116     emit_int8(0x66);
3117     emit_int8((unsigned char)0x90);
3118   }
3119   switch (i) {
3120     case 4:
3121       emit_int8(0x66);
3122     case 3:
3123       emit_int8(0x66);
3124     case 2:
3125       emit_int8(0x66);
3126     case 1:
3127       emit_int8((unsigned char)0x90);
3128       break;
3129     default:
3130       assert(i == 0, " ");
3131   }
3132 }
3133 
3134 void Assembler::notl(Register dst) {
3135   int encode = prefix_and_encode(dst->encoding());
3136   emit_int8((unsigned char)0xF7);
3137   emit_int8((unsigned char)(0xD0 | encode));
3138 }
3139 
3140 void Assembler::orl(Address dst, int32_t imm32) {
3141   InstructionMark im(this);
3142   prefix(dst);
3143   emit_arith_operand(0x81, rcx, dst, imm32);
3144 }
3145 
3146 void Assembler::orl(Register dst, int32_t imm32) {
3147   prefix(dst);
3148   emit_arith(0x81, 0xC8, dst, imm32);
3149 }
3150 
3151 void Assembler::orl(Register dst, Address src) {
3152   InstructionMark im(this);
3153   prefix(src, dst);
3154   emit_int8(0x0B);
3155   emit_operand(dst, src);
3156 }
3157 
3158 void Assembler::orl(Register dst, Register src) {
3159   (void) prefix_and_encode(dst->encoding(), src->encoding());
3160   emit_arith(0x0B, 0xC0, dst, src);
3161 }
3162 
3163 void Assembler::orl(Address dst, Register src) {
3164   InstructionMark im(this);
3165   prefix(dst, src);
3166   emit_int8(0x09);
3167   emit_operand(src, dst);
3168 }
3169 
3170 void Assembler::packuswb(XMMRegister dst, Address src) {
3171   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3172   assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
3173   InstructionMark im(this);
3174   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
3175   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
3176   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3177   emit_int8(0x67);
3178   emit_operand(dst, src);
3179 }
3180 
3181 void Assembler::packuswb(XMMRegister dst, XMMRegister src) {
3182   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3183   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
3184   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3185   emit_int8(0x67);
3186   emit_int8((unsigned char)(0xC0 | encode));
3187 }
3188 
3189 void Assembler::vpackuswb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
3190   assert(UseAVX > 0, "some form of AVX must be enabled");
3191   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
3192   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3193   emit_int8(0x67);
3194   emit_int8((unsigned char)(0xC0 | encode));
3195 }
3196 
3197 void Assembler::vpermq(XMMRegister dst, XMMRegister src, int imm8, int vector_len) {
3198   assert(VM_Version::supports_avx2(), "");
3199   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
3200   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3201   emit_int8(0x00);
3202   emit_int8(0xC0 | encode);
3203   emit_int8(imm8);
3204 }
3205 
3206 void Assembler::vperm2i128(XMMRegister dst,  XMMRegister nds, XMMRegister src, int imm8) {
3207   assert(VM_Version::supports_avx2(), "");
3208   InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3209   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3210   emit_int8(0x46);
3211   emit_int8(0xC0 | encode);
3212   emit_int8(imm8);
3213 }
3214 
3215 
3216 void Assembler::pause() {
3217   emit_int8((unsigned char)0xF3);
3218   emit_int8((unsigned char)0x90);
3219 }
3220 
3221 void Assembler::pcmpestri(XMMRegister dst, Address src, int imm8) {
3222   assert(VM_Version::supports_sse4_2(), "");
3223   InstructionMark im(this);
3224   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3225   simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3226   emit_int8(0x61);
3227   emit_operand(dst, src);
3228   emit_int8(imm8);
3229 }
3230 
3231 void Assembler::pcmpestri(XMMRegister dst, XMMRegister src, int imm8) {
3232   assert(VM_Version::supports_sse4_2(), "");
3233   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3234   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3235   emit_int8(0x61);
3236   emit_int8((unsigned char)(0xC0 | encode));
3237   emit_int8(imm8);
3238 }
3239 
3240 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
3241 void Assembler::pcmpeqb(XMMRegister dst, XMMRegister src) {
3242   assert(VM_Version::supports_sse2(), "");
3243   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3244   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3245   emit_int8(0x74);
3246   emit_int8((unsigned char)(0xC0 | encode));
3247 }
3248 
3249 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
3250 void Assembler::vpcmpeqb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
3251   assert(VM_Version::supports_avx(), "");
3252   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3253   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3254   emit_int8(0x74);
3255   emit_int8((unsigned char)(0xC0 | encode));
3256 }
3257 
3258 // In this context, kdst is written the mask used to process the equal components
3259 void Assembler::evpcmpeqb(KRegister kdst, XMMRegister nds, XMMRegister src, int vector_len) {
3260   assert(VM_Version::supports_avx512bw(), "");
3261   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
3262   attributes.set_is_evex_instruction();
3263   int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3264   emit_int8(0x74);
3265   emit_int8((unsigned char)(0xC0 | encode));
3266 }
3267 
3268 void Assembler::evpcmpeqb(KRegister kdst, XMMRegister nds, Address src, int vector_len) {
3269   assert(VM_Version::supports_avx512bw(), "");
3270   InstructionMark im(this);
3271   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
3272   attributes.set_is_evex_instruction();
3273   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
3274   int dst_enc = kdst->encoding();
3275   vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3276   emit_int8(0x74);
3277   emit_operand(as_Register(dst_enc), src);
3278 }
3279 
3280 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
3281 void Assembler::pcmpeqw(XMMRegister dst, XMMRegister src) {
3282   assert(VM_Version::supports_sse2(), "");
3283   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3284   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3285   emit_int8(0x75);
3286   emit_int8((unsigned char)(0xC0 | encode));
3287 }
3288 
3289 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
3290 void Assembler::vpcmpeqw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
3291   assert(VM_Version::supports_avx(), "");
3292   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3293   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3294   emit_int8(0x75);
3295   emit_int8((unsigned char)(0xC0 | encode));
3296 }
3297 
3298 // In this context, kdst is written the mask used to process the equal components
3299 void Assembler::evpcmpeqw(KRegister kdst, XMMRegister nds, XMMRegister src, int vector_len) {
3300   assert(VM_Version::supports_avx512bw(), "");
3301   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
3302   attributes.set_is_evex_instruction();
3303   int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3304   emit_int8(0x75);
3305   emit_int8((unsigned char)(0xC0 | encode));
3306 }
3307 
3308 void Assembler::evpcmpeqw(KRegister kdst, XMMRegister nds, Address src, int vector_len) {
3309   assert(VM_Version::supports_avx512bw(), "");
3310   InstructionMark im(this);
3311   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
3312   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
3313   attributes.set_is_evex_instruction();
3314   int dst_enc = kdst->encoding();
3315   vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3316   emit_int8(0x75);
3317   emit_operand(as_Register(dst_enc), src);
3318 }
3319 
3320 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
3321 void Assembler::pcmpeqd(XMMRegister dst, XMMRegister src) {
3322   assert(VM_Version::supports_sse2(), "");
3323   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3324   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3325   emit_int8(0x76);
3326   emit_int8((unsigned char)(0xC0 | encode));
3327 }
3328 
3329 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
3330 void Assembler::vpcmpeqd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
3331   assert(VM_Version::supports_avx(), "");
3332   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3333   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3334   emit_int8(0x76);
3335   emit_int8((unsigned char)(0xC0 | encode));
3336 }
3337 
3338 // In this context, kdst is written the mask used to process the equal components
3339 void Assembler::evpcmpeqd(KRegister kdst, XMMRegister nds, XMMRegister src, int vector_len) {
3340   assert(VM_Version::supports_evex(), "");
3341   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
3342   attributes.set_is_evex_instruction();
3343   int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3344   emit_int8(0x76);
3345   emit_int8((unsigned char)(0xC0 | encode));
3346 }
3347 
3348 void Assembler::evpcmpeqd(KRegister kdst, XMMRegister nds, Address src, int vector_len) {
3349   assert(VM_Version::supports_evex(), "");
3350   InstructionMark im(this);
3351   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
3352   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
3353   attributes.set_is_evex_instruction();
3354   int dst_enc = kdst->encoding();
3355   vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3356   emit_int8(0x76);
3357   emit_operand(as_Register(dst_enc), src);
3358 }
3359 
3360 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
3361 void Assembler::pcmpeqq(XMMRegister dst, XMMRegister src) {
3362   assert(VM_Version::supports_sse4_1(), "");
3363   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3364   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3365   emit_int8(0x29);
3366   emit_int8((unsigned char)(0xC0 | encode));
3367 }
3368 
3369 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
3370 void Assembler::vpcmpeqq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
3371   assert(VM_Version::supports_avx(), "");
3372   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3373   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3374   emit_int8(0x29);
3375   emit_int8((unsigned char)(0xC0 | encode));
3376 }
3377 
3378 // In this context, kdst is written the mask used to process the equal components
3379 void Assembler::evpcmpeqq(KRegister kdst, XMMRegister nds, XMMRegister src, int vector_len) {
3380   assert(VM_Version::supports_evex(), "");
3381   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
3382   attributes.set_is_evex_instruction();
3383   int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3384   emit_int8(0x29);
3385   emit_int8((unsigned char)(0xC0 | encode));
3386 }
3387 
3388 // In this context, kdst is written the mask used to process the equal components
3389 void Assembler::evpcmpeqq(KRegister kdst, XMMRegister nds, Address src, int vector_len) {
3390   assert(VM_Version::supports_evex(), "");
3391   InstructionMark im(this);
3392   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
3393   attributes.set_is_evex_instruction();
3394   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
3395   int dst_enc = kdst->encoding();
3396   vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3397   emit_int8(0x29);
3398   emit_operand(as_Register(dst_enc), src);
3399 }
3400 
3401 void Assembler::pmovmskb(Register dst, XMMRegister src) {
3402   assert(VM_Version::supports_sse2(), "");
3403   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3404   int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3405   emit_int8((unsigned char)0xD7);
3406   emit_int8((unsigned char)(0xC0 | encode));
3407 }
3408 
3409 void Assembler::vpmovmskb(Register dst, XMMRegister src) {
3410   assert(VM_Version::supports_avx2(), "");
3411   InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3412   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3413   emit_int8((unsigned char)0xD7);
3414   emit_int8((unsigned char)(0xC0 | encode));
3415 }
3416 
3417 void Assembler::pextrd(Register dst, XMMRegister src, int imm8) {
3418   assert(VM_Version::supports_sse4_1(), "");
3419   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
3420   int encode = simd_prefix_and_encode(src, xnoreg, as_XMMRegister(dst->encoding()), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3421   emit_int8(0x16);
3422   emit_int8((unsigned char)(0xC0 | encode));
3423   emit_int8(imm8);
3424 }
3425 
3426 void Assembler::pextrd(Address dst, XMMRegister src, int imm8) {
3427   assert(VM_Version::supports_sse4_1(), "");
3428   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
3429   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
3430   simd_prefix(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3431   emit_int8(0x16);
3432   emit_operand(src, dst);
3433   emit_int8(imm8);
3434 }
3435 
3436 void Assembler::pextrq(Register dst, XMMRegister src, int imm8) {
3437   assert(VM_Version::supports_sse4_1(), "");
3438   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
3439   int encode = simd_prefix_and_encode(src, xnoreg, as_XMMRegister(dst->encoding()), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3440   emit_int8(0x16);
3441   emit_int8((unsigned char)(0xC0 | encode));
3442   emit_int8(imm8);
3443 }
3444 
3445 void Assembler::pextrq(Address dst, XMMRegister src, int imm8) {
3446   assert(VM_Version::supports_sse4_1(), "");
3447   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
3448   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
3449   simd_prefix(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3450   emit_int8(0x16);
3451   emit_operand(src, dst);
3452   emit_int8(imm8);
3453 }
3454 
3455 void Assembler::pextrw(Register dst, XMMRegister src, int imm8) {
3456   assert(VM_Version::supports_sse2(), "");
3457   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
3458   int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3459   emit_int8((unsigned char)0xC5);
3460   emit_int8((unsigned char)(0xC0 | encode));
3461   emit_int8(imm8);
3462 }
3463 
3464 void Assembler::pextrw(Address dst, XMMRegister src, int imm8) {
3465   assert(VM_Version::supports_sse4_1(), "");
3466   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
3467   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_16bit);
3468   simd_prefix(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3469   emit_int8((unsigned char)0x15);
3470   emit_operand(src, dst);
3471   emit_int8(imm8);
3472 }
3473 
3474 void Assembler::pextrb(Address dst, XMMRegister src, int imm8) {
3475   assert(VM_Version::supports_sse4_1(), "");
3476   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
3477   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_8bit);
3478   simd_prefix(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3479   emit_int8(0x14);
3480   emit_operand(src, dst);
3481   emit_int8(imm8);
3482 }
3483 
3484 void Assembler::pinsrd(XMMRegister dst, Register src, int imm8) {
3485   assert(VM_Version::supports_sse4_1(), "");
3486   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
3487   int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3488   emit_int8(0x22);
3489   emit_int8((unsigned char)(0xC0 | encode));
3490   emit_int8(imm8);
3491 }
3492 
3493 void Assembler::pinsrd(XMMRegister dst, Address src, int imm8) {
3494   assert(VM_Version::supports_sse4_1(), "");
3495   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
3496   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
3497   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3498   emit_int8(0x22);
3499   emit_operand(dst,src);
3500   emit_int8(imm8);
3501 }
3502 
3503 void Assembler::pinsrq(XMMRegister dst, Register src, int imm8) {
3504   assert(VM_Version::supports_sse4_1(), "");
3505   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
3506   int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3507   emit_int8(0x22);
3508   emit_int8((unsigned char)(0xC0 | encode));
3509   emit_int8(imm8);
3510 }
3511 
3512 void Assembler::pinsrq(XMMRegister dst, Address src, int imm8) {
3513   assert(VM_Version::supports_sse4_1(), "");
3514   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
3515   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
3516   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3517   emit_int8(0x22);
3518   emit_operand(dst, src);
3519   emit_int8(imm8);
3520 }
3521 
3522 void Assembler::pinsrw(XMMRegister dst, Register src, int imm8) {
3523   assert(VM_Version::supports_sse2(), "");
3524   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
3525   int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3526   emit_int8((unsigned char)0xC4);
3527   emit_int8((unsigned char)(0xC0 | encode));
3528   emit_int8(imm8);
3529 }
3530 
3531 void Assembler::pinsrw(XMMRegister dst, Address src, int imm8) {
3532   assert(VM_Version::supports_sse2(), "");
3533   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
3534   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_16bit);
3535   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3536   emit_int8((unsigned char)0xC4);
3537   emit_operand(dst, src);
3538   emit_int8(imm8);
3539 }
3540 
3541 void Assembler::pinsrb(XMMRegister dst, Address src, int imm8) {
3542   assert(VM_Version::supports_sse4_1(), "");
3543   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
3544   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_8bit);
3545   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3546   emit_int8(0x20);
3547   emit_operand(dst, src);
3548   emit_int8(imm8);
3549 }
3550 
3551 void Assembler::pmovzxbw(XMMRegister dst, Address src) {
3552   assert(VM_Version::supports_sse4_1(), "");
3553   InstructionMark im(this);
3554   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
3555   attributes.set_address_attributes(/* tuple_type */ EVEX_HVM, /* input_size_in_bits */ EVEX_NObit);
3556   simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3557   emit_int8(0x30);
3558   emit_operand(dst, src);
3559 }
3560 
3561 void Assembler::pmovzxbw(XMMRegister dst, XMMRegister src) {
3562   assert(VM_Version::supports_sse4_1(), "");
3563   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
3564   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3565   emit_int8(0x30);
3566   emit_int8((unsigned char)(0xC0 | encode));
3567 }
3568 
3569 void Assembler::vpmovzxbw(XMMRegister dst, Address src, int vector_len) {
3570   assert(VM_Version::supports_avx(), "");
3571   InstructionMark im(this);
3572   assert(dst != xnoreg, "sanity");
3573   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
3574   attributes.set_address_attributes(/* tuple_type */ EVEX_HVM, /* input_size_in_bits */ EVEX_NObit);
3575   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3576   emit_int8(0x30);
3577   emit_operand(dst, src);
3578 }
3579 
3580 // generic
3581 void Assembler::pop(Register dst) {
3582   int encode = prefix_and_encode(dst->encoding());
3583   emit_int8(0x58 | encode);
3584 }
3585 
3586 void Assembler::popcntl(Register dst, Address src) {
3587   assert(VM_Version::supports_popcnt(), "must support");
3588   InstructionMark im(this);
3589   emit_int8((unsigned char)0xF3);
3590   prefix(src, dst);
3591   emit_int8(0x0F);
3592   emit_int8((unsigned char)0xB8);
3593   emit_operand(dst, src);
3594 }
3595 
3596 void Assembler::popcntl(Register dst, Register src) {
3597   assert(VM_Version::supports_popcnt(), "must support");
3598   emit_int8((unsigned char)0xF3);
3599   int encode = prefix_and_encode(dst->encoding(), src->encoding());
3600   emit_int8(0x0F);
3601   emit_int8((unsigned char)0xB8);
3602   emit_int8((unsigned char)(0xC0 | encode));
3603 }
3604 
3605 void Assembler::popf() {
3606   emit_int8((unsigned char)0x9D);
3607 }
3608 
3609 #ifndef _LP64 // no 32bit push/pop on amd64
3610 void Assembler::popl(Address dst) {
3611   // NOTE: this will adjust stack by 8byte on 64bits
3612   InstructionMark im(this);
3613   prefix(dst);
3614   emit_int8((unsigned char)0x8F);
3615   emit_operand(rax, dst);
3616 }
3617 #endif
3618 
3619 void Assembler::prefetch_prefix(Address src) {
3620   prefix(src);
3621   emit_int8(0x0F);
3622 }
3623 
3624 void Assembler::prefetchnta(Address src) {
3625   NOT_LP64(assert(VM_Version::supports_sse(), "must support"));
3626   InstructionMark im(this);
3627   prefetch_prefix(src);
3628   emit_int8(0x18);
3629   emit_operand(rax, src); // 0, src
3630 }
3631 
3632 void Assembler::prefetchr(Address src) {
3633   assert(VM_Version::supports_3dnow_prefetch(), "must support");
3634   InstructionMark im(this);
3635   prefetch_prefix(src);
3636   emit_int8(0x0D);
3637   emit_operand(rax, src); // 0, src
3638 }
3639 
3640 void Assembler::prefetcht0(Address src) {
3641   NOT_LP64(assert(VM_Version::supports_sse(), "must support"));
3642   InstructionMark im(this);
3643   prefetch_prefix(src);
3644   emit_int8(0x18);
3645   emit_operand(rcx, src); // 1, src
3646 }
3647 
3648 void Assembler::prefetcht1(Address src) {
3649   NOT_LP64(assert(VM_Version::supports_sse(), "must support"));
3650   InstructionMark im(this);
3651   prefetch_prefix(src);
3652   emit_int8(0x18);
3653   emit_operand(rdx, src); // 2, src
3654 }
3655 
3656 void Assembler::prefetcht2(Address src) {
3657   NOT_LP64(assert(VM_Version::supports_sse(), "must support"));
3658   InstructionMark im(this);
3659   prefetch_prefix(src);
3660   emit_int8(0x18);
3661   emit_operand(rbx, src); // 3, src
3662 }
3663 
3664 void Assembler::prefetchw(Address src) {
3665   assert(VM_Version::supports_3dnow_prefetch(), "must support");
3666   InstructionMark im(this);
3667   prefetch_prefix(src);
3668   emit_int8(0x0D);
3669   emit_operand(rcx, src); // 1, src
3670 }
3671 
3672 void Assembler::prefix(Prefix p) {
3673   emit_int8(p);
3674 }
3675 
3676 void Assembler::pshufb(XMMRegister dst, XMMRegister src) {
3677   assert(VM_Version::supports_ssse3(), "");
3678   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
3679   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3680   emit_int8(0x00);
3681   emit_int8((unsigned char)(0xC0 | encode));
3682 }
3683 
3684 void Assembler::vpshufb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
3685   assert(VM_Version::supports_ssse3(), "");
3686   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
3687   int encode = simd_prefix_and_encode(dst, nds, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3688   emit_int8(0x00);
3689   emit_int8((unsigned char)(0xC0 | encode));
3690 }
3691 
3692 void Assembler::pshufb(XMMRegister dst, Address src) {
3693   assert(VM_Version::supports_ssse3(), "");
3694   InstructionMark im(this);
3695   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
3696   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
3697   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3698   emit_int8(0x00);
3699   emit_operand(dst, src);
3700 }
3701 
3702 void Assembler::pshufd(XMMRegister dst, XMMRegister src, int mode) {
3703   assert(isByte(mode), "invalid value");
3704   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3705   int vector_len = VM_Version::supports_evex() ? AVX_512bit : AVX_128bit;
3706   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
3707   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3708   emit_int8(0x70);
3709   emit_int8((unsigned char)(0xC0 | encode));
3710   emit_int8(mode & 0xFF);
3711 }
3712 
3713 void Assembler::vpshufd(XMMRegister dst, XMMRegister src, int mode, int vector_len) {
3714   assert(isByte(mode), "invalid value");
3715   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3716   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
3717   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3718   emit_int8(0x70);
3719   emit_int8((unsigned char)(0xC0 | encode));
3720   emit_int8(mode & 0xFF);
3721 }
3722 
3723 void Assembler::pshufd(XMMRegister dst, Address src, int mode) {
3724   assert(isByte(mode), "invalid value");
3725   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3726   assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
3727   InstructionMark im(this);
3728   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
3729   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
3730   simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3731   emit_int8(0x70);
3732   emit_operand(dst, src);
3733   emit_int8(mode & 0xFF);
3734 }
3735 
3736 void Assembler::pshuflw(XMMRegister dst, XMMRegister src, int mode) {
3737   assert(isByte(mode), "invalid value");
3738   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3739   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
3740   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
3741   emit_int8(0x70);
3742   emit_int8((unsigned char)(0xC0 | encode));
3743   emit_int8(mode & 0xFF);
3744 }
3745 
3746 void Assembler::pshuflw(XMMRegister dst, Address src, int mode) {
3747   assert(isByte(mode), "invalid value");
3748   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3749   assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
3750   InstructionMark im(this);
3751   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
3752   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
3753   simd_prefix(dst, xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
3754   emit_int8(0x70);
3755   emit_operand(dst, src);
3756   emit_int8(mode & 0xFF);
3757 }
3758 
3759 void Assembler::psrldq(XMMRegister dst, int shift) {
3760   // Shift left 128 bit value in dst XMMRegister by shift number of bytes.
3761   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3762   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
3763   int encode = simd_prefix_and_encode(xmm3, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3764   emit_int8(0x73);
3765   emit_int8((unsigned char)(0xC0 | encode));
3766   emit_int8(shift);
3767 }
3768 
3769 void Assembler::vpsrldq(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
3770   // Shift left 128 bit value in dst XMMRegister by shift number of bytes.
3771   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3772   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
3773   // XMM3 is for /3 encoding: 66 0F 73 /3 ib
3774   int encode = simd_prefix_and_encode(xmm3, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3775   emit_int8(0x73);
3776   emit_int8((unsigned char)(0xC0 | encode));
3777   emit_int8(shift);
3778 }
3779 
3780 void Assembler::pslldq(XMMRegister dst, int shift) {
3781   // Shift left 128 bit value in dst XMMRegister by shift number of bytes.
3782   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3783   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
3784   // XMM7 is for /7 encoding: 66 0F 73 /7 ib
3785   int encode = simd_prefix_and_encode(xmm7, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3786   emit_int8(0x73);
3787   emit_int8((unsigned char)(0xC0 | encode));
3788   emit_int8(shift);
3789 }
3790 
3791 void Assembler::vpslldq(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
3792   // Shift left 128 bit value in dst XMMRegister by shift number of bytes.
3793   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3794   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
3795   // XMM7 is for /7 encoding: 66 0F 73 /7 ib
3796   int encode = simd_prefix_and_encode(xmm7, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3797   emit_int8(0x73);
3798   emit_int8((unsigned char)(0xC0 | encode));
3799   emit_int8(shift);
3800 }
3801 
3802 void Assembler::ptest(XMMRegister dst, Address src) {
3803   assert(VM_Version::supports_sse4_1(), "");
3804   assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
3805   InstructionMark im(this);
3806   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3807   simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3808   emit_int8(0x17);
3809   emit_operand(dst, src);
3810 }
3811 
3812 void Assembler::ptest(XMMRegister dst, XMMRegister src) {
3813   assert(VM_Version::supports_sse4_1(), "");
3814   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3815   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3816   emit_int8(0x17);
3817   emit_int8((unsigned char)(0xC0 | encode));
3818 }
3819 
3820 void Assembler::vptest(XMMRegister dst, Address src) {
3821   assert(VM_Version::supports_avx(), "");
3822   InstructionMark im(this);
3823   InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3824   assert(dst != xnoreg, "sanity");
3825   // swap src<->dst for encoding
3826   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3827   emit_int8(0x17);
3828   emit_operand(dst, src);
3829 }
3830 
3831 void Assembler::vptest(XMMRegister dst, XMMRegister src) {
3832   assert(VM_Version::supports_avx(), "");
3833   InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3834   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3835   emit_int8(0x17);
3836   emit_int8((unsigned char)(0xC0 | encode));
3837 }
3838 
3839 void Assembler::punpcklbw(XMMRegister dst, Address src) {
3840   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3841   assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
3842   InstructionMark im(this);
3843   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_vlbw, /* no_mask_reg */ false, /* uses_vl */ true);
3844   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
3845   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3846   emit_int8(0x60);
3847   emit_operand(dst, src);
3848 }
3849 
3850 void Assembler::punpcklbw(XMMRegister dst, XMMRegister src) {
3851   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3852   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_vlbw, /* no_mask_reg */ false, /* uses_vl */ true);
3853   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3854   emit_int8(0x60);
3855   emit_int8((unsigned char)(0xC0 | encode));
3856 }
3857 
3858 void Assembler::punpckldq(XMMRegister dst, Address src) {
3859   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3860   assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
3861   InstructionMark im(this);
3862   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
3863   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
3864   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3865   emit_int8(0x62);
3866   emit_operand(dst, src);
3867 }
3868 
3869 void Assembler::punpckldq(XMMRegister dst, XMMRegister src) {
3870   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3871   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
3872   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3873   emit_int8(0x62);
3874   emit_int8((unsigned char)(0xC0 | encode));
3875 }
3876 
3877 void Assembler::punpcklqdq(XMMRegister dst, XMMRegister src) {
3878   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3879   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
3880   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3881   emit_int8(0x6C);
3882   emit_int8((unsigned char)(0xC0 | encode));
3883 }
3884 
3885 void Assembler::push(int32_t imm32) {
3886   // in 64bits we push 64bits onto the stack but only
3887   // take a 32bit immediate
3888   emit_int8(0x68);
3889   emit_int32(imm32);
3890 }
3891 
3892 void Assembler::push(Register src) {
3893   int encode = prefix_and_encode(src->encoding());
3894 
3895   emit_int8(0x50 | encode);
3896 }
3897 
3898 void Assembler::pushf() {
3899   emit_int8((unsigned char)0x9C);
3900 }
3901 
3902 #ifndef _LP64 // no 32bit push/pop on amd64
3903 void Assembler::pushl(Address src) {
3904   // Note this will push 64bit on 64bit
3905   InstructionMark im(this);
3906   prefix(src);
3907   emit_int8((unsigned char)0xFF);
3908   emit_operand(rsi, src);
3909 }
3910 #endif
3911 
3912 void Assembler::rcll(Register dst, int imm8) {
3913   assert(isShiftCount(imm8), "illegal shift count");
3914   int encode = prefix_and_encode(dst->encoding());
3915   if (imm8 == 1) {
3916     emit_int8((unsigned char)0xD1);
3917     emit_int8((unsigned char)(0xD0 | encode));
3918   } else {
3919     emit_int8((unsigned char)0xC1);
3920     emit_int8((unsigned char)0xD0 | encode);
3921     emit_int8(imm8);
3922   }
3923 }
3924 
3925 void Assembler::rcpps(XMMRegister dst, XMMRegister src) {
3926   NOT_LP64(assert(VM_Version::supports_sse(), ""));
3927   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3928   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
3929   emit_int8(0x53);
3930   emit_int8((unsigned char)(0xC0 | encode));
3931 }
3932 
3933 void Assembler::rcpss(XMMRegister dst, XMMRegister src) {
3934   NOT_LP64(assert(VM_Version::supports_sse(), ""));
3935   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3936   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
3937   emit_int8(0x53);
3938   emit_int8((unsigned char)(0xC0 | encode));
3939 }
3940 
3941 void Assembler::rdtsc() {
3942   emit_int8((unsigned char)0x0F);
3943   emit_int8((unsigned char)0x31);
3944 }
3945 
3946 // copies data from [esi] to [edi] using rcx pointer sized words
3947 // generic
3948 void Assembler::rep_mov() {
3949   emit_int8((unsigned char)0xF3);
3950   // MOVSQ
3951   LP64_ONLY(prefix(REX_W));
3952   emit_int8((unsigned char)0xA5);
3953 }
3954 
3955 // sets rcx bytes with rax, value at [edi]
3956 void Assembler::rep_stosb() {
3957   emit_int8((unsigned char)0xF3); // REP
3958   LP64_ONLY(prefix(REX_W));
3959   emit_int8((unsigned char)0xAA); // STOSB
3960 }
3961 
3962 // sets rcx pointer sized words with rax, value at [edi]
3963 // generic
3964 void Assembler::rep_stos() {
3965   emit_int8((unsigned char)0xF3); // REP
3966   LP64_ONLY(prefix(REX_W));       // LP64:STOSQ, LP32:STOSD
3967   emit_int8((unsigned char)0xAB);
3968 }
3969 
3970 // scans rcx pointer sized words at [edi] for occurance of rax,
3971 // generic
3972 void Assembler::repne_scan() { // repne_scan
3973   emit_int8((unsigned char)0xF2);
3974   // SCASQ
3975   LP64_ONLY(prefix(REX_W));
3976   emit_int8((unsigned char)0xAF);
3977 }
3978 
3979 #ifdef _LP64
3980 // scans rcx 4 byte words at [edi] for occurance of rax,
3981 // generic
3982 void Assembler::repne_scanl() { // repne_scan
3983   emit_int8((unsigned char)0xF2);
3984   // SCASL
3985   emit_int8((unsigned char)0xAF);
3986 }
3987 #endif
3988 
3989 void Assembler::ret(int imm16) {
3990   if (imm16 == 0) {
3991     emit_int8((unsigned char)0xC3);
3992   } else {
3993     emit_int8((unsigned char)0xC2);
3994     emit_int16(imm16);
3995   }
3996 }
3997 
3998 void Assembler::sahf() {
3999 #ifdef _LP64
4000   // Not supported in 64bit mode
4001   ShouldNotReachHere();
4002 #endif
4003   emit_int8((unsigned char)0x9E);
4004 }
4005 
4006 void Assembler::sarl(Register dst, int imm8) {
4007   int encode = prefix_and_encode(dst->encoding());
4008   assert(isShiftCount(imm8), "illegal shift count");
4009   if (imm8 == 1) {
4010     emit_int8((unsigned char)0xD1);
4011     emit_int8((unsigned char)(0xF8 | encode));
4012   } else {
4013     emit_int8((unsigned char)0xC1);
4014     emit_int8((unsigned char)(0xF8 | encode));
4015     emit_int8(imm8);
4016   }
4017 }
4018 
4019 void Assembler::sarl(Register dst) {
4020   int encode = prefix_and_encode(dst->encoding());
4021   emit_int8((unsigned char)0xD3);
4022   emit_int8((unsigned char)(0xF8 | encode));
4023 }
4024 
4025 void Assembler::sbbl(Address dst, int32_t imm32) {
4026   InstructionMark im(this);
4027   prefix(dst);
4028   emit_arith_operand(0x81, rbx, dst, imm32);
4029 }
4030 
4031 void Assembler::sbbl(Register dst, int32_t imm32) {
4032   prefix(dst);
4033   emit_arith(0x81, 0xD8, dst, imm32);
4034 }
4035 
4036 
4037 void Assembler::sbbl(Register dst, Address src) {
4038   InstructionMark im(this);
4039   prefix(src, dst);
4040   emit_int8(0x1B);
4041   emit_operand(dst, src);
4042 }
4043 
4044 void Assembler::sbbl(Register dst, Register src) {
4045   (void) prefix_and_encode(dst->encoding(), src->encoding());
4046   emit_arith(0x1B, 0xC0, dst, src);
4047 }
4048 
4049 void Assembler::setb(Condition cc, Register dst) {
4050   assert(0 <= cc && cc < 16, "illegal cc");
4051   int encode = prefix_and_encode(dst->encoding(), true);
4052   emit_int8(0x0F);
4053   emit_int8((unsigned char)0x90 | cc);
4054   emit_int8((unsigned char)(0xC0 | encode));
4055 }
4056 
4057 void Assembler::palignr(XMMRegister dst, XMMRegister src, int imm8) {
4058   assert(VM_Version::supports_ssse3(), "");
4059   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ false);
4060   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
4061   emit_int8((unsigned char)0x0F);
4062   emit_int8((unsigned char)(0xC0 | encode));
4063   emit_int8(imm8);
4064 }
4065 
4066 void Assembler::vpalignr(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8, int vector_len) {
4067   assert(VM_Version::supports_ssse3(), "");
4068   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
4069   int encode = simd_prefix_and_encode(dst, nds, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
4070   emit_int8((unsigned char)0x0F);
4071   emit_int8((unsigned char)(0xC0 | encode));
4072   emit_int8(imm8);
4073 }
4074 
4075 void Assembler::pblendw(XMMRegister dst, XMMRegister src, int imm8) {
4076   assert(VM_Version::supports_sse4_1(), "");
4077   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
4078   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
4079   emit_int8((unsigned char)0x0E);
4080   emit_int8((unsigned char)(0xC0 | encode));
4081   emit_int8(imm8);
4082 }
4083 
4084 void Assembler::sha1rnds4(XMMRegister dst, XMMRegister src, int imm8) {
4085   assert(VM_Version::supports_sha(), "");
4086   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
4087   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F_3A, &attributes);
4088   emit_int8((unsigned char)0xCC);
4089   emit_int8((unsigned char)(0xC0 | encode));
4090   emit_int8((unsigned char)imm8);
4091 }
4092 
4093 void Assembler::sha1nexte(XMMRegister dst, XMMRegister src) {
4094   assert(VM_Version::supports_sha(), "");
4095   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
4096   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
4097   emit_int8((unsigned char)0xC8);
4098   emit_int8((unsigned char)(0xC0 | encode));
4099 }
4100 
4101 void Assembler::sha1msg1(XMMRegister dst, XMMRegister src) {
4102   assert(VM_Version::supports_sha(), "");
4103   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
4104   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
4105   emit_int8((unsigned char)0xC9);
4106   emit_int8((unsigned char)(0xC0 | encode));
4107 }
4108 
4109 void Assembler::sha1msg2(XMMRegister dst, XMMRegister src) {
4110   assert(VM_Version::supports_sha(), "");
4111   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
4112   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
4113   emit_int8((unsigned char)0xCA);
4114   emit_int8((unsigned char)(0xC0 | encode));
4115 }
4116 
4117 // xmm0 is implicit additional source to this instruction.
4118 void Assembler::sha256rnds2(XMMRegister dst, XMMRegister src) {
4119   assert(VM_Version::supports_sha(), "");
4120   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
4121   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
4122   emit_int8((unsigned char)0xCB);
4123   emit_int8((unsigned char)(0xC0 | encode));
4124 }
4125 
4126 void Assembler::sha256msg1(XMMRegister dst, XMMRegister src) {
4127   assert(VM_Version::supports_sha(), "");
4128   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
4129   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
4130   emit_int8((unsigned char)0xCC);
4131   emit_int8((unsigned char)(0xC0 | encode));
4132 }
4133 
4134 void Assembler::sha256msg2(XMMRegister dst, XMMRegister src) {
4135   assert(VM_Version::supports_sha(), "");
4136   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
4137   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
4138   emit_int8((unsigned char)0xCD);
4139   emit_int8((unsigned char)(0xC0 | encode));
4140 }
4141 
4142 
4143 void Assembler::shll(Register dst, int imm8) {
4144   assert(isShiftCount(imm8), "illegal shift count");
4145   int encode = prefix_and_encode(dst->encoding());
4146   if (imm8 == 1 ) {
4147     emit_int8((unsigned char)0xD1);
4148     emit_int8((unsigned char)(0xE0 | encode));
4149   } else {
4150     emit_int8((unsigned char)0xC1);
4151     emit_int8((unsigned char)(0xE0 | encode));
4152     emit_int8(imm8);
4153   }
4154 }
4155 
4156 void Assembler::shll(Register dst) {
4157   int encode = prefix_and_encode(dst->encoding());
4158   emit_int8((unsigned char)0xD3);
4159   emit_int8((unsigned char)(0xE0 | encode));
4160 }
4161 
4162 void Assembler::shrl(Register dst, int imm8) {
4163   assert(isShiftCount(imm8), "illegal shift count");
4164   int encode = prefix_and_encode(dst->encoding());
4165   emit_int8((unsigned char)0xC1);
4166   emit_int8((unsigned char)(0xE8 | encode));
4167   emit_int8(imm8);
4168 }
4169 
4170 void Assembler::shrl(Register dst) {
4171   int encode = prefix_and_encode(dst->encoding());
4172   emit_int8((unsigned char)0xD3);
4173   emit_int8((unsigned char)(0xE8 | encode));
4174 }
4175 
4176 // copies a single word from [esi] to [edi]
4177 void Assembler::smovl() {
4178   emit_int8((unsigned char)0xA5);
4179 }
4180 
4181 void Assembler::sqrtsd(XMMRegister dst, XMMRegister src) {
4182   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4183   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
4184   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
4185   emit_int8(0x51);
4186   emit_int8((unsigned char)(0xC0 | encode));
4187 }
4188 
4189 void Assembler::sqrtsd(XMMRegister dst, Address src) {
4190   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4191   InstructionMark im(this);
4192   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
4193   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
4194   simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
4195   emit_int8(0x51);
4196   emit_operand(dst, src);
4197 }
4198 
4199 void Assembler::sqrtss(XMMRegister dst, XMMRegister src) {
4200   NOT_LP64(assert(VM_Version::supports_sse(), ""));
4201   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
4202   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
4203   emit_int8(0x51);
4204   emit_int8((unsigned char)(0xC0 | encode));
4205 }
4206 
4207 void Assembler::std() {
4208   emit_int8((unsigned char)0xFD);
4209 }
4210 
4211 void Assembler::sqrtss(XMMRegister dst, Address src) {
4212   NOT_LP64(assert(VM_Version::supports_sse(), ""));
4213   InstructionMark im(this);
4214   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
4215   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
4216   simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
4217   emit_int8(0x51);
4218   emit_operand(dst, src);
4219 }
4220 
4221 void Assembler::stmxcsr( Address dst) {
4222   NOT_LP64(assert(VM_Version::supports_sse(), ""));
4223   InstructionMark im(this);
4224   prefix(dst);
4225   emit_int8(0x0F);
4226   emit_int8((unsigned char)0xAE);
4227   emit_operand(as_Register(3), dst);
4228 }
4229 
4230 void Assembler::subl(Address dst, int32_t imm32) {
4231   InstructionMark im(this);
4232   prefix(dst);
4233   emit_arith_operand(0x81, rbp, dst, imm32);
4234 }
4235 
4236 void Assembler::subl(Address dst, Register src) {
4237   InstructionMark im(this);
4238   prefix(dst, src);
4239   emit_int8(0x29);
4240   emit_operand(src, dst);
4241 }
4242 
4243 void Assembler::subl(Register dst, int32_t imm32) {
4244   prefix(dst);
4245   emit_arith(0x81, 0xE8, dst, imm32);
4246 }
4247 
4248 // Force generation of a 4 byte immediate value even if it fits into 8bit
4249 void Assembler::subl_imm32(Register dst, int32_t imm32) {
4250   prefix(dst);
4251   emit_arith_imm32(0x81, 0xE8, dst, imm32);
4252 }
4253 
4254 void Assembler::subl(Register dst, Address src) {
4255   InstructionMark im(this);
4256   prefix(src, dst);
4257   emit_int8(0x2B);
4258   emit_operand(dst, src);
4259 }
4260 
4261 void Assembler::subl(Register dst, Register src) {
4262   (void) prefix_and_encode(dst->encoding(), src->encoding());
4263   emit_arith(0x2B, 0xC0, dst, src);
4264 }
4265 
4266 void Assembler::subsd(XMMRegister dst, XMMRegister src) {
4267   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4268   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
4269   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
4270   emit_int8(0x5C);
4271   emit_int8((unsigned char)(0xC0 | encode));
4272 }
4273 
4274 void Assembler::subsd(XMMRegister dst, Address src) {
4275   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4276   InstructionMark im(this);
4277   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
4278   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
4279   simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
4280   emit_int8(0x5C);
4281   emit_operand(dst, src);
4282 }
4283 
4284 void Assembler::subss(XMMRegister dst, XMMRegister src) {
4285   NOT_LP64(assert(VM_Version::supports_sse(), ""));
4286   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false , /* uses_vl */ false);
4287   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
4288   emit_int8(0x5C);
4289   emit_int8((unsigned char)(0xC0 | encode));
4290 }
4291 
4292 void Assembler::subss(XMMRegister dst, Address src) {
4293   NOT_LP64(assert(VM_Version::supports_sse(), ""));
4294   InstructionMark im(this);
4295   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
4296   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
4297   simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
4298   emit_int8(0x5C);
4299   emit_operand(dst, src);
4300 }
4301 
4302 void Assembler::testb(Register dst, int imm8) {
4303   NOT_LP64(assert(dst->has_byte_register(), "must have byte register"));
4304   (void) prefix_and_encode(dst->encoding(), true);
4305   emit_arith_b(0xF6, 0xC0, dst, imm8);
4306 }
4307 
4308 void Assembler::testb(Address dst, int imm8) {
4309   InstructionMark im(this);
4310   prefix(dst);
4311   emit_int8((unsigned char)0xF6);
4312   emit_operand(rax, dst, 1);
4313   emit_int8(imm8);
4314 }
4315 
4316 void Assembler::testl(Register dst, int32_t imm32) {
4317   // not using emit_arith because test
4318   // doesn't support sign-extension of
4319   // 8bit operands
4320   int encode = dst->encoding();
4321   if (encode == 0) {
4322     emit_int8((unsigned char)0xA9);
4323   } else {
4324     encode = prefix_and_encode(encode);
4325     emit_int8((unsigned char)0xF7);
4326     emit_int8((unsigned char)(0xC0 | encode));
4327   }
4328   emit_int32(imm32);
4329 }
4330 
4331 void Assembler::testl(Register dst, Register src) {
4332   (void) prefix_and_encode(dst->encoding(), src->encoding());
4333   emit_arith(0x85, 0xC0, dst, src);
4334 }
4335 
4336 void Assembler::testl(Register dst, Address src) {
4337   InstructionMark im(this);
4338   prefix(src, dst);
4339   emit_int8((unsigned char)0x85);
4340   emit_operand(dst, src);
4341 }
4342 
4343 void Assembler::tzcntl(Register dst, Register src) {
4344   assert(VM_Version::supports_bmi1(), "tzcnt instruction not supported");
4345   emit_int8((unsigned char)0xF3);
4346   int encode = prefix_and_encode(dst->encoding(), src->encoding());
4347   emit_int8(0x0F);
4348   emit_int8((unsigned char)0xBC);
4349   emit_int8((unsigned char)0xC0 | encode);
4350 }
4351 
4352 void Assembler::tzcntq(Register dst, Register src) {
4353   assert(VM_Version::supports_bmi1(), "tzcnt instruction not supported");
4354   emit_int8((unsigned char)0xF3);
4355   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
4356   emit_int8(0x0F);
4357   emit_int8((unsigned char)0xBC);
4358   emit_int8((unsigned char)(0xC0 | encode));
4359 }
4360 
4361 void Assembler::ucomisd(XMMRegister dst, Address src) {
4362   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4363   InstructionMark im(this);
4364   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4365   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
4366   simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4367   emit_int8(0x2E);
4368   emit_operand(dst, src);
4369 }
4370 
4371 void Assembler::ucomisd(XMMRegister dst, XMMRegister src) {
4372   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4373   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4374   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4375   emit_int8(0x2E);
4376   emit_int8((unsigned char)(0xC0 | encode));
4377 }
4378 
4379 void Assembler::ucomiss(XMMRegister dst, Address src) {
4380   NOT_LP64(assert(VM_Version::supports_sse(), ""));
4381   InstructionMark im(this);
4382   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4383   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
4384   simd_prefix(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
4385   emit_int8(0x2E);
4386   emit_operand(dst, src);
4387 }
4388 
4389 void Assembler::ucomiss(XMMRegister dst, XMMRegister src) {
4390   NOT_LP64(assert(VM_Version::supports_sse(), ""));
4391   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4392   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
4393   emit_int8(0x2E);
4394   emit_int8((unsigned char)(0xC0 | encode));
4395 }
4396 
4397 void Assembler::xabort(int8_t imm8) {
4398   emit_int8((unsigned char)0xC6);
4399   emit_int8((unsigned char)0xF8);
4400   emit_int8((unsigned char)(imm8 & 0xFF));
4401 }
4402 
4403 void Assembler::xaddl(Address dst, Register src) {
4404   InstructionMark im(this);
4405   prefix(dst, src);
4406   emit_int8(0x0F);
4407   emit_int8((unsigned char)0xC1);
4408   emit_operand(src, dst);
4409 }
4410 
4411 void Assembler::xbegin(Label& abort, relocInfo::relocType rtype) {
4412   InstructionMark im(this);
4413   relocate(rtype);
4414   if (abort.is_bound()) {
4415     address entry = target(abort);
4416     assert(entry != NULL, "abort entry NULL");
4417     intptr_t offset = entry - pc();
4418     emit_int8((unsigned char)0xC7);
4419     emit_int8((unsigned char)0xF8);
4420     emit_int32(offset - 6); // 2 opcode + 4 address
4421   } else {
4422     abort.add_patch_at(code(), locator());
4423     emit_int8((unsigned char)0xC7);
4424     emit_int8((unsigned char)0xF8);
4425     emit_int32(0);
4426   }
4427 }
4428 
4429 void Assembler::xchgl(Register dst, Address src) { // xchg
4430   InstructionMark im(this);
4431   prefix(src, dst);
4432   emit_int8((unsigned char)0x87);
4433   emit_operand(dst, src);
4434 }
4435 
4436 void Assembler::xchgl(Register dst, Register src) {
4437   int encode = prefix_and_encode(dst->encoding(), src->encoding());
4438   emit_int8((unsigned char)0x87);
4439   emit_int8((unsigned char)(0xC0 | encode));
4440 }
4441 
4442 void Assembler::xend() {
4443   emit_int8((unsigned char)0x0F);
4444   emit_int8((unsigned char)0x01);
4445   emit_int8((unsigned char)0xD5);
4446 }
4447 
4448 void Assembler::xgetbv() {
4449   emit_int8(0x0F);
4450   emit_int8(0x01);
4451   emit_int8((unsigned char)0xD0);
4452 }
4453 
4454 void Assembler::xorl(Register dst, int32_t imm32) {
4455   prefix(dst);
4456   emit_arith(0x81, 0xF0, dst, imm32);
4457 }
4458 
4459 void Assembler::xorl(Register dst, Address src) {
4460   InstructionMark im(this);
4461   prefix(src, dst);
4462   emit_int8(0x33);
4463   emit_operand(dst, src);
4464 }
4465 
4466 void Assembler::xorl(Register dst, Register src) {
4467   (void) prefix_and_encode(dst->encoding(), src->encoding());
4468   emit_arith(0x33, 0xC0, dst, src);
4469 }
4470 
4471 void Assembler::xorb(Register dst, Address src) {
4472   InstructionMark im(this);
4473   prefix(src, dst);
4474   emit_int8(0x32);
4475   emit_operand(dst, src);
4476 }
4477 
4478 // AVX 3-operands scalar float-point arithmetic instructions
4479 
4480 void Assembler::vaddsd(XMMRegister dst, XMMRegister nds, Address src) {
4481   assert(VM_Version::supports_avx(), "");
4482   InstructionMark im(this);
4483   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
4484   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
4485   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
4486   emit_int8(0x58);
4487   emit_operand(dst, src);
4488 }
4489 
4490 void Assembler::vaddsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
4491   assert(VM_Version::supports_avx(), "");
4492   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
4493   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
4494   emit_int8(0x58);
4495   emit_int8((unsigned char)(0xC0 | encode));
4496 }
4497 
4498 void Assembler::vaddss(XMMRegister dst, XMMRegister nds, Address src) {
4499   assert(VM_Version::supports_avx(), "");
4500   InstructionMark im(this);
4501   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
4502   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
4503   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
4504   emit_int8(0x58);
4505   emit_operand(dst, src);
4506 }
4507 
4508 void Assembler::vaddss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
4509   assert(VM_Version::supports_avx(), "");
4510   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
4511   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
4512   emit_int8(0x58);
4513   emit_int8((unsigned char)(0xC0 | encode));
4514 }
4515 
4516 void Assembler::vdivsd(XMMRegister dst, XMMRegister nds, Address src) {
4517   assert(VM_Version::supports_avx(), "");
4518   InstructionMark im(this);
4519   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
4520   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
4521   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
4522   emit_int8(0x5E);
4523   emit_operand(dst, src);
4524 }
4525 
4526 void Assembler::vdivsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
4527   assert(VM_Version::supports_avx(), "");
4528   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
4529   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
4530   emit_int8(0x5E);
4531   emit_int8((unsigned char)(0xC0 | encode));
4532 }
4533 
4534 void Assembler::vdivss(XMMRegister dst, XMMRegister nds, Address src) {
4535   assert(VM_Version::supports_avx(), "");
4536   InstructionMark im(this);
4537   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
4538   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
4539   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
4540   emit_int8(0x5E);
4541   emit_operand(dst, src);
4542 }
4543 
4544 void Assembler::vdivss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
4545   assert(VM_Version::supports_avx(), "");
4546   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
4547   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
4548   emit_int8(0x5E);
4549   emit_int8((unsigned char)(0xC0 | encode));
4550 }
4551 
4552 void Assembler::vmulsd(XMMRegister dst, XMMRegister nds, Address src) {
4553   assert(VM_Version::supports_avx(), "");
4554   InstructionMark im(this);
4555   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
4556   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
4557   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
4558   emit_int8(0x59);
4559   emit_operand(dst, src);
4560 }
4561 
4562 void Assembler::vmulsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
4563   assert(VM_Version::supports_avx(), "");
4564   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
4565   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
4566   emit_int8(0x59);
4567   emit_int8((unsigned char)(0xC0 | encode));
4568 }
4569 
4570 void Assembler::vmulss(XMMRegister dst, XMMRegister nds, Address src) {
4571   assert(VM_Version::supports_avx(), "");
4572   InstructionMark im(this);
4573   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
4574   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
4575   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
4576   emit_int8(0x59);
4577   emit_operand(dst, src);
4578 }
4579 
4580 void Assembler::vmulss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
4581   assert(VM_Version::supports_avx(), "");
4582   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
4583   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
4584   emit_int8(0x59);
4585   emit_int8((unsigned char)(0xC0 | encode));
4586 }
4587 
4588 void Assembler::vsubsd(XMMRegister dst, XMMRegister nds, Address src) {
4589   assert(VM_Version::supports_avx(), "");
4590   InstructionMark im(this);
4591   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
4592   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
4593   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
4594   emit_int8(0x5C);
4595   emit_operand(dst, src);
4596 }
4597 
4598 void Assembler::vsubsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
4599   assert(VM_Version::supports_avx(), "");
4600   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
4601   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
4602   emit_int8(0x5C);
4603   emit_int8((unsigned char)(0xC0 | encode));
4604 }
4605 
4606 void Assembler::vsubss(XMMRegister dst, XMMRegister nds, Address src) {
4607   assert(VM_Version::supports_avx(), "");
4608   InstructionMark im(this);
4609   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
4610   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
4611   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
4612   emit_int8(0x5C);
4613   emit_operand(dst, src);
4614 }
4615 
4616 void Assembler::vsubss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
4617   assert(VM_Version::supports_avx(), "");
4618   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
4619   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
4620   emit_int8(0x5C);
4621   emit_int8((unsigned char)(0xC0 | encode));
4622 }
4623 
4624 //====================VECTOR ARITHMETIC=====================================
4625 
4626 // Float-point vector arithmetic
4627 
4628 void Assembler::addpd(XMMRegister dst, XMMRegister src) {
4629   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4630   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4631   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4632   emit_int8(0x58);
4633   emit_int8((unsigned char)(0xC0 | encode));
4634 }
4635 
4636 void Assembler::addpd(XMMRegister dst, Address src) {
4637   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4638   InstructionMark im(this);
4639   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4640   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
4641   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4642   emit_int8(0x58);
4643   emit_operand(dst, src);
4644 }
4645 
4646 
4647 void Assembler::addps(XMMRegister dst, XMMRegister src) {
4648   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4649   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4650   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
4651   emit_int8(0x58);
4652   emit_int8((unsigned char)(0xC0 | encode));
4653 }
4654 
4655 void Assembler::vaddpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4656   assert(VM_Version::supports_avx(), "");
4657   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4658   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4659   emit_int8(0x58);
4660   emit_int8((unsigned char)(0xC0 | encode));
4661 }
4662 
4663 void Assembler::vaddps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4664   assert(VM_Version::supports_avx(), "");
4665   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4666   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
4667   emit_int8(0x58);
4668   emit_int8((unsigned char)(0xC0 | encode));
4669 }
4670 
4671 void Assembler::vaddpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4672   assert(VM_Version::supports_avx(), "");
4673   InstructionMark im(this);
4674   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4675   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
4676   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4677   emit_int8(0x58);
4678   emit_operand(dst, src);
4679 }
4680 
4681 void Assembler::vaddps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4682   assert(VM_Version::supports_avx(), "");
4683   InstructionMark im(this);
4684   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4685   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
4686   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
4687   emit_int8(0x58);
4688   emit_operand(dst, src);
4689 }
4690 
4691 void Assembler::subpd(XMMRegister dst, XMMRegister src) {
4692   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4693   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4694   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4695   emit_int8(0x5C);
4696   emit_int8((unsigned char)(0xC0 | encode));
4697 }
4698 
4699 void Assembler::subps(XMMRegister dst, XMMRegister src) {
4700   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4701   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4702   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
4703   emit_int8(0x5C);
4704   emit_int8((unsigned char)(0xC0 | encode));
4705 }
4706 
4707 void Assembler::vsubpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4708   assert(VM_Version::supports_avx(), "");
4709   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4710   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4711   emit_int8(0x5C);
4712   emit_int8((unsigned char)(0xC0 | encode));
4713 }
4714 
4715 void Assembler::vsubps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4716   assert(VM_Version::supports_avx(), "");
4717   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4718   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
4719   emit_int8(0x5C);
4720   emit_int8((unsigned char)(0xC0 | encode));
4721 }
4722 
4723 void Assembler::vsubpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4724   assert(VM_Version::supports_avx(), "");
4725   InstructionMark im(this);
4726   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4727   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
4728   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4729   emit_int8(0x5C);
4730   emit_operand(dst, src);
4731 }
4732 
4733 void Assembler::vsubps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4734   assert(VM_Version::supports_avx(), "");
4735   InstructionMark im(this);
4736   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4737   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
4738   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
4739   emit_int8(0x5C);
4740   emit_operand(dst, src);
4741 }
4742 
4743 void Assembler::mulpd(XMMRegister dst, XMMRegister src) {
4744   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4745   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4746   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4747   emit_int8(0x59);
4748   emit_int8((unsigned char)(0xC0 | encode));
4749 }
4750 
4751 void Assembler::mulpd(XMMRegister dst, Address src) {
4752   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4753   InstructionMark im(this);
4754   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4755   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
4756   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4757   emit_int8(0x59);
4758   emit_operand(dst, src);
4759 }
4760 
4761 void Assembler::mulps(XMMRegister dst, XMMRegister src) {
4762   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4763   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4764   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
4765   emit_int8(0x59);
4766   emit_int8((unsigned char)(0xC0 | encode));
4767 }
4768 
4769 void Assembler::vmulpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4770   assert(VM_Version::supports_avx(), "");
4771   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4772   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4773   emit_int8(0x59);
4774   emit_int8((unsigned char)(0xC0 | encode));
4775 }
4776 
4777 void Assembler::vmulps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4778   assert(VM_Version::supports_avx(), "");
4779   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4780   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
4781   emit_int8(0x59);
4782   emit_int8((unsigned char)(0xC0 | encode));
4783 }
4784 
4785 void Assembler::vmulpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4786   assert(VM_Version::supports_avx(), "");
4787   InstructionMark im(this);
4788   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4789   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
4790   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4791   emit_int8(0x59);
4792   emit_operand(dst, src);
4793 }
4794 
4795 void Assembler::vmulps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4796   assert(VM_Version::supports_avx(), "");
4797   InstructionMark im(this);
4798   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4799   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
4800   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
4801   emit_int8(0x59);
4802   emit_operand(dst, src);
4803 }
4804 
4805 void Assembler::divpd(XMMRegister dst, XMMRegister src) {
4806   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4807   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4808   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4809   emit_int8(0x5E);
4810   emit_int8((unsigned char)(0xC0 | encode));
4811 }
4812 
4813 void Assembler::divps(XMMRegister dst, XMMRegister src) {
4814   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4815   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4816   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
4817   emit_int8(0x5E);
4818   emit_int8((unsigned char)(0xC0 | encode));
4819 }
4820 
4821 void Assembler::vdivpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4822   assert(VM_Version::supports_avx(), "");
4823   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4824   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4825   emit_int8(0x5E);
4826   emit_int8((unsigned char)(0xC0 | encode));
4827 }
4828 
4829 void Assembler::vdivps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4830   assert(VM_Version::supports_avx(), "");
4831   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4832   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
4833   emit_int8(0x5E);
4834   emit_int8((unsigned char)(0xC0 | encode));
4835 }
4836 
4837 void Assembler::vdivpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4838   assert(VM_Version::supports_avx(), "");
4839   InstructionMark im(this);
4840   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4841   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
4842   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4843   emit_int8(0x5E);
4844   emit_operand(dst, src);
4845 }
4846 
4847 void Assembler::vdivps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4848   assert(VM_Version::supports_avx(), "");
4849   InstructionMark im(this);
4850   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4851   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
4852   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
4853   emit_int8(0x5E);
4854   emit_operand(dst, src);
4855 }
4856 
4857 void Assembler::vsqrtpd(XMMRegister dst, XMMRegister src, int vector_len) {
4858   assert(VM_Version::supports_avx(), "");
4859   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4860   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4861   emit_int8(0x51);
4862   emit_int8((unsigned char)(0xC0 | encode));
4863 }
4864 
4865 void Assembler::vsqrtpd(XMMRegister dst, Address src, int vector_len) {
4866   assert(VM_Version::supports_avx(), "");
4867   InstructionMark im(this);
4868   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4869   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
4870   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4871   emit_int8(0x51);
4872   emit_operand(dst, src);
4873 }
4874 
4875 void Assembler::andpd(XMMRegister dst, XMMRegister src) {
4876   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4877   InstructionAttr attributes(AVX_128bit, /* rex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
4878   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4879   emit_int8(0x54);
4880   emit_int8((unsigned char)(0xC0 | encode));
4881 }
4882 
4883 void Assembler::andps(XMMRegister dst, XMMRegister src) {
4884   NOT_LP64(assert(VM_Version::supports_sse(), ""));
4885   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
4886   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
4887   emit_int8(0x54);
4888   emit_int8((unsigned char)(0xC0 | encode));
4889 }
4890 
4891 void Assembler::andps(XMMRegister dst, Address src) {
4892   NOT_LP64(assert(VM_Version::supports_sse(), ""));
4893   InstructionMark im(this);
4894   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
4895   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
4896   simd_prefix(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
4897   emit_int8(0x54);
4898   emit_operand(dst, src);
4899 }
4900 
4901 void Assembler::andpd(XMMRegister dst, Address src) {
4902   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4903   InstructionMark im(this);
4904   InstructionAttr attributes(AVX_128bit, /* rex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
4905   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
4906   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4907   emit_int8(0x54);
4908   emit_operand(dst, src);
4909 }
4910 
4911 void Assembler::vandpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4912   assert(VM_Version::supports_avx(), "");
4913   InstructionAttr attributes(vector_len, /* vex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
4914   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4915   emit_int8(0x54);
4916   emit_int8((unsigned char)(0xC0 | encode));
4917 }
4918 
4919 void Assembler::vandps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4920   assert(VM_Version::supports_avx(), "");
4921   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
4922   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
4923   emit_int8(0x54);
4924   emit_int8((unsigned char)(0xC0 | encode));
4925 }
4926 
4927 void Assembler::vandpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4928   assert(VM_Version::supports_avx(), "");
4929   InstructionMark im(this);
4930   InstructionAttr attributes(vector_len, /* vex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
4931   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
4932   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4933   emit_int8(0x54);
4934   emit_operand(dst, src);
4935 }
4936 
4937 void Assembler::vandps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4938   assert(VM_Version::supports_avx(), "");
4939   InstructionMark im(this);
4940   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
4941   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
4942   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
4943   emit_int8(0x54);
4944   emit_operand(dst, src);
4945 }
4946 
4947 void Assembler::unpckhpd(XMMRegister dst, XMMRegister src) {
4948   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4949   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4950   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4951   emit_int8(0x15);
4952   emit_int8((unsigned char)(0xC0 | encode));
4953 }
4954 
4955 void Assembler::unpcklpd(XMMRegister dst, XMMRegister src) {
4956   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4957   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4958   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4959   emit_int8(0x14);
4960   emit_int8((unsigned char)(0xC0 | encode));
4961 }
4962 
4963 void Assembler::xorpd(XMMRegister dst, XMMRegister src) {
4964   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4965   InstructionAttr attributes(AVX_128bit, /* rex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
4966   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4967   emit_int8(0x57);
4968   emit_int8((unsigned char)(0xC0 | encode));
4969 }
4970 
4971 void Assembler::xorps(XMMRegister dst, XMMRegister src) {
4972   NOT_LP64(assert(VM_Version::supports_sse(), ""));
4973   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
4974   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
4975   emit_int8(0x57);
4976   emit_int8((unsigned char)(0xC0 | encode));
4977 }
4978 
4979 void Assembler::xorpd(XMMRegister dst, Address src) {
4980   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4981   InstructionMark im(this);
4982   InstructionAttr attributes(AVX_128bit, /* rex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
4983   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
4984   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4985   emit_int8(0x57);
4986   emit_operand(dst, src);
4987 }
4988 
4989 void Assembler::xorps(XMMRegister dst, Address src) {
4990   NOT_LP64(assert(VM_Version::supports_sse(), ""));
4991   InstructionMark im(this);
4992   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
4993   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
4994   simd_prefix(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
4995   emit_int8(0x57);
4996   emit_operand(dst, src);
4997 }
4998 
4999 void Assembler::vxorpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5000   assert(VM_Version::supports_avx(), "");
5001   InstructionAttr attributes(vector_len, /* vex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
5002   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5003   emit_int8(0x57);
5004   emit_int8((unsigned char)(0xC0 | encode));
5005 }
5006 
5007 void Assembler::vxorps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5008   assert(VM_Version::supports_avx(), "");
5009   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
5010   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
5011   emit_int8(0x57);
5012   emit_int8((unsigned char)(0xC0 | encode));
5013 }
5014 
5015 void Assembler::vxorpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5016   assert(VM_Version::supports_avx(), "");
5017   InstructionMark im(this);
5018   InstructionAttr attributes(vector_len, /* vex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
5019   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
5020   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5021   emit_int8(0x57);
5022   emit_operand(dst, src);
5023 }
5024 
5025 void Assembler::vxorps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5026   assert(VM_Version::supports_avx(), "");
5027   InstructionMark im(this);
5028   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
5029   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
5030   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
5031   emit_int8(0x57);
5032   emit_operand(dst, src);
5033 }
5034 
5035 // Integer vector arithmetic
5036 void Assembler::vphaddw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5037   assert(VM_Version::supports_avx() && (vector_len == 0) ||
5038          VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
5039   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
5040   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5041   emit_int8(0x01);
5042   emit_int8((unsigned char)(0xC0 | encode));
5043 }
5044 
5045 void Assembler::vphaddd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5046   assert(VM_Version::supports_avx() && (vector_len == 0) ||
5047          VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
5048   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
5049   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5050   emit_int8(0x02);
5051   emit_int8((unsigned char)(0xC0 | encode));
5052 }
5053 
5054 void Assembler::paddb(XMMRegister dst, XMMRegister src) {
5055   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5056   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5057   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5058   emit_int8((unsigned char)0xFC);
5059   emit_int8((unsigned char)(0xC0 | encode));
5060 }
5061 
5062 void Assembler::paddw(XMMRegister dst, XMMRegister src) {
5063   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5064   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5065   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5066   emit_int8((unsigned char)0xFD);
5067   emit_int8((unsigned char)(0xC0 | encode));
5068 }
5069 
5070 void Assembler::paddd(XMMRegister dst, XMMRegister src) {
5071   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5072   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5073   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5074   emit_int8((unsigned char)0xFE);
5075   emit_int8((unsigned char)(0xC0 | encode));
5076 }
5077 
5078 void Assembler::paddd(XMMRegister dst, Address src) {
5079   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5080   InstructionMark im(this);
5081   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5082   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5083   emit_int8((unsigned char)0xFE);
5084   emit_operand(dst, src);
5085 }
5086 
5087 void Assembler::paddq(XMMRegister dst, XMMRegister src) {
5088   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5089   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5090   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5091   emit_int8((unsigned char)0xD4);
5092   emit_int8((unsigned char)(0xC0 | encode));
5093 }
5094 
5095 void Assembler::phaddw(XMMRegister dst, XMMRegister src) {
5096   assert(VM_Version::supports_sse3(), "");
5097   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
5098   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5099   emit_int8(0x01);
5100   emit_int8((unsigned char)(0xC0 | encode));
5101 }
5102 
5103 void Assembler::phaddd(XMMRegister dst, XMMRegister src) {
5104   assert(VM_Version::supports_sse3(), "");
5105   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
5106   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5107   emit_int8(0x02);
5108   emit_int8((unsigned char)(0xC0 | encode));
5109 }
5110 
5111 void Assembler::vpaddb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5112   assert(UseAVX > 0, "requires some form of AVX");
5113   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5114   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5115   emit_int8((unsigned char)0xFC);
5116   emit_int8((unsigned char)(0xC0 | encode));
5117 }
5118 
5119 void Assembler::vpaddw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5120   assert(UseAVX > 0, "requires some form of AVX");
5121   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5122   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5123   emit_int8((unsigned char)0xFD);
5124   emit_int8((unsigned char)(0xC0 | encode));
5125 }
5126 
5127 void Assembler::vpaddd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5128   assert(UseAVX > 0, "requires some form of AVX");
5129   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5130   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5131   emit_int8((unsigned char)0xFE);
5132   emit_int8((unsigned char)(0xC0 | encode));
5133 }
5134 
5135 void Assembler::vpaddq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5136   assert(UseAVX > 0, "requires some form of AVX");
5137   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5138   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5139   emit_int8((unsigned char)0xD4);
5140   emit_int8((unsigned char)(0xC0 | encode));
5141 }
5142 
5143 void Assembler::vpaddb(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5144   assert(UseAVX > 0, "requires some form of AVX");
5145   InstructionMark im(this);
5146   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5147   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
5148   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5149   emit_int8((unsigned char)0xFC);
5150   emit_operand(dst, src);
5151 }
5152 
5153 void Assembler::vpaddw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5154   assert(UseAVX > 0, "requires some form of AVX");
5155   InstructionMark im(this);
5156   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5157   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
5158   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5159   emit_int8((unsigned char)0xFD);
5160   emit_operand(dst, src);
5161 }
5162 
5163 void Assembler::vpaddd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5164   assert(UseAVX > 0, "requires some form of AVX");
5165   InstructionMark im(this);
5166   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5167   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
5168   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5169   emit_int8((unsigned char)0xFE);
5170   emit_operand(dst, src);
5171 }
5172 
5173 void Assembler::vpaddq(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5174   assert(UseAVX > 0, "requires some form of AVX");
5175   InstructionMark im(this);
5176   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5177   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
5178   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5179   emit_int8((unsigned char)0xD4);
5180   emit_operand(dst, src);
5181 }
5182 
5183 void Assembler::psubb(XMMRegister dst, XMMRegister src) {
5184   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5185   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5186   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5187   emit_int8((unsigned char)0xF8);
5188   emit_int8((unsigned char)(0xC0 | encode));
5189 }
5190 
5191 void Assembler::psubw(XMMRegister dst, XMMRegister src) {
5192   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5193   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5194   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5195   emit_int8((unsigned char)0xF9);
5196   emit_int8((unsigned char)(0xC0 | encode));
5197 }
5198 
5199 void Assembler::psubd(XMMRegister dst, XMMRegister src) {
5200   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5201   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5202   emit_int8((unsigned char)0xFA);
5203   emit_int8((unsigned char)(0xC0 | encode));
5204 }
5205 
5206 void Assembler::psubq(XMMRegister dst, XMMRegister src) {
5207   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5208   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5209   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5210   emit_int8((unsigned char)0xFB);
5211   emit_int8((unsigned char)(0xC0 | encode));
5212 }
5213 
5214 void Assembler::vpsubb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5215   assert(UseAVX > 0, "requires some form of AVX");
5216   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5217   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5218   emit_int8((unsigned char)0xF8);
5219   emit_int8((unsigned char)(0xC0 | encode));
5220 }
5221 
5222 void Assembler::vpsubw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5223   assert(UseAVX > 0, "requires some form of AVX");
5224   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5225   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5226   emit_int8((unsigned char)0xF9);
5227   emit_int8((unsigned char)(0xC0 | encode));
5228 }
5229 
5230 void Assembler::vpsubd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5231   assert(UseAVX > 0, "requires some form of AVX");
5232   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5233   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5234   emit_int8((unsigned char)0xFA);
5235   emit_int8((unsigned char)(0xC0 | encode));
5236 }
5237 
5238 void Assembler::vpsubq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5239   assert(UseAVX > 0, "requires some form of AVX");
5240   InstructionAttr attributes(vector_len, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5241   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5242   emit_int8((unsigned char)0xFB);
5243   emit_int8((unsigned char)(0xC0 | encode));
5244 }
5245 
5246 void Assembler::vpsubb(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5247   assert(UseAVX > 0, "requires some form of AVX");
5248   InstructionMark im(this);
5249   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5250   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
5251   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5252   emit_int8((unsigned char)0xF8);
5253   emit_operand(dst, src);
5254 }
5255 
5256 void Assembler::vpsubw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5257   assert(UseAVX > 0, "requires some form of AVX");
5258   InstructionMark im(this);
5259   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5260   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
5261   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5262   emit_int8((unsigned char)0xF9);
5263   emit_operand(dst, src);
5264 }
5265 
5266 void Assembler::vpsubd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5267   assert(UseAVX > 0, "requires some form of AVX");
5268   InstructionMark im(this);
5269   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5270   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
5271   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5272   emit_int8((unsigned char)0xFA);
5273   emit_operand(dst, src);
5274 }
5275 
5276 void Assembler::vpsubq(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5277   assert(UseAVX > 0, "requires some form of AVX");
5278   InstructionMark im(this);
5279   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5280   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
5281   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5282   emit_int8((unsigned char)0xFB);
5283   emit_operand(dst, src);
5284 }
5285 
5286 void Assembler::pmullw(XMMRegister dst, XMMRegister src) {
5287   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5288   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5289   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5290   emit_int8((unsigned char)0xD5);
5291   emit_int8((unsigned char)(0xC0 | encode));
5292 }
5293 
5294 void Assembler::pmulld(XMMRegister dst, XMMRegister src) {
5295   assert(VM_Version::supports_sse4_1(), "");
5296   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5297   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5298   emit_int8(0x40);
5299   emit_int8((unsigned char)(0xC0 | encode));
5300 }
5301 
5302 void Assembler::vpmullw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5303   assert(UseAVX > 0, "requires some form of AVX");
5304   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5305   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5306   emit_int8((unsigned char)0xD5);
5307   emit_int8((unsigned char)(0xC0 | encode));
5308 }
5309 
5310 void Assembler::vpmulld(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5311   assert(UseAVX > 0, "requires some form of AVX");
5312   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5313   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5314   emit_int8(0x40);
5315   emit_int8((unsigned char)(0xC0 | encode));
5316 }
5317 
5318 void Assembler::vpmullq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5319   assert(UseAVX > 2, "requires some form of AVX");
5320   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
5321   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5322   emit_int8(0x40);
5323   emit_int8((unsigned char)(0xC0 | encode));
5324 }
5325 
5326 void Assembler::vpmullw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5327   assert(UseAVX > 0, "requires some form of AVX");
5328   InstructionMark im(this);
5329   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5330   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
5331   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5332   emit_int8((unsigned char)0xD5);
5333   emit_operand(dst, src);
5334 }
5335 
5336 void Assembler::vpmulld(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5337   assert(UseAVX > 0, "requires some form of AVX");
5338   InstructionMark im(this);
5339   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5340   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
5341   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5342   emit_int8(0x40);
5343   emit_operand(dst, src);
5344 }
5345 
5346 void Assembler::vpmullq(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5347   assert(UseAVX > 0, "requires some form of AVX");
5348   InstructionMark im(this);
5349   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
5350   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
5351   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5352   emit_int8(0x40);
5353   emit_operand(dst, src);
5354 }
5355 
5356 // Shift packed integers left by specified number of bits.
5357 void Assembler::psllw(XMMRegister dst, int shift) {
5358   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5359   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5360   // XMM6 is for /6 encoding: 66 0F 71 /6 ib
5361   int encode = simd_prefix_and_encode(xmm6, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5362   emit_int8(0x71);
5363   emit_int8((unsigned char)(0xC0 | encode));
5364   emit_int8(shift & 0xFF);
5365 }
5366 
5367 void Assembler::pslld(XMMRegister dst, int shift) {
5368   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5369   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5370   // XMM6 is for /6 encoding: 66 0F 72 /6 ib
5371   int encode = simd_prefix_and_encode(xmm6, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5372   emit_int8(0x72);
5373   emit_int8((unsigned char)(0xC0 | encode));
5374   emit_int8(shift & 0xFF);
5375 }
5376 
5377 void Assembler::psllq(XMMRegister dst, int shift) {
5378   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5379   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5380   // XMM6 is for /6 encoding: 66 0F 73 /6 ib
5381   int encode = simd_prefix_and_encode(xmm6, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5382   emit_int8(0x73);
5383   emit_int8((unsigned char)(0xC0 | encode));
5384   emit_int8(shift & 0xFF);
5385 }
5386 
5387 void Assembler::psllw(XMMRegister dst, XMMRegister shift) {
5388   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5389   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5390   int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5391   emit_int8((unsigned char)0xF1);
5392   emit_int8((unsigned char)(0xC0 | encode));
5393 }
5394 
5395 void Assembler::pslld(XMMRegister dst, XMMRegister shift) {
5396   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5397   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5398   int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5399   emit_int8((unsigned char)0xF2);
5400   emit_int8((unsigned char)(0xC0 | encode));
5401 }
5402 
5403 void Assembler::psllq(XMMRegister dst, XMMRegister shift) {
5404   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5405   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5406   int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5407   emit_int8((unsigned char)0xF3);
5408   emit_int8((unsigned char)(0xC0 | encode));
5409 }
5410 
5411 void Assembler::vpsllw(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
5412   assert(UseAVX > 0, "requires some form of AVX");
5413   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5414   // XMM6 is for /6 encoding: 66 0F 71 /6 ib
5415   int encode = vex_prefix_and_encode(xmm6->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5416   emit_int8(0x71);
5417   emit_int8((unsigned char)(0xC0 | encode));
5418   emit_int8(shift & 0xFF);
5419 }
5420 
5421 void Assembler::vpslld(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
5422   assert(UseAVX > 0, "requires some form of AVX");
5423   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5424   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5425   // XMM6 is for /6 encoding: 66 0F 72 /6 ib
5426   int encode = vex_prefix_and_encode(xmm6->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5427   emit_int8(0x72);
5428   emit_int8((unsigned char)(0xC0 | encode));
5429   emit_int8(shift & 0xFF);
5430 }
5431 
5432 void Assembler::vpsllq(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
5433   assert(UseAVX > 0, "requires some form of AVX");
5434   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5435   // XMM6 is for /6 encoding: 66 0F 73 /6 ib
5436   int encode = vex_prefix_and_encode(xmm6->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5437   emit_int8(0x73);
5438   emit_int8((unsigned char)(0xC0 | encode));
5439   emit_int8(shift & 0xFF);
5440 }
5441 
5442 void Assembler::vpsllw(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
5443   assert(UseAVX > 0, "requires some form of AVX");
5444   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5445   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5446   emit_int8((unsigned char)0xF1);
5447   emit_int8((unsigned char)(0xC0 | encode));
5448 }
5449 
5450 void Assembler::vpslld(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
5451   assert(UseAVX > 0, "requires some form of AVX");
5452   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5453   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5454   emit_int8((unsigned char)0xF2);
5455   emit_int8((unsigned char)(0xC0 | encode));
5456 }
5457 
5458 void Assembler::vpsllq(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
5459   assert(UseAVX > 0, "requires some form of AVX");
5460   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5461   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5462   emit_int8((unsigned char)0xF3);
5463   emit_int8((unsigned char)(0xC0 | encode));
5464 }
5465 
5466 // Shift packed integers logically right by specified number of bits.
5467 void Assembler::psrlw(XMMRegister dst, int shift) {
5468   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5469   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5470   // XMM2 is for /2 encoding: 66 0F 71 /2 ib
5471   int encode = simd_prefix_and_encode(xmm2, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5472   emit_int8(0x71);
5473   emit_int8((unsigned char)(0xC0 | encode));
5474   emit_int8(shift & 0xFF);
5475 }
5476 
5477 void Assembler::psrld(XMMRegister dst, int shift) {
5478   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5479   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5480   // XMM2 is for /2 encoding: 66 0F 72 /2 ib
5481   int encode = simd_prefix_and_encode(xmm2, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5482   emit_int8(0x72);
5483   emit_int8((unsigned char)(0xC0 | encode));
5484   emit_int8(shift & 0xFF);
5485 }
5486 
5487 void Assembler::psrlq(XMMRegister dst, int shift) {
5488   // Do not confuse it with psrldq SSE2 instruction which
5489   // shifts 128 bit value in xmm register by number of bytes.
5490   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5491   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5492   // XMM2 is for /2 encoding: 66 0F 73 /2 ib
5493   int encode = simd_prefix_and_encode(xmm2, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5494   emit_int8(0x73);
5495   emit_int8((unsigned char)(0xC0 | encode));
5496   emit_int8(shift & 0xFF);
5497 }
5498 
5499 void Assembler::psrlw(XMMRegister dst, XMMRegister shift) {
5500   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5501   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5502   int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5503   emit_int8((unsigned char)0xD1);
5504   emit_int8((unsigned char)(0xC0 | encode));
5505 }
5506 
5507 void Assembler::psrld(XMMRegister dst, XMMRegister shift) {
5508   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5509   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5510   int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5511   emit_int8((unsigned char)0xD2);
5512   emit_int8((unsigned char)(0xC0 | encode));
5513 }
5514 
5515 void Assembler::psrlq(XMMRegister dst, XMMRegister shift) {
5516   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5517   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5518   int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5519   emit_int8((unsigned char)0xD3);
5520   emit_int8((unsigned char)(0xC0 | encode));
5521 }
5522 
5523 void Assembler::vpsrlw(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
5524   assert(UseAVX > 0, "requires some form of AVX");
5525   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5526   // XMM2 is for /2 encoding: 66 0F 71 /2 ib
5527   int encode = vex_prefix_and_encode(xmm2->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5528   emit_int8(0x71);
5529   emit_int8((unsigned char)(0xC0 | encode));
5530   emit_int8(shift & 0xFF);
5531 }
5532 
5533 void Assembler::vpsrld(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
5534   assert(UseAVX > 0, "requires some form of AVX");
5535   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5536   // XMM2 is for /2 encoding: 66 0F 72 /2 ib
5537   int encode = vex_prefix_and_encode(xmm2->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5538   emit_int8(0x72);
5539   emit_int8((unsigned char)(0xC0 | encode));
5540   emit_int8(shift & 0xFF);
5541 }
5542 
5543 void Assembler::vpsrlq(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
5544   assert(UseAVX > 0, "requires some form of AVX");
5545   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5546   // XMM2 is for /2 encoding: 66 0F 73 /2 ib
5547   int encode = vex_prefix_and_encode(xmm2->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5548   emit_int8(0x73);
5549   emit_int8((unsigned char)(0xC0 | encode));
5550   emit_int8(shift & 0xFF);
5551 }
5552 
5553 void Assembler::vpsrlw(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
5554   assert(UseAVX > 0, "requires some form of AVX");
5555   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5556   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5557   emit_int8((unsigned char)0xD1);
5558   emit_int8((unsigned char)(0xC0 | encode));
5559 }
5560 
5561 void Assembler::vpsrld(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
5562   assert(UseAVX > 0, "requires some form of AVX");
5563   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5564   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5565   emit_int8((unsigned char)0xD2);
5566   emit_int8((unsigned char)(0xC0 | encode));
5567 }
5568 
5569 void Assembler::vpsrlq(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
5570   assert(UseAVX > 0, "requires some form of AVX");
5571   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5572   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5573   emit_int8((unsigned char)0xD3);
5574   emit_int8((unsigned char)(0xC0 | encode));
5575 }
5576 
5577 // Shift packed integers arithmetically right by specified number of bits.
5578 void Assembler::psraw(XMMRegister dst, int shift) {
5579   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5580   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5581   // XMM4 is for /4 encoding: 66 0F 71 /4 ib
5582   int encode = simd_prefix_and_encode(xmm4, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5583   emit_int8(0x71);
5584   emit_int8((unsigned char)(0xC0 | encode));
5585   emit_int8(shift & 0xFF);
5586 }
5587 
5588 void Assembler::psrad(XMMRegister dst, int shift) {
5589   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5590   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5591   // XMM4 is for /4 encoding: 66 0F 72 /4 ib
5592   int encode = simd_prefix_and_encode(xmm4, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5593   emit_int8(0x72);
5594   emit_int8((unsigned char)(0xC0 | encode));
5595   emit_int8(shift & 0xFF);
5596 }
5597 
5598 void Assembler::psraw(XMMRegister dst, XMMRegister shift) {
5599   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5600   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5601   int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5602   emit_int8((unsigned char)0xE1);
5603   emit_int8((unsigned char)(0xC0 | encode));
5604 }
5605 
5606 void Assembler::psrad(XMMRegister dst, XMMRegister shift) {
5607   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5608   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5609   int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5610   emit_int8((unsigned char)0xE2);
5611   emit_int8((unsigned char)(0xC0 | encode));
5612 }
5613 
5614 void Assembler::vpsraw(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
5615   assert(UseAVX > 0, "requires some form of AVX");
5616   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5617   // XMM4 is for /4 encoding: 66 0F 71 /4 ib
5618   int encode = vex_prefix_and_encode(xmm4->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5619   emit_int8(0x71);
5620   emit_int8((unsigned char)(0xC0 | encode));
5621   emit_int8(shift & 0xFF);
5622 }
5623 
5624 void Assembler::vpsrad(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
5625   assert(UseAVX > 0, "requires some form of AVX");
5626   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5627   // XMM4 is for /4 encoding: 66 0F 71 /4 ib
5628   int encode = vex_prefix_and_encode(xmm4->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5629   emit_int8(0x72);
5630   emit_int8((unsigned char)(0xC0 | encode));
5631   emit_int8(shift & 0xFF);
5632 }
5633 
5634 void Assembler::vpsraw(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
5635   assert(UseAVX > 0, "requires some form of AVX");
5636   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5637   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5638   emit_int8((unsigned char)0xE1);
5639   emit_int8((unsigned char)(0xC0 | encode));
5640 }
5641 
5642 void Assembler::vpsrad(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
5643   assert(UseAVX > 0, "requires some form of AVX");
5644   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5645   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5646   emit_int8((unsigned char)0xE2);
5647   emit_int8((unsigned char)(0xC0 | encode));
5648 }
5649 
5650 
5651 // logical operations packed integers
5652 void Assembler::pand(XMMRegister dst, XMMRegister src) {
5653   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5654   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5655   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5656   emit_int8((unsigned char)0xDB);
5657   emit_int8((unsigned char)(0xC0 | encode));
5658 }
5659 
5660 void Assembler::vpand(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5661   assert(UseAVX > 0, "requires some form of AVX");
5662   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5663   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5664   emit_int8((unsigned char)0xDB);
5665   emit_int8((unsigned char)(0xC0 | encode));
5666 }
5667 
5668 void Assembler::vpand(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5669   assert(UseAVX > 0, "requires some form of AVX");
5670   InstructionMark im(this);
5671   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5672   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
5673   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5674   emit_int8((unsigned char)0xDB);
5675   emit_operand(dst, src);
5676 }
5677 
5678 void Assembler::pandn(XMMRegister dst, XMMRegister src) {
5679   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5680   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5681   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5682   emit_int8((unsigned char)0xDF);
5683   emit_int8((unsigned char)(0xC0 | encode));
5684 }
5685 
5686 void Assembler::por(XMMRegister dst, XMMRegister src) {
5687   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5688   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5689   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5690   emit_int8((unsigned char)0xEB);
5691   emit_int8((unsigned char)(0xC0 | encode));
5692 }
5693 
5694 void Assembler::vpor(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5695   assert(UseAVX > 0, "requires some form of AVX");
5696   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5697   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5698   emit_int8((unsigned char)0xEB);
5699   emit_int8((unsigned char)(0xC0 | encode));
5700 }
5701 
5702 void Assembler::vpor(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5703   assert(UseAVX > 0, "requires some form of AVX");
5704   InstructionMark im(this);
5705   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5706   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
5707   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5708   emit_int8((unsigned char)0xEB);
5709   emit_operand(dst, src);
5710 }
5711 
5712 void Assembler::pxor(XMMRegister dst, XMMRegister src) {
5713   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5714   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5715   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5716   emit_int8((unsigned char)0xEF);
5717   emit_int8((unsigned char)(0xC0 | encode));
5718 }
5719 
5720 void Assembler::vpxor(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5721   assert(UseAVX > 0, "requires some form of AVX");
5722   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5723   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5724   emit_int8((unsigned char)0xEF);
5725   emit_int8((unsigned char)(0xC0 | encode));
5726 }
5727 
5728 void Assembler::vpxor(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5729   assert(UseAVX > 0, "requires some form of AVX");
5730   InstructionMark im(this);
5731   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5732   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
5733   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5734   emit_int8((unsigned char)0xEF);
5735   emit_operand(dst, src);
5736 }
5737 
5738 
5739 // vinserti forms
5740 
5741 void Assembler::vinserti128(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) {
5742   assert(VM_Version::supports_avx2(), "");
5743   assert(imm8 <= 0x01, "imm8: %u", imm8);
5744   int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_256bit;
5745   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
5746   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
5747   emit_int8(0x38);
5748   emit_int8((unsigned char)(0xC0 | encode));
5749   // 0x00 - insert into lower 128 bits
5750   // 0x01 - insert into upper 128 bits
5751   emit_int8(imm8 & 0x01);
5752 }
5753 
5754 void Assembler::vinserti128(XMMRegister dst, XMMRegister nds, Address src, uint8_t imm8) {
5755   assert(VM_Version::supports_avx2(), "");
5756   assert(dst != xnoreg, "sanity");
5757   assert(imm8 <= 0x01, "imm8: %u", imm8);
5758   int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_256bit;
5759   InstructionMark im(this);
5760   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
5761   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
5762   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
5763   emit_int8(0x38);
5764   emit_operand(dst, src);
5765   // 0x00 - insert into lower 128 bits
5766   // 0x01 - insert into upper 128 bits
5767   emit_int8(imm8 & 0x01);
5768 }
5769 
5770 void Assembler::vinserti32x4(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) {
5771   assert(VM_Version::supports_evex(), "");
5772   assert(imm8 <= 0x03, "imm8: %u", imm8);
5773   InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
5774   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
5775   emit_int8(0x38);
5776   emit_int8((unsigned char)(0xC0 | encode));
5777   // 0x00 - insert into q0 128 bits (0..127)
5778   // 0x01 - insert into q1 128 bits (128..255)
5779   // 0x02 - insert into q2 128 bits (256..383)
5780   // 0x03 - insert into q3 128 bits (384..511)
5781   emit_int8(imm8 & 0x03);
5782 }
5783 
5784 void Assembler::vinserti32x4(XMMRegister dst, XMMRegister nds, Address src, uint8_t imm8) {
5785   assert(VM_Version::supports_avx(), "");
5786   assert(dst != xnoreg, "sanity");
5787   assert(imm8 <= 0x03, "imm8: %u", imm8);
5788   int vector_len = VM_Version::supports_evex() ? AVX_512bit : AVX_256bit;
5789   InstructionMark im(this);
5790   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
5791   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
5792   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
5793   emit_int8(0x18);
5794   emit_operand(dst, src);
5795   // 0x00 - insert into q0 128 bits (0..127)
5796   // 0x01 - insert into q1 128 bits (128..255)
5797   // 0x02 - insert into q2 128 bits (256..383)
5798   // 0x03 - insert into q3 128 bits (384..511)
5799   emit_int8(imm8 & 0x03);
5800 }
5801 
5802 void Assembler::vinserti64x4(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) {
5803   assert(VM_Version::supports_evex(), "");
5804   assert(imm8 <= 0x01, "imm8: %u", imm8);
5805   InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
5806   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
5807   emit_int8(0x38);
5808   emit_int8((unsigned char)(0xC0 | encode));
5809   // 0x00 - insert into lower 256 bits
5810   // 0x01 - insert into upper 256 bits
5811   emit_int8(imm8 & 0x01);
5812 }
5813 
5814 
5815 // vinsertf forms
5816 
5817 void Assembler::vinsertf128(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) {
5818   assert(VM_Version::supports_avx(), "");
5819   assert(imm8 <= 0x01, "imm8: %u", imm8);
5820   int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_256bit;
5821   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
5822   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
5823   emit_int8(0x18);
5824   emit_int8((unsigned char)(0xC0 | encode));
5825   // 0x00 - insert into lower 128 bits
5826   // 0x01 - insert into upper 128 bits
5827   emit_int8(imm8 & 0x01);
5828 }
5829 
5830 void Assembler::vinsertf128(XMMRegister dst, XMMRegister nds, Address src, uint8_t imm8) {
5831   assert(VM_Version::supports_avx(), "");
5832   assert(dst != xnoreg, "sanity");
5833   assert(imm8 <= 0x01, "imm8: %u", imm8);
5834   int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_256bit;
5835   InstructionMark im(this);
5836   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
5837   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
5838   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
5839   emit_int8(0x18);
5840   emit_operand(dst, src);
5841   // 0x00 - insert into lower 128 bits
5842   // 0x01 - insert into upper 128 bits
5843   emit_int8(imm8 & 0x01);
5844 }
5845 
5846 void Assembler::vinsertf32x4(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) {
5847   assert(VM_Version::supports_evex(), "");
5848   assert(imm8 <= 0x03, "imm8: %u", imm8);
5849   InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
5850   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
5851   emit_int8(0x18);
5852   emit_int8((unsigned char)(0xC0 | encode));
5853   // 0x00 - insert into q0 128 bits (0..127)
5854   // 0x01 - insert into q1 128 bits (128..255)
5855   // 0x02 - insert into q2 128 bits (256..383)
5856   // 0x03 - insert into q3 128 bits (384..511)
5857   emit_int8(imm8 & 0x03);
5858 }
5859 
5860 void Assembler::vinsertf32x4(XMMRegister dst, XMMRegister nds, Address src, uint8_t imm8) {
5861   assert(VM_Version::supports_avx(), "");
5862   assert(dst != xnoreg, "sanity");
5863   assert(imm8 <= 0x03, "imm8: %u", imm8);
5864   int vector_len = VM_Version::supports_evex() ? AVX_512bit : AVX_256bit;
5865   InstructionMark im(this);
5866   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
5867   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
5868   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
5869   emit_int8(0x18);
5870   emit_operand(dst, src);
5871   // 0x00 - insert into q0 128 bits (0..127)
5872   // 0x01 - insert into q1 128 bits (128..255)
5873   // 0x02 - insert into q2 128 bits (256..383)
5874   // 0x03 - insert into q3 128 bits (384..511)
5875   emit_int8(imm8 & 0x03);
5876 }
5877 
5878 void Assembler::vinsertf64x4(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) {
5879   assert(VM_Version::supports_evex(), "");
5880   assert(imm8 <= 0x01, "imm8: %u", imm8);
5881   InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
5882   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
5883   emit_int8(0x1A);
5884   emit_int8((unsigned char)(0xC0 | encode));
5885   // 0x00 - insert into lower 256 bits
5886   // 0x01 - insert into upper 256 bits
5887   emit_int8(imm8 & 0x01);
5888 }
5889 
5890 void Assembler::vinsertf64x4(XMMRegister dst, XMMRegister nds, Address src, uint8_t imm8) {
5891   assert(VM_Version::supports_evex(), "");
5892   assert(dst != xnoreg, "sanity");
5893   assert(imm8 <= 0x01, "imm8: %u", imm8);
5894   InstructionMark im(this);
5895   InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
5896   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_64bit);
5897   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
5898   emit_int8(0x1A);
5899   emit_operand(dst, src);
5900   // 0x00 - insert into lower 256 bits
5901   // 0x01 - insert into upper 256 bits
5902   emit_int8(imm8 & 0x01);
5903 }
5904 
5905 
5906 // vextracti forms
5907 
5908 void Assembler::vextracti128(XMMRegister dst, XMMRegister src, uint8_t imm8) {
5909   assert(VM_Version::supports_avx(), "");
5910   assert(imm8 <= 0x01, "imm8: %u", imm8);
5911   int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_256bit;
5912   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
5913   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
5914   emit_int8(0x39);
5915   emit_int8((unsigned char)(0xC0 | encode));
5916   // 0x00 - extract from lower 128 bits
5917   // 0x01 - extract from upper 128 bits
5918   emit_int8(imm8 & 0x01);
5919 }
5920 
5921 void Assembler::vextracti128(Address dst, XMMRegister src, uint8_t imm8) {
5922   assert(VM_Version::supports_avx2(), "");
5923   assert(src != xnoreg, "sanity");
5924   assert(imm8 <= 0x01, "imm8: %u", imm8);
5925   int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_256bit;
5926   InstructionMark im(this);
5927   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
5928   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
5929   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
5930   emit_int8(0x39);
5931   emit_operand(src, dst);
5932   // 0x00 - extract from lower 128 bits
5933   // 0x01 - extract from upper 128 bits
5934   emit_int8(imm8 & 0x01);
5935 }
5936 
5937 void Assembler::vextracti32x4(XMMRegister dst, XMMRegister src, uint8_t imm8) {
5938   assert(VM_Version::supports_avx(), "");
5939   assert(imm8 <= 0x03, "imm8: %u", imm8);
5940   int vector_len = VM_Version::supports_evex() ? AVX_512bit : AVX_256bit;
5941   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
5942   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
5943   emit_int8(0x39);
5944   emit_int8((unsigned char)(0xC0 | encode));
5945   // 0x00 - extract from bits 127:0
5946   // 0x01 - extract from bits 255:128
5947   // 0x02 - extract from bits 383:256
5948   // 0x03 - extract from bits 511:384
5949   emit_int8(imm8 & 0x03);
5950 }
5951 
5952 void Assembler::vextracti32x4(Address dst, XMMRegister src, uint8_t imm8) {
5953   assert(VM_Version::supports_evex(), "");
5954   assert(src != xnoreg, "sanity");
5955   assert(imm8 <= 0x03, "imm8: %u", imm8);
5956   InstructionMark im(this);
5957   InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
5958   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
5959   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
5960   emit_int8(0x39);
5961   emit_operand(src, dst);
5962   // 0x00 - extract from bits 127:0
5963   // 0x01 - extract from bits 255:128
5964   // 0x02 - extract from bits 383:256
5965   // 0x03 - extract from bits 511:384
5966   emit_int8(imm8 & 0x03);
5967 }
5968 
5969 void Assembler::vextracti64x2(XMMRegister dst, XMMRegister src, uint8_t imm8) {
5970   assert(VM_Version::supports_evex(), "");
5971   assert(imm8 <= 0x03, "imm8: %u", imm8);
5972   InstructionAttr attributes(AVX_512bit, /* vex_w */ !_legacy_mode_dq, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
5973   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
5974   emit_int8(0x39);
5975   emit_int8((unsigned char)(0xC0 | encode));
5976   // 0x00 - extract from bits 127:0
5977   // 0x01 - extract from bits 255:128
5978   // 0x02 - extract from bits 383:256
5979   // 0x03 - extract from bits 511:384
5980   emit_int8(imm8 & 0x03);
5981 }
5982 
5983 void Assembler::vextracti64x4(XMMRegister dst, XMMRegister src, uint8_t imm8) {
5984   assert(VM_Version::supports_evex(), "");
5985   assert(imm8 <= 0x01, "imm8: %u", imm8);
5986   InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
5987   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
5988   emit_int8(0x3B);
5989   emit_int8((unsigned char)(0xC0 | encode));
5990   // 0x00 - extract from lower 256 bits
5991   // 0x01 - extract from upper 256 bits
5992   emit_int8(imm8 & 0x01);
5993 }
5994 
5995 
5996 // vextractf forms
5997 
5998 void Assembler::vextractf128(XMMRegister dst, XMMRegister src, uint8_t imm8) {
5999   assert(VM_Version::supports_avx(), "");
6000   assert(imm8 <= 0x01, "imm8: %u", imm8);
6001   int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_256bit;
6002   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6003   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6004   emit_int8(0x19);
6005   emit_int8((unsigned char)(0xC0 | encode));
6006   // 0x00 - extract from lower 128 bits
6007   // 0x01 - extract from upper 128 bits
6008   emit_int8(imm8 & 0x01);
6009 }
6010 
6011 void Assembler::vextractf128(Address dst, XMMRegister src, uint8_t imm8) {
6012   assert(VM_Version::supports_avx(), "");
6013   assert(src != xnoreg, "sanity");
6014   assert(imm8 <= 0x01, "imm8: %u", imm8);
6015   int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_256bit;
6016   InstructionMark im(this);
6017   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6018   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
6019   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6020   emit_int8(0x19);
6021   emit_operand(src, dst);
6022   // 0x00 - extract from lower 128 bits
6023   // 0x01 - extract from upper 128 bits
6024   emit_int8(imm8 & 0x01);
6025 }
6026 
6027 void Assembler::vextractf32x4(XMMRegister dst, XMMRegister src, uint8_t imm8) {
6028   assert(VM_Version::supports_avx(), "");
6029   assert(imm8 <= 0x03, "imm8: %u", imm8);
6030   int vector_len = VM_Version::supports_evex() ? AVX_512bit : AVX_256bit;
6031   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6032   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6033   emit_int8(0x19);
6034   emit_int8((unsigned char)(0xC0 | encode));
6035   // 0x00 - extract from bits 127:0
6036   // 0x01 - extract from bits 255:128
6037   // 0x02 - extract from bits 383:256
6038   // 0x03 - extract from bits 511:384
6039   emit_int8(imm8 & 0x03);
6040 }
6041 
6042 void Assembler::vextractf32x4(Address dst, XMMRegister src, uint8_t imm8) {
6043   assert(VM_Version::supports_evex(), "");
6044   assert(src != xnoreg, "sanity");
6045   assert(imm8 <= 0x03, "imm8: %u", imm8);
6046   InstructionMark im(this);
6047   InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6048   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
6049   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6050   emit_int8(0x19);
6051   emit_operand(src, dst);
6052   // 0x00 - extract from bits 127:0
6053   // 0x01 - extract from bits 255:128
6054   // 0x02 - extract from bits 383:256
6055   // 0x03 - extract from bits 511:384
6056   emit_int8(imm8 & 0x03);
6057 }
6058 
6059 void Assembler::vextractf64x2(XMMRegister dst, XMMRegister src, uint8_t imm8) {
6060   assert(VM_Version::supports_evex(), "");
6061   assert(imm8 <= 0x03, "imm8: %u", imm8);
6062   InstructionAttr attributes(AVX_512bit, /* vex_w */ !_legacy_mode_dq, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6063   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6064   emit_int8(0x19);
6065   emit_int8((unsigned char)(0xC0 | encode));
6066   // 0x00 - extract from bits 127:0
6067   // 0x01 - extract from bits 255:128
6068   // 0x02 - extract from bits 383:256
6069   // 0x03 - extract from bits 511:384
6070   emit_int8(imm8 & 0x03);
6071 }
6072 
6073 void Assembler::vextractf64x4(XMMRegister dst, XMMRegister src, uint8_t imm8) {
6074   assert(VM_Version::supports_evex(), "");
6075   assert(imm8 <= 0x01, "imm8: %u", imm8);
6076   InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6077   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6078   emit_int8(0x1B);
6079   emit_int8((unsigned char)(0xC0 | encode));
6080   // 0x00 - extract from lower 256 bits
6081   // 0x01 - extract from upper 256 bits
6082   emit_int8(imm8 & 0x01);
6083 }
6084 
6085 void Assembler::vextractf64x4(Address dst, XMMRegister src, uint8_t imm8) {
6086   assert(VM_Version::supports_evex(), "");
6087   assert(src != xnoreg, "sanity");
6088   assert(imm8 <= 0x01, "imm8: %u", imm8);
6089   InstructionMark im(this);
6090   InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6091   attributes.set_address_attributes(/* tuple_type */ EVEX_T4,/* input_size_in_bits */  EVEX_64bit);
6092   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6093   emit_int8(0x1B);
6094   emit_operand(src, dst);
6095   // 0x00 - extract from lower 256 bits
6096   // 0x01 - extract from upper 256 bits
6097   emit_int8(imm8 & 0x01);
6098 }
6099 
6100 
6101 // legacy word/dword replicate
6102 void Assembler::vpbroadcastw(XMMRegister dst, XMMRegister src) {
6103   assert(VM_Version::supports_avx2(), "");
6104   InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
6105   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6106   emit_int8(0x79);
6107   emit_int8((unsigned char)(0xC0 | encode));
6108 }
6109 
6110 void Assembler::vpbroadcastd(XMMRegister dst, XMMRegister src) {
6111   assert(VM_Version::supports_avx2(), "");
6112   InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6113   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6114   emit_int8(0x58);
6115   emit_int8((unsigned char)(0xC0 | encode));
6116 }
6117 
6118 
6119 // xmm/mem sourced byte/word/dword/qword replicate
6120 
6121 // duplicate 1-byte integer data from src into programmed locations in dest : requires AVX512BW and AVX512VL
6122 void Assembler::evpbroadcastb(XMMRegister dst, XMMRegister src, int vector_len) {
6123   assert(VM_Version::supports_evex(), "");
6124   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
6125   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6126   emit_int8(0x78);
6127   emit_int8((unsigned char)(0xC0 | encode));
6128 }
6129 
6130 void Assembler::evpbroadcastb(XMMRegister dst, Address src, int vector_len) {
6131   assert(VM_Version::supports_evex(), "");
6132   assert(dst != xnoreg, "sanity");
6133   InstructionMark im(this);
6134   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
6135   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_8bit);
6136   // swap src<->dst for encoding
6137   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6138   emit_int8(0x78);
6139   emit_operand(dst, src);
6140 }
6141 
6142 // duplicate 2-byte integer data from src into programmed locations in dest : requires AVX512BW and AVX512VL
6143 void Assembler::evpbroadcastw(XMMRegister dst, XMMRegister src, int vector_len) {
6144   assert(VM_Version::supports_evex(), "");
6145   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
6146   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6147   emit_int8(0x79);
6148   emit_int8((unsigned char)(0xC0 | encode));
6149 }
6150 
6151 void Assembler::evpbroadcastw(XMMRegister dst, Address src, int vector_len) {
6152   assert(VM_Version::supports_evex(), "");
6153   assert(dst != xnoreg, "sanity");
6154   InstructionMark im(this);
6155   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
6156   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_16bit);
6157   // swap src<->dst for encoding
6158   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6159   emit_int8(0x79);
6160   emit_operand(dst, src);
6161 }
6162 
6163 // duplicate 4-byte integer data from src into programmed locations in dest : requires AVX512VL
6164 void Assembler::evpbroadcastd(XMMRegister dst, XMMRegister src, int vector_len) {
6165   assert(VM_Version::supports_evex(), "");
6166   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6167   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6168   emit_int8(0x58);
6169   emit_int8((unsigned char)(0xC0 | encode));
6170 }
6171 
6172 void Assembler::evpbroadcastd(XMMRegister dst, Address src, int vector_len) {
6173   assert(VM_Version::supports_evex(), "");
6174   assert(dst != xnoreg, "sanity");
6175   InstructionMark im(this);
6176   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6177   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
6178   // swap src<->dst for encoding
6179   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6180   emit_int8(0x58);
6181   emit_operand(dst, src);
6182 }
6183 
6184 // duplicate 8-byte integer data from src into programmed locations in dest : requires AVX512VL
6185 void Assembler::evpbroadcastq(XMMRegister dst, XMMRegister src, int vector_len) {
6186   assert(VM_Version::supports_evex(), "");
6187   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6188   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6189   emit_int8(0x59);
6190   emit_int8((unsigned char)(0xC0 | encode));
6191 }
6192 
6193 void Assembler::evpbroadcastq(XMMRegister dst, Address src, int vector_len) {
6194   assert(VM_Version::supports_evex(), "");
6195   assert(dst != xnoreg, "sanity");
6196   InstructionMark im(this);
6197   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6198   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
6199   // swap src<->dst for encoding
6200   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6201   emit_int8(0x59);
6202   emit_operand(dst, src);
6203 }
6204 
6205 
6206 // scalar single/double precision replicate
6207 
6208 // duplicate single precision data from src into programmed locations in dest : requires AVX512VL
6209 void Assembler::evpbroadcastss(XMMRegister dst, XMMRegister src, int vector_len) {
6210   assert(VM_Version::supports_evex(), "");
6211   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6212   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6213   emit_int8(0x18);
6214   emit_int8((unsigned char)(0xC0 | encode));
6215 }
6216 
6217 void Assembler::evpbroadcastss(XMMRegister dst, Address src, int vector_len) {
6218   assert(VM_Version::supports_evex(), "");
6219   assert(dst != xnoreg, "sanity");
6220   InstructionMark im(this);
6221   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6222   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
6223   // swap src<->dst for encoding
6224   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6225   emit_int8(0x18);
6226   emit_operand(dst, src);
6227 }
6228 
6229 // duplicate double precision data from src into programmed locations in dest : requires AVX512VL
6230 void Assembler::evpbroadcastsd(XMMRegister dst, XMMRegister src, int vector_len) {
6231   assert(VM_Version::supports_evex(), "");
6232   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6233   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6234   emit_int8(0x19);
6235   emit_int8((unsigned char)(0xC0 | encode));
6236 }
6237 
6238 void Assembler::evpbroadcastsd(XMMRegister dst, Address src, int vector_len) {
6239   assert(VM_Version::supports_evex(), "");
6240   assert(dst != xnoreg, "sanity");
6241   InstructionMark im(this);
6242   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6243   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
6244   // swap src<->dst for encoding
6245   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6246   emit_int8(0x19);
6247   emit_operand(dst, src);
6248 }
6249 
6250 
6251 // gpr source broadcast forms
6252 
6253 // duplicate 1-byte integer data from src into programmed locations in dest : requires AVX512BW and AVX512VL
6254 void Assembler::evpbroadcastb(XMMRegister dst, Register src, int vector_len) {
6255   assert(VM_Version::supports_evex(), "");
6256   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
6257   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6258   if (attributes.is_evex_instruction()) {
6259     emit_int8(0x7A);
6260   } else {
6261     emit_int8(0x78);
6262   }
6263   emit_int8((unsigned char)(0xC0 | encode));
6264 }
6265 
6266 // duplicate 2-byte integer data from src into programmed locations in dest : requires AVX512BW and AVX512VL
6267 void Assembler::evpbroadcastw(XMMRegister dst, Register src, int vector_len) {
6268   assert(VM_Version::supports_evex(), "");
6269   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
6270   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6271   if (attributes.is_evex_instruction()) {
6272     emit_int8(0x7B);
6273   } else {
6274     emit_int8(0x79);
6275   }
6276   emit_int8((unsigned char)(0xC0 | encode));
6277 }
6278 
6279 // duplicate 4-byte integer data from src into programmed locations in dest : requires AVX512VL
6280 void Assembler::evpbroadcastd(XMMRegister dst, Register src, int vector_len) {
6281   assert(VM_Version::supports_evex(), "");
6282   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6283   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6284   if (attributes.is_evex_instruction()) {
6285     emit_int8(0x7C);
6286   } else {
6287     emit_int8(0x58);
6288   }
6289   emit_int8((unsigned char)(0xC0 | encode));
6290 }
6291 
6292 // duplicate 8-byte integer data from src into programmed locations in dest : requires AVX512VL
6293 void Assembler::evpbroadcastq(XMMRegister dst, Register src, int vector_len) {
6294   assert(VM_Version::supports_evex(), "");
6295   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6296   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6297   if (attributes.is_evex_instruction()) {
6298     emit_int8(0x7C);
6299   } else {
6300     emit_int8(0x59);
6301   }
6302   emit_int8((unsigned char)(0xC0 | encode));
6303 }
6304 
6305 
6306 // Carry-Less Multiplication Quadword
6307 void Assembler::pclmulqdq(XMMRegister dst, XMMRegister src, int mask) {
6308   assert(VM_Version::supports_clmul(), "");
6309   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
6310   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6311   emit_int8(0x44);
6312   emit_int8((unsigned char)(0xC0 | encode));
6313   emit_int8((unsigned char)mask);
6314 }
6315 
6316 // Carry-Less Multiplication Quadword
6317 void Assembler::vpclmulqdq(XMMRegister dst, XMMRegister nds, XMMRegister src, int mask) {
6318   assert(VM_Version::supports_avx() && VM_Version::supports_clmul(), "");
6319   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
6320   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6321   emit_int8(0x44);
6322   emit_int8((unsigned char)(0xC0 | encode));
6323   emit_int8((unsigned char)mask);
6324 }
6325 
6326 void Assembler::vzeroupper() {
6327   assert(VM_Version::supports_avx(), "");
6328   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
6329   (void)vex_prefix_and_encode(0, 0, 0, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
6330   emit_int8(0x77);
6331 }
6332 
6333 
6334 #ifndef _LP64
6335 // 32bit only pieces of the assembler
6336 
6337 void Assembler::cmp_literal32(Register src1, int32_t imm32, RelocationHolder const& rspec) {
6338   // NO PREFIX AS NEVER 64BIT
6339   InstructionMark im(this);
6340   emit_int8((unsigned char)0x81);
6341   emit_int8((unsigned char)(0xF8 | src1->encoding()));
6342   emit_data(imm32, rspec, 0);
6343 }
6344 
6345 void Assembler::cmp_literal32(Address src1, int32_t imm32, RelocationHolder const& rspec) {
6346   // NO PREFIX AS NEVER 64BIT (not even 32bit versions of 64bit regs
6347   InstructionMark im(this);
6348   emit_int8((unsigned char)0x81);
6349   emit_operand(rdi, src1);
6350   emit_data(imm32, rspec, 0);
6351 }
6352 
6353 // The 64-bit (32bit platform) cmpxchg compares the value at adr with the contents of rdx:rax,
6354 // and stores rcx:rbx into adr if so; otherwise, the value at adr is loaded
6355 // into rdx:rax.  The ZF is set if the compared values were equal, and cleared otherwise.
6356 void Assembler::cmpxchg8(Address adr) {
6357   InstructionMark im(this);
6358   emit_int8(0x0F);
6359   emit_int8((unsigned char)0xC7);
6360   emit_operand(rcx, adr);
6361 }
6362 
6363 void Assembler::decl(Register dst) {
6364   // Don't use it directly. Use MacroAssembler::decrementl() instead.
6365  emit_int8(0x48 | dst->encoding());
6366 }
6367 
6368 #endif // _LP64
6369 
6370 // 64bit typically doesn't use the x87 but needs to for the trig funcs
6371 
6372 void Assembler::fabs() {
6373   emit_int8((unsigned char)0xD9);
6374   emit_int8((unsigned char)0xE1);
6375 }
6376 
6377 void Assembler::fadd(int i) {
6378   emit_farith(0xD8, 0xC0, i);
6379 }
6380 
6381 void Assembler::fadd_d(Address src) {
6382   InstructionMark im(this);
6383   emit_int8((unsigned char)0xDC);
6384   emit_operand32(rax, src);
6385 }
6386 
6387 void Assembler::fadd_s(Address src) {
6388   InstructionMark im(this);
6389   emit_int8((unsigned char)0xD8);
6390   emit_operand32(rax, src);
6391 }
6392 
6393 void Assembler::fadda(int i) {
6394   emit_farith(0xDC, 0xC0, i);
6395 }
6396 
6397 void Assembler::faddp(int i) {
6398   emit_farith(0xDE, 0xC0, i);
6399 }
6400 
6401 void Assembler::fchs() {
6402   emit_int8((unsigned char)0xD9);
6403   emit_int8((unsigned char)0xE0);
6404 }
6405 
6406 void Assembler::fcom(int i) {
6407   emit_farith(0xD8, 0xD0, i);
6408 }
6409 
6410 void Assembler::fcomp(int i) {
6411   emit_farith(0xD8, 0xD8, i);
6412 }
6413 
6414 void Assembler::fcomp_d(Address src) {
6415   InstructionMark im(this);
6416   emit_int8((unsigned char)0xDC);
6417   emit_operand32(rbx, src);
6418 }
6419 
6420 void Assembler::fcomp_s(Address src) {
6421   InstructionMark im(this);
6422   emit_int8((unsigned char)0xD8);
6423   emit_operand32(rbx, src);
6424 }
6425 
6426 void Assembler::fcompp() {
6427   emit_int8((unsigned char)0xDE);
6428   emit_int8((unsigned char)0xD9);
6429 }
6430 
6431 void Assembler::fcos() {
6432   emit_int8((unsigned char)0xD9);
6433   emit_int8((unsigned char)0xFF);
6434 }
6435 
6436 void Assembler::fdecstp() {
6437   emit_int8((unsigned char)0xD9);
6438   emit_int8((unsigned char)0xF6);
6439 }
6440 
6441 void Assembler::fdiv(int i) {
6442   emit_farith(0xD8, 0xF0, i);
6443 }
6444 
6445 void Assembler::fdiv_d(Address src) {
6446   InstructionMark im(this);
6447   emit_int8((unsigned char)0xDC);
6448   emit_operand32(rsi, src);
6449 }
6450 
6451 void Assembler::fdiv_s(Address src) {
6452   InstructionMark im(this);
6453   emit_int8((unsigned char)0xD8);
6454   emit_operand32(rsi, src);
6455 }
6456 
6457 void Assembler::fdiva(int i) {
6458   emit_farith(0xDC, 0xF8, i);
6459 }
6460 
6461 // Note: The Intel manual (Pentium Processor User's Manual, Vol.3, 1994)
6462 //       is erroneous for some of the floating-point instructions below.
6463 
6464 void Assembler::fdivp(int i) {
6465   emit_farith(0xDE, 0xF8, i);                    // ST(0) <- ST(0) / ST(1) and pop (Intel manual wrong)
6466 }
6467 
6468 void Assembler::fdivr(int i) {
6469   emit_farith(0xD8, 0xF8, i);
6470 }
6471 
6472 void Assembler::fdivr_d(Address src) {
6473   InstructionMark im(this);
6474   emit_int8((unsigned char)0xDC);
6475   emit_operand32(rdi, src);
6476 }
6477 
6478 void Assembler::fdivr_s(Address src) {
6479   InstructionMark im(this);
6480   emit_int8((unsigned char)0xD8);
6481   emit_operand32(rdi, src);
6482 }
6483 
6484 void Assembler::fdivra(int i) {
6485   emit_farith(0xDC, 0xF0, i);
6486 }
6487 
6488 void Assembler::fdivrp(int i) {
6489   emit_farith(0xDE, 0xF0, i);                    // ST(0) <- ST(1) / ST(0) and pop (Intel manual wrong)
6490 }
6491 
6492 void Assembler::ffree(int i) {
6493   emit_farith(0xDD, 0xC0, i);
6494 }
6495 
6496 void Assembler::fild_d(Address adr) {
6497   InstructionMark im(this);
6498   emit_int8((unsigned char)0xDF);
6499   emit_operand32(rbp, adr);
6500 }
6501 
6502 void Assembler::fild_s(Address adr) {
6503   InstructionMark im(this);
6504   emit_int8((unsigned char)0xDB);
6505   emit_operand32(rax, adr);
6506 }
6507 
6508 void Assembler::fincstp() {
6509   emit_int8((unsigned char)0xD9);
6510   emit_int8((unsigned char)0xF7);
6511 }
6512 
6513 void Assembler::finit() {
6514   emit_int8((unsigned char)0x9B);
6515   emit_int8((unsigned char)0xDB);
6516   emit_int8((unsigned char)0xE3);
6517 }
6518 
6519 void Assembler::fist_s(Address adr) {
6520   InstructionMark im(this);
6521   emit_int8((unsigned char)0xDB);
6522   emit_operand32(rdx, adr);
6523 }
6524 
6525 void Assembler::fistp_d(Address adr) {
6526   InstructionMark im(this);
6527   emit_int8((unsigned char)0xDF);
6528   emit_operand32(rdi, adr);
6529 }
6530 
6531 void Assembler::fistp_s(Address adr) {
6532   InstructionMark im(this);
6533   emit_int8((unsigned char)0xDB);
6534   emit_operand32(rbx, adr);
6535 }
6536 
6537 void Assembler::fld1() {
6538   emit_int8((unsigned char)0xD9);
6539   emit_int8((unsigned char)0xE8);
6540 }
6541 
6542 void Assembler::fld_d(Address adr) {
6543   InstructionMark im(this);
6544   emit_int8((unsigned char)0xDD);
6545   emit_operand32(rax, adr);
6546 }
6547 
6548 void Assembler::fld_s(Address adr) {
6549   InstructionMark im(this);
6550   emit_int8((unsigned char)0xD9);
6551   emit_operand32(rax, adr);
6552 }
6553 
6554 
6555 void Assembler::fld_s(int index) {
6556   emit_farith(0xD9, 0xC0, index);
6557 }
6558 
6559 void Assembler::fld_x(Address adr) {
6560   InstructionMark im(this);
6561   emit_int8((unsigned char)0xDB);
6562   emit_operand32(rbp, adr);
6563 }
6564 
6565 void Assembler::fldcw(Address src) {
6566   InstructionMark im(this);
6567   emit_int8((unsigned char)0xD9);
6568   emit_operand32(rbp, src);
6569 }
6570 
6571 void Assembler::fldenv(Address src) {
6572   InstructionMark im(this);
6573   emit_int8((unsigned char)0xD9);
6574   emit_operand32(rsp, src);
6575 }
6576 
6577 void Assembler::fldlg2() {
6578   emit_int8((unsigned char)0xD9);
6579   emit_int8((unsigned char)0xEC);
6580 }
6581 
6582 void Assembler::fldln2() {
6583   emit_int8((unsigned char)0xD9);
6584   emit_int8((unsigned char)0xED);
6585 }
6586 
6587 void Assembler::fldz() {
6588   emit_int8((unsigned char)0xD9);
6589   emit_int8((unsigned char)0xEE);
6590 }
6591 
6592 void Assembler::flog() {
6593   fldln2();
6594   fxch();
6595   fyl2x();
6596 }
6597 
6598 void Assembler::flog10() {
6599   fldlg2();
6600   fxch();
6601   fyl2x();
6602 }
6603 
6604 void Assembler::fmul(int i) {
6605   emit_farith(0xD8, 0xC8, i);
6606 }
6607 
6608 void Assembler::fmul_d(Address src) {
6609   InstructionMark im(this);
6610   emit_int8((unsigned char)0xDC);
6611   emit_operand32(rcx, src);
6612 }
6613 
6614 void Assembler::fmul_s(Address src) {
6615   InstructionMark im(this);
6616   emit_int8((unsigned char)0xD8);
6617   emit_operand32(rcx, src);
6618 }
6619 
6620 void Assembler::fmula(int i) {
6621   emit_farith(0xDC, 0xC8, i);
6622 }
6623 
6624 void Assembler::fmulp(int i) {
6625   emit_farith(0xDE, 0xC8, i);
6626 }
6627 
6628 void Assembler::fnsave(Address dst) {
6629   InstructionMark im(this);
6630   emit_int8((unsigned char)0xDD);
6631   emit_operand32(rsi, dst);
6632 }
6633 
6634 void Assembler::fnstcw(Address src) {
6635   InstructionMark im(this);
6636   emit_int8((unsigned char)0x9B);
6637   emit_int8((unsigned char)0xD9);
6638   emit_operand32(rdi, src);
6639 }
6640 
6641 void Assembler::fnstsw_ax() {
6642   emit_int8((unsigned char)0xDF);
6643   emit_int8((unsigned char)0xE0);
6644 }
6645 
6646 void Assembler::fprem() {
6647   emit_int8((unsigned char)0xD9);
6648   emit_int8((unsigned char)0xF8);
6649 }
6650 
6651 void Assembler::fprem1() {
6652   emit_int8((unsigned char)0xD9);
6653   emit_int8((unsigned char)0xF5);
6654 }
6655 
6656 void Assembler::frstor(Address src) {
6657   InstructionMark im(this);
6658   emit_int8((unsigned char)0xDD);
6659   emit_operand32(rsp, src);
6660 }
6661 
6662 void Assembler::fsin() {
6663   emit_int8((unsigned char)0xD9);
6664   emit_int8((unsigned char)0xFE);
6665 }
6666 
6667 void Assembler::fsqrt() {
6668   emit_int8((unsigned char)0xD9);
6669   emit_int8((unsigned char)0xFA);
6670 }
6671 
6672 void Assembler::fst_d(Address adr) {
6673   InstructionMark im(this);
6674   emit_int8((unsigned char)0xDD);
6675   emit_operand32(rdx, adr);
6676 }
6677 
6678 void Assembler::fst_s(Address adr) {
6679   InstructionMark im(this);
6680   emit_int8((unsigned char)0xD9);
6681   emit_operand32(rdx, adr);
6682 }
6683 
6684 void Assembler::fstp_d(Address adr) {
6685   InstructionMark im(this);
6686   emit_int8((unsigned char)0xDD);
6687   emit_operand32(rbx, adr);
6688 }
6689 
6690 void Assembler::fstp_d(int index) {
6691   emit_farith(0xDD, 0xD8, index);
6692 }
6693 
6694 void Assembler::fstp_s(Address adr) {
6695   InstructionMark im(this);
6696   emit_int8((unsigned char)0xD9);
6697   emit_operand32(rbx, adr);
6698 }
6699 
6700 void Assembler::fstp_x(Address adr) {
6701   InstructionMark im(this);
6702   emit_int8((unsigned char)0xDB);
6703   emit_operand32(rdi, adr);
6704 }
6705 
6706 void Assembler::fsub(int i) {
6707   emit_farith(0xD8, 0xE0, i);
6708 }
6709 
6710 void Assembler::fsub_d(Address src) {
6711   InstructionMark im(this);
6712   emit_int8((unsigned char)0xDC);
6713   emit_operand32(rsp, src);
6714 }
6715 
6716 void Assembler::fsub_s(Address src) {
6717   InstructionMark im(this);
6718   emit_int8((unsigned char)0xD8);
6719   emit_operand32(rsp, src);
6720 }
6721 
6722 void Assembler::fsuba(int i) {
6723   emit_farith(0xDC, 0xE8, i);
6724 }
6725 
6726 void Assembler::fsubp(int i) {
6727   emit_farith(0xDE, 0xE8, i);                    // ST(0) <- ST(0) - ST(1) and pop (Intel manual wrong)
6728 }
6729 
6730 void Assembler::fsubr(int i) {
6731   emit_farith(0xD8, 0xE8, i);
6732 }
6733 
6734 void Assembler::fsubr_d(Address src) {
6735   InstructionMark im(this);
6736   emit_int8((unsigned char)0xDC);
6737   emit_operand32(rbp, src);
6738 }
6739 
6740 void Assembler::fsubr_s(Address src) {
6741   InstructionMark im(this);
6742   emit_int8((unsigned char)0xD8);
6743   emit_operand32(rbp, src);
6744 }
6745 
6746 void Assembler::fsubra(int i) {
6747   emit_farith(0xDC, 0xE0, i);
6748 }
6749 
6750 void Assembler::fsubrp(int i) {
6751   emit_farith(0xDE, 0xE0, i);                    // ST(0) <- ST(1) - ST(0) and pop (Intel manual wrong)
6752 }
6753 
6754 void Assembler::ftan() {
6755   emit_int8((unsigned char)0xD9);
6756   emit_int8((unsigned char)0xF2);
6757   emit_int8((unsigned char)0xDD);
6758   emit_int8((unsigned char)0xD8);
6759 }
6760 
6761 void Assembler::ftst() {
6762   emit_int8((unsigned char)0xD9);
6763   emit_int8((unsigned char)0xE4);
6764 }
6765 
6766 void Assembler::fucomi(int i) {
6767   // make sure the instruction is supported (introduced for P6, together with cmov)
6768   guarantee(VM_Version::supports_cmov(), "illegal instruction");
6769   emit_farith(0xDB, 0xE8, i);
6770 }
6771 
6772 void Assembler::fucomip(int i) {
6773   // make sure the instruction is supported (introduced for P6, together with cmov)
6774   guarantee(VM_Version::supports_cmov(), "illegal instruction");
6775   emit_farith(0xDF, 0xE8, i);
6776 }
6777 
6778 void Assembler::fwait() {
6779   emit_int8((unsigned char)0x9B);
6780 }
6781 
6782 void Assembler::fxch(int i) {
6783   emit_farith(0xD9, 0xC8, i);
6784 }
6785 
6786 void Assembler::fyl2x() {
6787   emit_int8((unsigned char)0xD9);
6788   emit_int8((unsigned char)0xF1);
6789 }
6790 
6791 void Assembler::frndint() {
6792   emit_int8((unsigned char)0xD9);
6793   emit_int8((unsigned char)0xFC);
6794 }
6795 
6796 void Assembler::f2xm1() {
6797   emit_int8((unsigned char)0xD9);
6798   emit_int8((unsigned char)0xF0);
6799 }
6800 
6801 void Assembler::fldl2e() {
6802   emit_int8((unsigned char)0xD9);
6803   emit_int8((unsigned char)0xEA);
6804 }
6805 
6806 // SSE SIMD prefix byte values corresponding to VexSimdPrefix encoding.
6807 static int simd_pre[4] = { 0, 0x66, 0xF3, 0xF2 };
6808 // SSE opcode second byte values (first is 0x0F) corresponding to VexOpcode encoding.
6809 static int simd_opc[4] = { 0,    0, 0x38, 0x3A };
6810 
6811 // Generate SSE legacy REX prefix and SIMD opcode based on VEX encoding.
6812 void Assembler::rex_prefix(Address adr, XMMRegister xreg, VexSimdPrefix pre, VexOpcode opc, bool rex_w) {
6813   if (pre > 0) {
6814     emit_int8(simd_pre[pre]);
6815   }
6816   if (rex_w) {
6817     prefixq(adr, xreg);
6818   } else {
6819     prefix(adr, xreg);
6820   }
6821   if (opc > 0) {
6822     emit_int8(0x0F);
6823     int opc2 = simd_opc[opc];
6824     if (opc2 > 0) {
6825       emit_int8(opc2);
6826     }
6827   }
6828 }
6829 
6830 int Assembler::rex_prefix_and_encode(int dst_enc, int src_enc, VexSimdPrefix pre, VexOpcode opc, bool rex_w) {
6831   if (pre > 0) {
6832     emit_int8(simd_pre[pre]);
6833   }
6834   int encode = (rex_w) ? prefixq_and_encode(dst_enc, src_enc) : prefix_and_encode(dst_enc, src_enc);
6835   if (opc > 0) {
6836     emit_int8(0x0F);
6837     int opc2 = simd_opc[opc];
6838     if (opc2 > 0) {
6839       emit_int8(opc2);
6840     }
6841   }
6842   return encode;
6843 }
6844 
6845 
6846 void Assembler::vex_prefix(bool vex_r, bool vex_b, bool vex_x, int nds_enc, VexSimdPrefix pre, VexOpcode opc) {
6847   int vector_len = _attributes->get_vector_len();
6848   bool vex_w = _attributes->is_rex_vex_w();
6849   if (vex_b || vex_x || vex_w || (opc == VEX_OPCODE_0F_38) || (opc == VEX_OPCODE_0F_3A)) {
6850     prefix(VEX_3bytes);
6851 
6852     int byte1 = (vex_r ? VEX_R : 0) | (vex_x ? VEX_X : 0) | (vex_b ? VEX_B : 0);
6853     byte1 = (~byte1) & 0xE0;
6854     byte1 |= opc;
6855     emit_int8(byte1);
6856 
6857     int byte2 = ((~nds_enc) & 0xf) << 3;
6858     byte2 |= (vex_w ? VEX_W : 0) | ((vector_len > 0) ? 4 : 0) | pre;
6859     emit_int8(byte2);
6860   } else {
6861     prefix(VEX_2bytes);
6862 
6863     int byte1 = vex_r ? VEX_R : 0;
6864     byte1 = (~byte1) & 0x80;
6865     byte1 |= ((~nds_enc) & 0xf) << 3;
6866     byte1 |= ((vector_len > 0 ) ? 4 : 0) | pre;
6867     emit_int8(byte1);
6868   }
6869 }
6870 
6871 // This is a 4 byte encoding
6872 void Assembler::evex_prefix(bool vex_r, bool vex_b, bool vex_x, bool evex_r, bool evex_v, int nds_enc, VexSimdPrefix pre, VexOpcode opc){
6873   // EVEX 0x62 prefix
6874   prefix(EVEX_4bytes);
6875   bool vex_w = _attributes->is_rex_vex_w();
6876   int evex_encoding = (vex_w ? VEX_W : 0);
6877   // EVEX.b is not currently used for broadcast of single element or data rounding modes
6878   _attributes->set_evex_encoding(evex_encoding);
6879 
6880   // P0: byte 2, initialized to RXBR`00mm
6881   // instead of not'd
6882   int byte2 = (vex_r ? VEX_R : 0) | (vex_x ? VEX_X : 0) | (vex_b ? VEX_B : 0) | (evex_r ? EVEX_Rb : 0);
6883   byte2 = (~byte2) & 0xF0;
6884   // confine opc opcode extensions in mm bits to lower two bits
6885   // of form {0F, 0F_38, 0F_3A}
6886   byte2 |= opc;
6887   emit_int8(byte2);
6888 
6889   // P1: byte 3 as Wvvvv1pp
6890   int byte3 = ((~nds_enc) & 0xf) << 3;
6891   // p[10] is always 1
6892   byte3 |= EVEX_F;
6893   byte3 |= (vex_w & 1) << 7;
6894   // confine pre opcode extensions in pp bits to lower two bits
6895   // of form {66, F3, F2}
6896   byte3 |= pre;
6897   emit_int8(byte3);
6898 
6899   // P2: byte 4 as zL'Lbv'aaa
6900   int byte4 = (_attributes->is_no_reg_mask()) ? 0 : 1; // kregs are implemented in the low 3 bits as aaa (hard code k1, it will be initialized for now)
6901   // EVEX.v` for extending EVEX.vvvv or VIDX
6902   byte4 |= (evex_v ? 0: EVEX_V);
6903   // third EXEC.b for broadcast actions
6904   byte4 |= (_attributes->is_extended_context() ? EVEX_Rb : 0);
6905   // fourth EVEX.L'L for vector length : 0 is 128, 1 is 256, 2 is 512, currently we do not support 1024
6906   byte4 |= ((_attributes->get_vector_len())& 0x3) << 5;
6907   // last is EVEX.z for zero/merge actions
6908   byte4 |= (_attributes->is_clear_context() ? EVEX_Z : 0);
6909   emit_int8(byte4);
6910 }
6911 
6912 void Assembler::vex_prefix(Address adr, int nds_enc, int xreg_enc, VexSimdPrefix pre, VexOpcode opc, InstructionAttr *attributes) {
6913   bool vex_r = ((xreg_enc & 8) == 8) ? 1 : 0;
6914   bool vex_b = adr.base_needs_rex();
6915   bool vex_x = adr.index_needs_rex();
6916   set_attributes(attributes);
6917   attributes->set_current_assembler(this);
6918 
6919   // if vector length is turned off, revert to AVX for vectors smaller than 512-bit
6920   if (UseAVX > 2 && _legacy_mode_vl && attributes->uses_vl()) {
6921     switch (attributes->get_vector_len()) {
6922     case AVX_128bit:
6923     case AVX_256bit:
6924       attributes->set_is_legacy_mode();
6925       break;
6926     }
6927   }
6928 
6929   // For pure EVEX check and see if this instruction
6930   // is allowed in legacy mode and has resources which will
6931   // fit in it.  Pure EVEX instructions will use set_is_evex_instruction in their definition,
6932   // else that field is set when we encode to EVEX
6933   if (UseAVX > 2 && !attributes->is_legacy_mode() &&
6934       !_is_managed && !attributes->is_evex_instruction()) {
6935     if (!_legacy_mode_vl && attributes->get_vector_len() != AVX_512bit) {
6936       bool check_register_bank = NOT_IA32(true) IA32_ONLY(false);
6937       if (check_register_bank) {
6938         // check nds_enc and xreg_enc for upper bank usage
6939         if (nds_enc < 16 && xreg_enc < 16) {
6940           attributes->set_is_legacy_mode();
6941         }
6942       } else {
6943         attributes->set_is_legacy_mode();
6944       }
6945     }
6946   }
6947 
6948   _is_managed = false;
6949   if (UseAVX > 2 && !attributes->is_legacy_mode())
6950   {
6951     bool evex_r = (xreg_enc >= 16);
6952     bool evex_v = (nds_enc >= 16);
6953     attributes->set_is_evex_instruction();
6954     evex_prefix(vex_r, vex_b, vex_x, evex_r, evex_v, nds_enc, pre, opc);
6955   } else {
6956     vex_prefix(vex_r, vex_b, vex_x, nds_enc, pre, opc);
6957   }
6958 }
6959 
6960 int Assembler::vex_prefix_and_encode(int dst_enc, int nds_enc, int src_enc, VexSimdPrefix pre, VexOpcode opc, InstructionAttr *attributes) {
6961   bool vex_r = ((dst_enc & 8) == 8) ? 1 : 0;
6962   bool vex_b = ((src_enc & 8) == 8) ? 1 : 0;
6963   bool vex_x = false;
6964   set_attributes(attributes);
6965   attributes->set_current_assembler(this);
6966   bool check_register_bank = NOT_IA32(true) IA32_ONLY(false);
6967 
6968   // if vector length is turned off, revert to AVX for vectors smaller than 512-bit
6969   if (UseAVX > 2 && _legacy_mode_vl && attributes->uses_vl()) {
6970     switch (attributes->get_vector_len()) {
6971     case AVX_128bit:
6972     case AVX_256bit:
6973       if (check_register_bank) {
6974         if (dst_enc >= 16 || nds_enc >= 16 || src_enc >= 16) {
6975           // up propagate arithmetic instructions to meet RA requirements
6976           attributes->set_vector_len(AVX_512bit);
6977         } else {
6978           attributes->set_is_legacy_mode();
6979         }
6980       } else {
6981         attributes->set_is_legacy_mode();
6982       }
6983       break;
6984     }
6985   }
6986 
6987   // For pure EVEX check and see if this instruction
6988   // is allowed in legacy mode and has resources which will
6989   // fit in it.  Pure EVEX instructions will use set_is_evex_instruction in their definition,
6990   // else that field is set when we encode to EVEX
6991   if (UseAVX > 2 && !attributes->is_legacy_mode() &&
6992       !_is_managed && !attributes->is_evex_instruction()) {
6993     if (!_legacy_mode_vl && attributes->get_vector_len() != AVX_512bit) {
6994       if (check_register_bank) {
6995         // check dst_enc, nds_enc and src_enc for upper bank usage
6996         if (dst_enc < 16 && nds_enc < 16 && src_enc < 16) {
6997           attributes->set_is_legacy_mode();
6998         }
6999       } else {
7000         attributes->set_is_legacy_mode();
7001       }
7002     }
7003   }
7004 
7005   _is_managed = false;
7006   if (UseAVX > 2 && !attributes->is_legacy_mode())
7007   {
7008     bool evex_r = (dst_enc >= 16);
7009     bool evex_v = (nds_enc >= 16);
7010     // can use vex_x as bank extender on rm encoding
7011     vex_x = (src_enc >= 16);
7012     attributes->set_is_evex_instruction();
7013     evex_prefix(vex_r, vex_b, vex_x, evex_r, evex_v, nds_enc, pre, opc);
7014   } else {
7015     vex_prefix(vex_r, vex_b, vex_x, nds_enc, pre, opc);
7016   }
7017 
7018   // return modrm byte components for operands
7019   return (((dst_enc & 7) << 3) | (src_enc & 7));
7020 }
7021 
7022 
7023 void Assembler::simd_prefix(XMMRegister xreg, XMMRegister nds, Address adr, VexSimdPrefix pre,
7024                             VexOpcode opc, InstructionAttr *attributes) {
7025   if (UseAVX > 0) {
7026     int xreg_enc = xreg->encoding();
7027     int nds_enc = nds->is_valid() ? nds->encoding() : 0;
7028     vex_prefix(adr, nds_enc, xreg_enc, pre, opc, attributes);
7029   } else {
7030     assert((nds == xreg) || (nds == xnoreg), "wrong sse encoding");
7031     rex_prefix(adr, xreg, pre, opc, attributes->is_rex_vex_w());
7032   }
7033 }
7034 
7035 int Assembler::simd_prefix_and_encode(XMMRegister dst, XMMRegister nds, XMMRegister src, VexSimdPrefix pre,
7036                                       VexOpcode opc, InstructionAttr *attributes) {
7037   int dst_enc = dst->encoding();
7038   int src_enc = src->encoding();
7039   if (UseAVX > 0) {
7040     int nds_enc = nds->is_valid() ? nds->encoding() : 0;
7041     return vex_prefix_and_encode(dst_enc, nds_enc, src_enc, pre, opc, attributes);
7042   } else {
7043     assert((nds == dst) || (nds == src) || (nds == xnoreg), "wrong sse encoding");
7044     return rex_prefix_and_encode(dst_enc, src_enc, pre, opc, attributes->is_rex_vex_w());
7045   }
7046 }
7047 
7048 void Assembler::cmppd(XMMRegister dst, XMMRegister nds, XMMRegister src, int cop, int vector_len) {
7049   assert(VM_Version::supports_avx(), "");
7050   assert(!VM_Version::supports_evex(), "");
7051   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
7052   int encode = simd_prefix_and_encode(dst, nds, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7053   emit_int8((unsigned char)0xC2);
7054   emit_int8((unsigned char)(0xC0 | encode));
7055   emit_int8((unsigned char)(0xF & cop));
7056 }
7057 
7058 void Assembler::vpblendd(XMMRegister dst, XMMRegister nds, XMMRegister src1, XMMRegister src2, int vector_len) {
7059   assert(VM_Version::supports_avx(), "");
7060   assert(!VM_Version::supports_evex(), "");
7061   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
7062   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src1->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
7063   emit_int8((unsigned char)0x4B);
7064   emit_int8((unsigned char)(0xC0 | encode));
7065   int src2_enc = src2->encoding();
7066   emit_int8((unsigned char)(0xF0 & src2_enc<<4));
7067 }
7068 
7069 
7070 #ifndef _LP64
7071 
7072 void Assembler::incl(Register dst) {
7073   // Don't use it directly. Use MacroAssembler::incrementl() instead.
7074   emit_int8(0x40 | dst->encoding());
7075 }
7076 
7077 void Assembler::lea(Register dst, Address src) {
7078   leal(dst, src);
7079 }
7080 
7081 void Assembler::mov_literal32(Address dst, int32_t imm32, RelocationHolder const& rspec) {
7082   InstructionMark im(this);
7083   emit_int8((unsigned char)0xC7);
7084   emit_operand(rax, dst);
7085   emit_data((int)imm32, rspec, 0);
7086 }
7087 
7088 void Assembler::mov_literal32(Register dst, int32_t imm32, RelocationHolder const& rspec) {
7089   InstructionMark im(this);
7090   int encode = prefix_and_encode(dst->encoding());
7091   emit_int8((unsigned char)(0xB8 | encode));
7092   emit_data((int)imm32, rspec, 0);
7093 }
7094 
7095 void Assembler::popa() { // 32bit
7096   emit_int8(0x61);
7097 }
7098 
7099 void Assembler::push_literal32(int32_t imm32, RelocationHolder const& rspec) {
7100   InstructionMark im(this);
7101   emit_int8(0x68);
7102   emit_data(imm32, rspec, 0);
7103 }
7104 
7105 void Assembler::pusha() { // 32bit
7106   emit_int8(0x60);
7107 }
7108 
7109 void Assembler::set_byte_if_not_zero(Register dst) {
7110   emit_int8(0x0F);
7111   emit_int8((unsigned char)0x95);
7112   emit_int8((unsigned char)(0xE0 | dst->encoding()));
7113 }
7114 
7115 void Assembler::shldl(Register dst, Register src) {
7116   emit_int8(0x0F);
7117   emit_int8((unsigned char)0xA5);
7118   emit_int8((unsigned char)(0xC0 | src->encoding() << 3 | dst->encoding()));
7119 }
7120 
7121 // 0F A4 / r ib
7122 void Assembler::shldl(Register dst, Register src, int8_t imm8) {
7123   emit_int8(0x0F);
7124   emit_int8((unsigned char)0xA4);
7125   emit_int8((unsigned char)(0xC0 | src->encoding() << 3 | dst->encoding()));
7126   emit_int8(imm8);
7127 }
7128 
7129 void Assembler::shrdl(Register dst, Register src) {
7130   emit_int8(0x0F);
7131   emit_int8((unsigned char)0xAD);
7132   emit_int8((unsigned char)(0xC0 | src->encoding() << 3 | dst->encoding()));
7133 }
7134 
7135 #else // LP64
7136 
7137 void Assembler::set_byte_if_not_zero(Register dst) {
7138   int enc = prefix_and_encode(dst->encoding(), true);
7139   emit_int8(0x0F);
7140   emit_int8((unsigned char)0x95);
7141   emit_int8((unsigned char)(0xE0 | enc));
7142 }
7143 
7144 // 64bit only pieces of the assembler
7145 // This should only be used by 64bit instructions that can use rip-relative
7146 // it cannot be used by instructions that want an immediate value.
7147 
7148 bool Assembler::reachable(AddressLiteral adr) {
7149   int64_t disp;
7150   // None will force a 64bit literal to the code stream. Likely a placeholder
7151   // for something that will be patched later and we need to certain it will
7152   // always be reachable.
7153   if (adr.reloc() == relocInfo::none) {
7154     return false;
7155   }
7156   if (adr.reloc() == relocInfo::internal_word_type) {
7157     // This should be rip relative and easily reachable.
7158     return true;
7159   }
7160   if (adr.reloc() == relocInfo::virtual_call_type ||
7161       adr.reloc() == relocInfo::opt_virtual_call_type ||
7162       adr.reloc() == relocInfo::static_call_type ||
7163       adr.reloc() == relocInfo::static_stub_type ) {
7164     // This should be rip relative within the code cache and easily
7165     // reachable until we get huge code caches. (At which point
7166     // ic code is going to have issues).
7167     return true;
7168   }
7169   if (adr.reloc() != relocInfo::external_word_type &&
7170       adr.reloc() != relocInfo::poll_return_type &&  // these are really external_word but need special
7171       adr.reloc() != relocInfo::poll_type &&         // relocs to identify them
7172       adr.reloc() != relocInfo::runtime_call_type ) {
7173     return false;
7174   }
7175 
7176   // Stress the correction code
7177   if (ForceUnreachable) {
7178     // Must be runtimecall reloc, see if it is in the codecache
7179     // Flipping stuff in the codecache to be unreachable causes issues
7180     // with things like inline caches where the additional instructions
7181     // are not handled.
7182     if (CodeCache::find_blob(adr._target) == NULL) {
7183       return false;
7184     }
7185   }
7186   // For external_word_type/runtime_call_type if it is reachable from where we
7187   // are now (possibly a temp buffer) and where we might end up
7188   // anywhere in the codeCache then we are always reachable.
7189   // This would have to change if we ever save/restore shared code
7190   // to be more pessimistic.
7191   disp = (int64_t)adr._target - ((int64_t)CodeCache::low_bound() + sizeof(int));
7192   if (!is_simm32(disp)) return false;
7193   disp = (int64_t)adr._target - ((int64_t)CodeCache::high_bound() + sizeof(int));
7194   if (!is_simm32(disp)) return false;
7195 
7196   disp = (int64_t)adr._target - ((int64_t)pc() + sizeof(int));
7197 
7198   // Because rip relative is a disp + address_of_next_instruction and we
7199   // don't know the value of address_of_next_instruction we apply a fudge factor
7200   // to make sure we will be ok no matter the size of the instruction we get placed into.
7201   // We don't have to fudge the checks above here because they are already worst case.
7202 
7203   // 12 == override/rex byte, opcode byte, rm byte, sib byte, a 4-byte disp , 4-byte literal
7204   // + 4 because better safe than sorry.
7205   const int fudge = 12 + 4;
7206   if (disp < 0) {
7207     disp -= fudge;
7208   } else {
7209     disp += fudge;
7210   }
7211   return is_simm32(disp);
7212 }
7213 
7214 // Check if the polling page is not reachable from the code cache using rip-relative
7215 // addressing.
7216 bool Assembler::is_polling_page_far() {
7217   intptr_t addr = (intptr_t)os::get_polling_page();
7218   return ForceUnreachable ||
7219          !is_simm32(addr - (intptr_t)CodeCache::low_bound()) ||
7220          !is_simm32(addr - (intptr_t)CodeCache::high_bound());
7221 }
7222 
7223 void Assembler::emit_data64(jlong data,
7224                             relocInfo::relocType rtype,
7225                             int format) {
7226   if (rtype == relocInfo::none) {
7227     emit_int64(data);
7228   } else {
7229     emit_data64(data, Relocation::spec_simple(rtype), format);
7230   }
7231 }
7232 
7233 void Assembler::emit_data64(jlong data,
7234                             RelocationHolder const& rspec,
7235                             int format) {
7236   assert(imm_operand == 0, "default format must be immediate in this file");
7237   assert(imm_operand == format, "must be immediate");
7238   assert(inst_mark() != NULL, "must be inside InstructionMark");
7239   // Do not use AbstractAssembler::relocate, which is not intended for
7240   // embedded words.  Instead, relocate to the enclosing instruction.
7241   code_section()->relocate(inst_mark(), rspec, format);
7242 #ifdef ASSERT
7243   check_relocation(rspec, format);
7244 #endif
7245   emit_int64(data);
7246 }
7247 
7248 int Assembler::prefix_and_encode(int reg_enc, bool byteinst) {
7249   if (reg_enc >= 8) {
7250     prefix(REX_B);
7251     reg_enc -= 8;
7252   } else if (byteinst && reg_enc >= 4) {
7253     prefix(REX);
7254   }
7255   return reg_enc;
7256 }
7257 
7258 int Assembler::prefixq_and_encode(int reg_enc) {
7259   if (reg_enc < 8) {
7260     prefix(REX_W);
7261   } else {
7262     prefix(REX_WB);
7263     reg_enc -= 8;
7264   }
7265   return reg_enc;
7266 }
7267 
7268 int Assembler::prefix_and_encode(int dst_enc, bool dst_is_byte, int src_enc, bool src_is_byte) {
7269   if (dst_enc < 8) {
7270     if (src_enc >= 8) {
7271       prefix(REX_B);
7272       src_enc -= 8;
7273     } else if ((src_is_byte && src_enc >= 4) || (dst_is_byte && dst_enc >= 4)) {
7274       prefix(REX);
7275     }
7276   } else {
7277     if (src_enc < 8) {
7278       prefix(REX_R);
7279     } else {
7280       prefix(REX_RB);
7281       src_enc -= 8;
7282     }
7283     dst_enc -= 8;
7284   }
7285   return dst_enc << 3 | src_enc;
7286 }
7287 
7288 int Assembler::prefixq_and_encode(int dst_enc, int src_enc) {
7289   if (dst_enc < 8) {
7290     if (src_enc < 8) {
7291       prefix(REX_W);
7292     } else {
7293       prefix(REX_WB);
7294       src_enc -= 8;
7295     }
7296   } else {
7297     if (src_enc < 8) {
7298       prefix(REX_WR);
7299     } else {
7300       prefix(REX_WRB);
7301       src_enc -= 8;
7302     }
7303     dst_enc -= 8;
7304   }
7305   return dst_enc << 3 | src_enc;
7306 }
7307 
7308 void Assembler::prefix(Register reg) {
7309   if (reg->encoding() >= 8) {
7310     prefix(REX_B);
7311   }
7312 }
7313 
7314 void Assembler::prefix(Register dst, Register src, Prefix p) {
7315   if (src->encoding() >= 8) {
7316     p = (Prefix)(p | REX_B);
7317   }
7318   if (dst->encoding() >= 8) {
7319     p = (Prefix)( p | REX_R);
7320   }
7321   if (p != Prefix_EMPTY) {
7322     // do not generate an empty prefix
7323     prefix(p);
7324   }
7325 }
7326 
7327 void Assembler::prefix(Register dst, Address adr, Prefix p) {
7328   if (adr.base_needs_rex()) {
7329     if (adr.index_needs_rex()) {
7330       assert(false, "prefix(Register dst, Address adr, Prefix p) does not support handling of an X");
7331     } else {
7332       prefix(REX_B);
7333     }
7334   } else {
7335     if (adr.index_needs_rex()) {
7336       assert(false, "prefix(Register dst, Address adr, Prefix p) does not support handling of an X");
7337     }
7338   }
7339   if (dst->encoding() >= 8) {
7340     p = (Prefix)(p | REX_R);
7341   }
7342   if (p != Prefix_EMPTY) {
7343     // do not generate an empty prefix
7344     prefix(p);
7345   }
7346 }
7347 
7348 void Assembler::prefix(Address adr) {
7349   if (adr.base_needs_rex()) {
7350     if (adr.index_needs_rex()) {
7351       prefix(REX_XB);
7352     } else {
7353       prefix(REX_B);
7354     }
7355   } else {
7356     if (adr.index_needs_rex()) {
7357       prefix(REX_X);
7358     }
7359   }
7360 }
7361 
7362 void Assembler::prefixq(Address adr) {
7363   if (adr.base_needs_rex()) {
7364     if (adr.index_needs_rex()) {
7365       prefix(REX_WXB);
7366     } else {
7367       prefix(REX_WB);
7368     }
7369   } else {
7370     if (adr.index_needs_rex()) {
7371       prefix(REX_WX);
7372     } else {
7373       prefix(REX_W);
7374     }
7375   }
7376 }
7377 
7378 
7379 void Assembler::prefix(Address adr, Register reg, bool byteinst) {
7380   if (reg->encoding() < 8) {
7381     if (adr.base_needs_rex()) {
7382       if (adr.index_needs_rex()) {
7383         prefix(REX_XB);
7384       } else {
7385         prefix(REX_B);
7386       }
7387     } else {
7388       if (adr.index_needs_rex()) {
7389         prefix(REX_X);
7390       } else if (byteinst && reg->encoding() >= 4 ) {
7391         prefix(REX);
7392       }
7393     }
7394   } else {
7395     if (adr.base_needs_rex()) {
7396       if (adr.index_needs_rex()) {
7397         prefix(REX_RXB);
7398       } else {
7399         prefix(REX_RB);
7400       }
7401     } else {
7402       if (adr.index_needs_rex()) {
7403         prefix(REX_RX);
7404       } else {
7405         prefix(REX_R);
7406       }
7407     }
7408   }
7409 }
7410 
7411 void Assembler::prefixq(Address adr, Register src) {
7412   if (src->encoding() < 8) {
7413     if (adr.base_needs_rex()) {
7414       if (adr.index_needs_rex()) {
7415         prefix(REX_WXB);
7416       } else {
7417         prefix(REX_WB);
7418       }
7419     } else {
7420       if (adr.index_needs_rex()) {
7421         prefix(REX_WX);
7422       } else {
7423         prefix(REX_W);
7424       }
7425     }
7426   } else {
7427     if (adr.base_needs_rex()) {
7428       if (adr.index_needs_rex()) {
7429         prefix(REX_WRXB);
7430       } else {
7431         prefix(REX_WRB);
7432       }
7433     } else {
7434       if (adr.index_needs_rex()) {
7435         prefix(REX_WRX);
7436       } else {
7437         prefix(REX_WR);
7438       }
7439     }
7440   }
7441 }
7442 
7443 void Assembler::prefix(Address adr, XMMRegister reg) {
7444   if (reg->encoding() < 8) {
7445     if (adr.base_needs_rex()) {
7446       if (adr.index_needs_rex()) {
7447         prefix(REX_XB);
7448       } else {
7449         prefix(REX_B);
7450       }
7451     } else {
7452       if (adr.index_needs_rex()) {
7453         prefix(REX_X);
7454       }
7455     }
7456   } else {
7457     if (adr.base_needs_rex()) {
7458       if (adr.index_needs_rex()) {
7459         prefix(REX_RXB);
7460       } else {
7461         prefix(REX_RB);
7462       }
7463     } else {
7464       if (adr.index_needs_rex()) {
7465         prefix(REX_RX);
7466       } else {
7467         prefix(REX_R);
7468       }
7469     }
7470   }
7471 }
7472 
7473 void Assembler::prefixq(Address adr, XMMRegister src) {
7474   if (src->encoding() < 8) {
7475     if (adr.base_needs_rex()) {
7476       if (adr.index_needs_rex()) {
7477         prefix(REX_WXB);
7478       } else {
7479         prefix(REX_WB);
7480       }
7481     } else {
7482       if (adr.index_needs_rex()) {
7483         prefix(REX_WX);
7484       } else {
7485         prefix(REX_W);
7486       }
7487     }
7488   } else {
7489     if (adr.base_needs_rex()) {
7490       if (adr.index_needs_rex()) {
7491         prefix(REX_WRXB);
7492       } else {
7493         prefix(REX_WRB);
7494       }
7495     } else {
7496       if (adr.index_needs_rex()) {
7497         prefix(REX_WRX);
7498       } else {
7499         prefix(REX_WR);
7500       }
7501     }
7502   }
7503 }
7504 
7505 void Assembler::adcq(Register dst, int32_t imm32) {
7506   (void) prefixq_and_encode(dst->encoding());
7507   emit_arith(0x81, 0xD0, dst, imm32);
7508 }
7509 
7510 void Assembler::adcq(Register dst, Address src) {
7511   InstructionMark im(this);
7512   prefixq(src, dst);
7513   emit_int8(0x13);
7514   emit_operand(dst, src);
7515 }
7516 
7517 void Assembler::adcq(Register dst, Register src) {
7518   (void) prefixq_and_encode(dst->encoding(), src->encoding());
7519   emit_arith(0x13, 0xC0, dst, src);
7520 }
7521 
7522 void Assembler::addq(Address dst, int32_t imm32) {
7523   InstructionMark im(this);
7524   prefixq(dst);
7525   emit_arith_operand(0x81, rax, dst,imm32);
7526 }
7527 
7528 void Assembler::addq(Address dst, Register src) {
7529   InstructionMark im(this);
7530   prefixq(dst, src);
7531   emit_int8(0x01);
7532   emit_operand(src, dst);
7533 }
7534 
7535 void Assembler::addq(Register dst, int32_t imm32) {
7536   (void) prefixq_and_encode(dst->encoding());
7537   emit_arith(0x81, 0xC0, dst, imm32);
7538 }
7539 
7540 void Assembler::addq(Register dst, Address src) {
7541   InstructionMark im(this);
7542   prefixq(src, dst);
7543   emit_int8(0x03);
7544   emit_operand(dst, src);
7545 }
7546 
7547 void Assembler::addq(Register dst, Register src) {
7548   (void) prefixq_and_encode(dst->encoding(), src->encoding());
7549   emit_arith(0x03, 0xC0, dst, src);
7550 }
7551 
7552 void Assembler::adcxq(Register dst, Register src) {
7553   //assert(VM_Version::supports_adx(), "adx instructions not supported");
7554   emit_int8((unsigned char)0x66);
7555   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
7556   emit_int8(0x0F);
7557   emit_int8(0x38);
7558   emit_int8((unsigned char)0xF6);
7559   emit_int8((unsigned char)(0xC0 | encode));
7560 }
7561 
7562 void Assembler::adoxq(Register dst, Register src) {
7563   //assert(VM_Version::supports_adx(), "adx instructions not supported");
7564   emit_int8((unsigned char)0xF3);
7565   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
7566   emit_int8(0x0F);
7567   emit_int8(0x38);
7568   emit_int8((unsigned char)0xF6);
7569   emit_int8((unsigned char)(0xC0 | encode));
7570 }
7571 
7572 void Assembler::andq(Address dst, int32_t imm32) {
7573   InstructionMark im(this);
7574   prefixq(dst);
7575   emit_int8((unsigned char)0x81);
7576   emit_operand(rsp, dst, 4);
7577   emit_int32(imm32);
7578 }
7579 
7580 void Assembler::andq(Register dst, int32_t imm32) {
7581   (void) prefixq_and_encode(dst->encoding());
7582   emit_arith(0x81, 0xE0, dst, imm32);
7583 }
7584 
7585 void Assembler::andq(Register dst, Address src) {
7586   InstructionMark im(this);
7587   prefixq(src, dst);
7588   emit_int8(0x23);
7589   emit_operand(dst, src);
7590 }
7591 
7592 void Assembler::andq(Register dst, Register src) {
7593   (void) prefixq_and_encode(dst->encoding(), src->encoding());
7594   emit_arith(0x23, 0xC0, dst, src);
7595 }
7596 
7597 void Assembler::andnq(Register dst, Register src1, Register src2) {
7598   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
7599   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
7600   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
7601   emit_int8((unsigned char)0xF2);
7602   emit_int8((unsigned char)(0xC0 | encode));
7603 }
7604 
7605 void Assembler::andnq(Register dst, Register src1, Address src2) {
7606   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
7607   InstructionMark im(this);
7608   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
7609   vex_prefix(src2, src1->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
7610   emit_int8((unsigned char)0xF2);
7611   emit_operand(dst, src2);
7612 }
7613 
7614 void Assembler::bsfq(Register dst, Register src) {
7615   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
7616   emit_int8(0x0F);
7617   emit_int8((unsigned char)0xBC);
7618   emit_int8((unsigned char)(0xC0 | encode));
7619 }
7620 
7621 void Assembler::bsrq(Register dst, Register src) {
7622   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
7623   emit_int8(0x0F);
7624   emit_int8((unsigned char)0xBD);
7625   emit_int8((unsigned char)(0xC0 | encode));
7626 }
7627 
7628 void Assembler::bswapq(Register reg) {
7629   int encode = prefixq_and_encode(reg->encoding());
7630   emit_int8(0x0F);
7631   emit_int8((unsigned char)(0xC8 | encode));
7632 }
7633 
7634 void Assembler::blsiq(Register dst, Register src) {
7635   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
7636   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
7637   int encode = vex_prefix_and_encode(rbx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
7638   emit_int8((unsigned char)0xF3);
7639   emit_int8((unsigned char)(0xC0 | encode));
7640 }
7641 
7642 void Assembler::blsiq(Register dst, Address src) {
7643   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
7644   InstructionMark im(this);
7645   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
7646   vex_prefix(src, dst->encoding(), rbx->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
7647   emit_int8((unsigned char)0xF3);
7648   emit_operand(rbx, src);
7649 }
7650 
7651 void Assembler::blsmskq(Register dst, Register src) {
7652   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
7653   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
7654   int encode = vex_prefix_and_encode(rdx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
7655   emit_int8((unsigned char)0xF3);
7656   emit_int8((unsigned char)(0xC0 | encode));
7657 }
7658 
7659 void Assembler::blsmskq(Register dst, Address src) {
7660   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
7661   InstructionMark im(this);
7662   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
7663   vex_prefix(src, dst->encoding(), rdx->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
7664   emit_int8((unsigned char)0xF3);
7665   emit_operand(rdx, src);
7666 }
7667 
7668 void Assembler::blsrq(Register dst, Register src) {
7669   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
7670   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
7671   int encode = vex_prefix_and_encode(rcx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
7672   emit_int8((unsigned char)0xF3);
7673   emit_int8((unsigned char)(0xC0 | encode));
7674 }
7675 
7676 void Assembler::blsrq(Register dst, Address src) {
7677   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
7678   InstructionMark im(this);
7679   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
7680   vex_prefix(src, dst->encoding(), rcx->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
7681   emit_int8((unsigned char)0xF3);
7682   emit_operand(rcx, src);
7683 }
7684 
7685 void Assembler::cdqq() {
7686   prefix(REX_W);
7687   emit_int8((unsigned char)0x99);
7688 }
7689 
7690 void Assembler::clflush(Address adr) {
7691   prefix(adr);
7692   emit_int8(0x0F);
7693   emit_int8((unsigned char)0xAE);
7694   emit_operand(rdi, adr);
7695 }
7696 
7697 void Assembler::cmovq(Condition cc, Register dst, Register src) {
7698   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
7699   emit_int8(0x0F);
7700   emit_int8(0x40 | cc);
7701   emit_int8((unsigned char)(0xC0 | encode));
7702 }
7703 
7704 void Assembler::cmovq(Condition cc, Register dst, Address src) {
7705   InstructionMark im(this);
7706   prefixq(src, dst);
7707   emit_int8(0x0F);
7708   emit_int8(0x40 | cc);
7709   emit_operand(dst, src);
7710 }
7711 
7712 void Assembler::cmpq(Address dst, int32_t imm32) {
7713   InstructionMark im(this);
7714   prefixq(dst);
7715   emit_int8((unsigned char)0x81);
7716   emit_operand(rdi, dst, 4);
7717   emit_int32(imm32);
7718 }
7719 
7720 void Assembler::cmpq(Register dst, int32_t imm32) {
7721   (void) prefixq_and_encode(dst->encoding());
7722   emit_arith(0x81, 0xF8, dst, imm32);
7723 }
7724 
7725 void Assembler::cmpq(Address dst, Register src) {
7726   InstructionMark im(this);
7727   prefixq(dst, src);
7728   emit_int8(0x3B);
7729   emit_operand(src, dst);
7730 }
7731 
7732 void Assembler::cmpq(Register dst, Register src) {
7733   (void) prefixq_and_encode(dst->encoding(), src->encoding());
7734   emit_arith(0x3B, 0xC0, dst, src);
7735 }
7736 
7737 void Assembler::cmpq(Register dst, Address  src) {
7738   InstructionMark im(this);
7739   prefixq(src, dst);
7740   emit_int8(0x3B);
7741   emit_operand(dst, src);
7742 }
7743 
7744 void Assembler::cmpxchgq(Register reg, Address adr) {
7745   InstructionMark im(this);
7746   prefixq(adr, reg);
7747   emit_int8(0x0F);
7748   emit_int8((unsigned char)0xB1);
7749   emit_operand(reg, adr);
7750 }
7751 
7752 void Assembler::cvtsi2sdq(XMMRegister dst, Register src) {
7753   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
7754   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
7755   int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
7756   emit_int8(0x2A);
7757   emit_int8((unsigned char)(0xC0 | encode));
7758 }
7759 
7760 void Assembler::cvtsi2sdq(XMMRegister dst, Address src) {
7761   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
7762   InstructionMark im(this);
7763   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
7764   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
7765   simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
7766   emit_int8(0x2A);
7767   emit_operand(dst, src);
7768 }
7769 
7770 void Assembler::cvtsi2ssq(XMMRegister dst, Address src) {
7771   NOT_LP64(assert(VM_Version::supports_sse(), ""));
7772   InstructionMark im(this);
7773   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
7774   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
7775   simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
7776   emit_int8(0x2A);
7777   emit_operand(dst, src);
7778 }
7779 
7780 void Assembler::cvttsd2siq(Register dst, XMMRegister src) {
7781   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
7782   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
7783   int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
7784   emit_int8(0x2C);
7785   emit_int8((unsigned char)(0xC0 | encode));
7786 }
7787 
7788 void Assembler::cvttss2siq(Register dst, XMMRegister src) {
7789   NOT_LP64(assert(VM_Version::supports_sse(), ""));
7790   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
7791   int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
7792   emit_int8(0x2C);
7793   emit_int8((unsigned char)(0xC0 | encode));
7794 }
7795 
7796 void Assembler::decl(Register dst) {
7797   // Don't use it directly. Use MacroAssembler::decrementl() instead.
7798   // Use two-byte form (one-byte form is a REX prefix in 64-bit mode)
7799   int encode = prefix_and_encode(dst->encoding());
7800   emit_int8((unsigned char)0xFF);
7801   emit_int8((unsigned char)(0xC8 | encode));
7802 }
7803 
7804 void Assembler::decq(Register dst) {
7805   // Don't use it directly. Use MacroAssembler::decrementq() instead.
7806   // Use two-byte form (one-byte from is a REX prefix in 64-bit mode)
7807   int encode = prefixq_and_encode(dst->encoding());
7808   emit_int8((unsigned char)0xFF);
7809   emit_int8(0xC8 | encode);
7810 }
7811 
7812 void Assembler::decq(Address dst) {
7813   // Don't use it directly. Use MacroAssembler::decrementq() instead.
7814   InstructionMark im(this);
7815   prefixq(dst);
7816   emit_int8((unsigned char)0xFF);
7817   emit_operand(rcx, dst);
7818 }
7819 
7820 void Assembler::fxrstor(Address src) {
7821   prefixq(src);
7822   emit_int8(0x0F);
7823   emit_int8((unsigned char)0xAE);
7824   emit_operand(as_Register(1), src);
7825 }
7826 
7827 void Assembler::xrstor(Address src) {
7828   prefixq(src);
7829   emit_int8(0x0F);
7830   emit_int8((unsigned char)0xAE);
7831   emit_operand(as_Register(5), src);
7832 }
7833 
7834 void Assembler::fxsave(Address dst) {
7835   prefixq(dst);
7836   emit_int8(0x0F);
7837   emit_int8((unsigned char)0xAE);
7838   emit_operand(as_Register(0), dst);
7839 }
7840 
7841 void Assembler::xsave(Address dst) {
7842   prefixq(dst);
7843   emit_int8(0x0F);
7844   emit_int8((unsigned char)0xAE);
7845   emit_operand(as_Register(4), dst);
7846 }
7847 
7848 void Assembler::idivq(Register src) {
7849   int encode = prefixq_and_encode(src->encoding());
7850   emit_int8((unsigned char)0xF7);
7851   emit_int8((unsigned char)(0xF8 | encode));
7852 }
7853 
7854 void Assembler::imulq(Register dst, Register src) {
7855   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
7856   emit_int8(0x0F);
7857   emit_int8((unsigned char)0xAF);
7858   emit_int8((unsigned char)(0xC0 | encode));
7859 }
7860 
7861 void Assembler::imulq(Register dst, Register src, int value) {
7862   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
7863   if (is8bit(value)) {
7864     emit_int8(0x6B);
7865     emit_int8((unsigned char)(0xC0 | encode));
7866     emit_int8(value & 0xFF);
7867   } else {
7868     emit_int8(0x69);
7869     emit_int8((unsigned char)(0xC0 | encode));
7870     emit_int32(value);
7871   }
7872 }
7873 
7874 void Assembler::imulq(Register dst, Address src) {
7875   InstructionMark im(this);
7876   prefixq(src, dst);
7877   emit_int8(0x0F);
7878   emit_int8((unsigned char) 0xAF);
7879   emit_operand(dst, src);
7880 }
7881 
7882 void Assembler::incl(Register dst) {
7883   // Don't use it directly. Use MacroAssembler::incrementl() instead.
7884   // Use two-byte form (one-byte from is a REX prefix in 64-bit mode)
7885   int encode = prefix_and_encode(dst->encoding());
7886   emit_int8((unsigned char)0xFF);
7887   emit_int8((unsigned char)(0xC0 | encode));
7888 }
7889 
7890 void Assembler::incq(Register dst) {
7891   // Don't use it directly. Use MacroAssembler::incrementq() instead.
7892   // Use two-byte form (one-byte from is a REX prefix in 64-bit mode)
7893   int encode = prefixq_and_encode(dst->encoding());
7894   emit_int8((unsigned char)0xFF);
7895   emit_int8((unsigned char)(0xC0 | encode));
7896 }
7897 
7898 void Assembler::incq(Address dst) {
7899   // Don't use it directly. Use MacroAssembler::incrementq() instead.
7900   InstructionMark im(this);
7901   prefixq(dst);
7902   emit_int8((unsigned char)0xFF);
7903   emit_operand(rax, dst);
7904 }
7905 
7906 void Assembler::lea(Register dst, Address src) {
7907   leaq(dst, src);
7908 }
7909 
7910 void Assembler::leaq(Register dst, Address src) {
7911   InstructionMark im(this);
7912   prefixq(src, dst);
7913   emit_int8((unsigned char)0x8D);
7914   emit_operand(dst, src);
7915 }
7916 
7917 void Assembler::mov64(Register dst, int64_t imm64) {
7918   InstructionMark im(this);
7919   int encode = prefixq_and_encode(dst->encoding());
7920   emit_int8((unsigned char)(0xB8 | encode));
7921   emit_int64(imm64);
7922 }
7923 
7924 void Assembler::mov_literal64(Register dst, intptr_t imm64, RelocationHolder const& rspec) {
7925   InstructionMark im(this);
7926   int encode = prefixq_and_encode(dst->encoding());
7927   emit_int8(0xB8 | encode);
7928   emit_data64(imm64, rspec);
7929 }
7930 
7931 void Assembler::mov_narrow_oop(Register dst, int32_t imm32, RelocationHolder const& rspec) {
7932   InstructionMark im(this);
7933   int encode = prefix_and_encode(dst->encoding());
7934   emit_int8((unsigned char)(0xB8 | encode));
7935   emit_data((int)imm32, rspec, narrow_oop_operand);
7936 }
7937 
7938 void Assembler::mov_narrow_oop(Address dst, int32_t imm32,  RelocationHolder const& rspec) {
7939   InstructionMark im(this);
7940   prefix(dst);
7941   emit_int8((unsigned char)0xC7);
7942   emit_operand(rax, dst, 4);
7943   emit_data((int)imm32, rspec, narrow_oop_operand);
7944 }
7945 
7946 void Assembler::cmp_narrow_oop(Register src1, int32_t imm32, RelocationHolder const& rspec) {
7947   InstructionMark im(this);
7948   int encode = prefix_and_encode(src1->encoding());
7949   emit_int8((unsigned char)0x81);
7950   emit_int8((unsigned char)(0xF8 | encode));
7951   emit_data((int)imm32, rspec, narrow_oop_operand);
7952 }
7953 
7954 void Assembler::cmp_narrow_oop(Address src1, int32_t imm32, RelocationHolder const& rspec) {
7955   InstructionMark im(this);
7956   prefix(src1);
7957   emit_int8((unsigned char)0x81);
7958   emit_operand(rax, src1, 4);
7959   emit_data((int)imm32, rspec, narrow_oop_operand);
7960 }
7961 
7962 void Assembler::lzcntq(Register dst, Register src) {
7963   assert(VM_Version::supports_lzcnt(), "encoding is treated as BSR");
7964   emit_int8((unsigned char)0xF3);
7965   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
7966   emit_int8(0x0F);
7967   emit_int8((unsigned char)0xBD);
7968   emit_int8((unsigned char)(0xC0 | encode));
7969 }
7970 
7971 void Assembler::movdq(XMMRegister dst, Register src) {
7972   // table D-1 says MMX/SSE2
7973   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
7974   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
7975   int encode = simd_prefix_and_encode(dst, xnoreg, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7976   emit_int8(0x6E);
7977   emit_int8((unsigned char)(0xC0 | encode));
7978 }
7979 
7980 void Assembler::movdq(Register dst, XMMRegister src) {
7981   // table D-1 says MMX/SSE2
7982   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
7983   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
7984   // swap src/dst to get correct prefix
7985   int encode = simd_prefix_and_encode(src, xnoreg, as_XMMRegister(dst->encoding()), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7986   emit_int8(0x7E);
7987   emit_int8((unsigned char)(0xC0 | encode));
7988 }
7989 
7990 void Assembler::movq(Register dst, Register src) {
7991   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
7992   emit_int8((unsigned char)0x8B);
7993   emit_int8((unsigned char)(0xC0 | encode));
7994 }
7995 
7996 void Assembler::movq(Register dst, Address src) {
7997   InstructionMark im(this);
7998   prefixq(src, dst);
7999   emit_int8((unsigned char)0x8B);
8000   emit_operand(dst, src);
8001 }
8002 
8003 void Assembler::movq(Address dst, Register src) {
8004   InstructionMark im(this);
8005   prefixq(dst, src);
8006   emit_int8((unsigned char)0x89);
8007   emit_operand(src, dst);
8008 }
8009 
8010 void Assembler::movsbq(Register dst, Address src) {
8011   InstructionMark im(this);
8012   prefixq(src, dst);
8013   emit_int8(0x0F);
8014   emit_int8((unsigned char)0xBE);
8015   emit_operand(dst, src);
8016 }
8017 
8018 void Assembler::movsbq(Register dst, Register src) {
8019   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
8020   emit_int8(0x0F);
8021   emit_int8((unsigned char)0xBE);
8022   emit_int8((unsigned char)(0xC0 | encode));
8023 }
8024 
8025 void Assembler::movslq(Register dst, int32_t imm32) {
8026   // dbx shows movslq(rcx, 3) as movq     $0x0000000049000000,(%rbx)
8027   // and movslq(r8, 3); as movl     $0x0000000048000000,(%rbx)
8028   // as a result we shouldn't use until tested at runtime...
8029   ShouldNotReachHere();
8030   InstructionMark im(this);
8031   int encode = prefixq_and_encode(dst->encoding());
8032   emit_int8((unsigned char)(0xC7 | encode));
8033   emit_int32(imm32);
8034 }
8035 
8036 void Assembler::movslq(Address dst, int32_t imm32) {
8037   assert(is_simm32(imm32), "lost bits");
8038   InstructionMark im(this);
8039   prefixq(dst);
8040   emit_int8((unsigned char)0xC7);
8041   emit_operand(rax, dst, 4);
8042   emit_int32(imm32);
8043 }
8044 
8045 void Assembler::movslq(Register dst, Address src) {
8046   InstructionMark im(this);
8047   prefixq(src, dst);
8048   emit_int8(0x63);
8049   emit_operand(dst, src);
8050 }
8051 
8052 void Assembler::movslq(Register dst, Register src) {
8053   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
8054   emit_int8(0x63);
8055   emit_int8((unsigned char)(0xC0 | encode));
8056 }
8057 
8058 void Assembler::movswq(Register dst, Address src) {
8059   InstructionMark im(this);
8060   prefixq(src, dst);
8061   emit_int8(0x0F);
8062   emit_int8((unsigned char)0xBF);
8063   emit_operand(dst, src);
8064 }
8065 
8066 void Assembler::movswq(Register dst, Register src) {
8067   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
8068   emit_int8((unsigned char)0x0F);
8069   emit_int8((unsigned char)0xBF);
8070   emit_int8((unsigned char)(0xC0 | encode));
8071 }
8072 
8073 void Assembler::movzbq(Register dst, Address src) {
8074   InstructionMark im(this);
8075   prefixq(src, dst);
8076   emit_int8((unsigned char)0x0F);
8077   emit_int8((unsigned char)0xB6);
8078   emit_operand(dst, src);
8079 }
8080 
8081 void Assembler::movzbq(Register dst, Register src) {
8082   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
8083   emit_int8(0x0F);
8084   emit_int8((unsigned char)0xB6);
8085   emit_int8(0xC0 | encode);
8086 }
8087 
8088 void Assembler::movzwq(Register dst, Address src) {
8089   InstructionMark im(this);
8090   prefixq(src, dst);
8091   emit_int8((unsigned char)0x0F);
8092   emit_int8((unsigned char)0xB7);
8093   emit_operand(dst, src);
8094 }
8095 
8096 void Assembler::movzwq(Register dst, Register src) {
8097   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
8098   emit_int8((unsigned char)0x0F);
8099   emit_int8((unsigned char)0xB7);
8100   emit_int8((unsigned char)(0xC0 | encode));
8101 }
8102 
8103 void Assembler::mulq(Address src) {
8104   InstructionMark im(this);
8105   prefixq(src);
8106   emit_int8((unsigned char)0xF7);
8107   emit_operand(rsp, src);
8108 }
8109 
8110 void Assembler::mulq(Register src) {
8111   int encode = prefixq_and_encode(src->encoding());
8112   emit_int8((unsigned char)0xF7);
8113   emit_int8((unsigned char)(0xE0 | encode));
8114 }
8115 
8116 void Assembler::mulxq(Register dst1, Register dst2, Register src) {
8117   assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
8118   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
8119   int encode = vex_prefix_and_encode(dst1->encoding(), dst2->encoding(), src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_38, &attributes);
8120   emit_int8((unsigned char)0xF6);
8121   emit_int8((unsigned char)(0xC0 | encode));
8122 }
8123 
8124 void Assembler::negq(Register dst) {
8125   int encode = prefixq_and_encode(dst->encoding());
8126   emit_int8((unsigned char)0xF7);
8127   emit_int8((unsigned char)(0xD8 | encode));
8128 }
8129 
8130 void Assembler::notq(Register dst) {
8131   int encode = prefixq_and_encode(dst->encoding());
8132   emit_int8((unsigned char)0xF7);
8133   emit_int8((unsigned char)(0xD0 | encode));
8134 }
8135 
8136 void Assembler::orq(Address dst, int32_t imm32) {
8137   InstructionMark im(this);
8138   prefixq(dst);
8139   emit_int8((unsigned char)0x81);
8140   emit_operand(rcx, dst, 4);
8141   emit_int32(imm32);
8142 }
8143 
8144 void Assembler::orq(Register dst, int32_t imm32) {
8145   (void) prefixq_and_encode(dst->encoding());
8146   emit_arith(0x81, 0xC8, dst, imm32);
8147 }
8148 
8149 void Assembler::orq(Register dst, Address src) {
8150   InstructionMark im(this);
8151   prefixq(src, dst);
8152   emit_int8(0x0B);
8153   emit_operand(dst, src);
8154 }
8155 
8156 void Assembler::orq(Register dst, Register src) {
8157   (void) prefixq_and_encode(dst->encoding(), src->encoding());
8158   emit_arith(0x0B, 0xC0, dst, src);
8159 }
8160 
8161 void Assembler::popa() { // 64bit
8162   movq(r15, Address(rsp, 0));
8163   movq(r14, Address(rsp, wordSize));
8164   movq(r13, Address(rsp, 2 * wordSize));
8165   movq(r12, Address(rsp, 3 * wordSize));
8166   movq(r11, Address(rsp, 4 * wordSize));
8167   movq(r10, Address(rsp, 5 * wordSize));
8168   movq(r9,  Address(rsp, 6 * wordSize));
8169   movq(r8,  Address(rsp, 7 * wordSize));
8170   movq(rdi, Address(rsp, 8 * wordSize));
8171   movq(rsi, Address(rsp, 9 * wordSize));
8172   movq(rbp, Address(rsp, 10 * wordSize));
8173   // skip rsp
8174   movq(rbx, Address(rsp, 12 * wordSize));
8175   movq(rdx, Address(rsp, 13 * wordSize));
8176   movq(rcx, Address(rsp, 14 * wordSize));
8177   movq(rax, Address(rsp, 15 * wordSize));
8178 
8179   addq(rsp, 16 * wordSize);
8180 }
8181 
8182 void Assembler::popcntq(Register dst, Address src) {
8183   assert(VM_Version::supports_popcnt(), "must support");
8184   InstructionMark im(this);
8185   emit_int8((unsigned char)0xF3);
8186   prefixq(src, dst);
8187   emit_int8((unsigned char)0x0F);
8188   emit_int8((unsigned char)0xB8);
8189   emit_operand(dst, src);
8190 }
8191 
8192 void Assembler::popcntq(Register dst, Register src) {
8193   assert(VM_Version::supports_popcnt(), "must support");
8194   emit_int8((unsigned char)0xF3);
8195   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
8196   emit_int8((unsigned char)0x0F);
8197   emit_int8((unsigned char)0xB8);
8198   emit_int8((unsigned char)(0xC0 | encode));
8199 }
8200 
8201 void Assembler::popq(Address dst) {
8202   InstructionMark im(this);
8203   prefixq(dst);
8204   emit_int8((unsigned char)0x8F);
8205   emit_operand(rax, dst);
8206 }
8207 
8208 void Assembler::pusha() { // 64bit
8209   // we have to store original rsp.  ABI says that 128 bytes
8210   // below rsp are local scratch.
8211   movq(Address(rsp, -5 * wordSize), rsp);
8212 
8213   subq(rsp, 16 * wordSize);
8214 
8215   movq(Address(rsp, 15 * wordSize), rax);
8216   movq(Address(rsp, 14 * wordSize), rcx);
8217   movq(Address(rsp, 13 * wordSize), rdx);
8218   movq(Address(rsp, 12 * wordSize), rbx);
8219   // skip rsp
8220   movq(Address(rsp, 10 * wordSize), rbp);
8221   movq(Address(rsp, 9 * wordSize), rsi);
8222   movq(Address(rsp, 8 * wordSize), rdi);
8223   movq(Address(rsp, 7 * wordSize), r8);
8224   movq(Address(rsp, 6 * wordSize), r9);
8225   movq(Address(rsp, 5 * wordSize), r10);
8226   movq(Address(rsp, 4 * wordSize), r11);
8227   movq(Address(rsp, 3 * wordSize), r12);
8228   movq(Address(rsp, 2 * wordSize), r13);
8229   movq(Address(rsp, wordSize), r14);
8230   movq(Address(rsp, 0), r15);
8231 }
8232 
8233 void Assembler::pushq(Address src) {
8234   InstructionMark im(this);
8235   prefixq(src);
8236   emit_int8((unsigned char)0xFF);
8237   emit_operand(rsi, src);
8238 }
8239 
8240 void Assembler::rclq(Register dst, int imm8) {
8241   assert(isShiftCount(imm8 >> 1), "illegal shift count");
8242   int encode = prefixq_and_encode(dst->encoding());
8243   if (imm8 == 1) {
8244     emit_int8((unsigned char)0xD1);
8245     emit_int8((unsigned char)(0xD0 | encode));
8246   } else {
8247     emit_int8((unsigned char)0xC1);
8248     emit_int8((unsigned char)(0xD0 | encode));
8249     emit_int8(imm8);
8250   }
8251 }
8252 
8253 void Assembler::rcrq(Register dst, int imm8) {
8254   assert(isShiftCount(imm8 >> 1), "illegal shift count");
8255   int encode = prefixq_and_encode(dst->encoding());
8256   if (imm8 == 1) {
8257     emit_int8((unsigned char)0xD1);
8258     emit_int8((unsigned char)(0xD8 | encode));
8259   } else {
8260     emit_int8((unsigned char)0xC1);
8261     emit_int8((unsigned char)(0xD8 | encode));
8262     emit_int8(imm8);
8263   }
8264 }
8265 
8266 void Assembler::rorq(Register dst, int imm8) {
8267   assert(isShiftCount(imm8 >> 1), "illegal shift count");
8268   int encode = prefixq_and_encode(dst->encoding());
8269   if (imm8 == 1) {
8270     emit_int8((unsigned char)0xD1);
8271     emit_int8((unsigned char)(0xC8 | encode));
8272   } else {
8273     emit_int8((unsigned char)0xC1);
8274     emit_int8((unsigned char)(0xc8 | encode));
8275     emit_int8(imm8);
8276   }
8277 }
8278 
8279 void Assembler::rorxq(Register dst, Register src, int imm8) {
8280   assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
8281   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
8282   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_3A, &attributes);
8283   emit_int8((unsigned char)0xF0);
8284   emit_int8((unsigned char)(0xC0 | encode));
8285   emit_int8(imm8);
8286 }
8287 
8288 void Assembler::rorxd(Register dst, Register src, int imm8) {
8289   assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
8290   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
8291   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_3A, &attributes);
8292   emit_int8((unsigned char)0xF0);
8293   emit_int8((unsigned char)(0xC0 | encode));
8294   emit_int8(imm8);
8295 }
8296 
8297 void Assembler::sarq(Register dst, int imm8) {
8298   assert(isShiftCount(imm8 >> 1), "illegal shift count");
8299   int encode = prefixq_and_encode(dst->encoding());
8300   if (imm8 == 1) {
8301     emit_int8((unsigned char)0xD1);
8302     emit_int8((unsigned char)(0xF8 | encode));
8303   } else {
8304     emit_int8((unsigned char)0xC1);
8305     emit_int8((unsigned char)(0xF8 | encode));
8306     emit_int8(imm8);
8307   }
8308 }
8309 
8310 void Assembler::sarq(Register dst) {
8311   int encode = prefixq_and_encode(dst->encoding());
8312   emit_int8((unsigned char)0xD3);
8313   emit_int8((unsigned char)(0xF8 | encode));
8314 }
8315 
8316 void Assembler::sbbq(Address dst, int32_t imm32) {
8317   InstructionMark im(this);
8318   prefixq(dst);
8319   emit_arith_operand(0x81, rbx, dst, imm32);
8320 }
8321 
8322 void Assembler::sbbq(Register dst, int32_t imm32) {
8323   (void) prefixq_and_encode(dst->encoding());
8324   emit_arith(0x81, 0xD8, dst, imm32);
8325 }
8326 
8327 void Assembler::sbbq(Register dst, Address src) {
8328   InstructionMark im(this);
8329   prefixq(src, dst);
8330   emit_int8(0x1B);
8331   emit_operand(dst, src);
8332 }
8333 
8334 void Assembler::sbbq(Register dst, Register src) {
8335   (void) prefixq_and_encode(dst->encoding(), src->encoding());
8336   emit_arith(0x1B, 0xC0, dst, src);
8337 }
8338 
8339 void Assembler::shlq(Register dst, int imm8) {
8340   assert(isShiftCount(imm8 >> 1), "illegal shift count");
8341   int encode = prefixq_and_encode(dst->encoding());
8342   if (imm8 == 1) {
8343     emit_int8((unsigned char)0xD1);
8344     emit_int8((unsigned char)(0xE0 | encode));
8345   } else {
8346     emit_int8((unsigned char)0xC1);
8347     emit_int8((unsigned char)(0xE0 | encode));
8348     emit_int8(imm8);
8349   }
8350 }
8351 
8352 void Assembler::shlq(Register dst) {
8353   int encode = prefixq_and_encode(dst->encoding());
8354   emit_int8((unsigned char)0xD3);
8355   emit_int8((unsigned char)(0xE0 | encode));
8356 }
8357 
8358 void Assembler::shrq(Register dst, int imm8) {
8359   assert(isShiftCount(imm8 >> 1), "illegal shift count");
8360   int encode = prefixq_and_encode(dst->encoding());
8361   emit_int8((unsigned char)0xC1);
8362   emit_int8((unsigned char)(0xE8 | encode));
8363   emit_int8(imm8);
8364 }
8365 
8366 void Assembler::shrq(Register dst) {
8367   int encode = prefixq_and_encode(dst->encoding());
8368   emit_int8((unsigned char)0xD3);
8369   emit_int8(0xE8 | encode);
8370 }
8371 
8372 void Assembler::subq(Address dst, int32_t imm32) {
8373   InstructionMark im(this);
8374   prefixq(dst);
8375   emit_arith_operand(0x81, rbp, dst, imm32);
8376 }
8377 
8378 void Assembler::subq(Address dst, Register src) {
8379   InstructionMark im(this);
8380   prefixq(dst, src);
8381   emit_int8(0x29);
8382   emit_operand(src, dst);
8383 }
8384 
8385 void Assembler::subq(Register dst, int32_t imm32) {
8386   (void) prefixq_and_encode(dst->encoding());
8387   emit_arith(0x81, 0xE8, dst, imm32);
8388 }
8389 
8390 // Force generation of a 4 byte immediate value even if it fits into 8bit
8391 void Assembler::subq_imm32(Register dst, int32_t imm32) {
8392   (void) prefixq_and_encode(dst->encoding());
8393   emit_arith_imm32(0x81, 0xE8, dst, imm32);
8394 }
8395 
8396 void Assembler::subq(Register dst, Address src) {
8397   InstructionMark im(this);
8398   prefixq(src, dst);
8399   emit_int8(0x2B);
8400   emit_operand(dst, src);
8401 }
8402 
8403 void Assembler::subq(Register dst, Register src) {
8404   (void) prefixq_and_encode(dst->encoding(), src->encoding());
8405   emit_arith(0x2B, 0xC0, dst, src);
8406 }
8407 
8408 void Assembler::testq(Register dst, int32_t imm32) {
8409   // not using emit_arith because test
8410   // doesn't support sign-extension of
8411   // 8bit operands
8412   int encode = dst->encoding();
8413   if (encode == 0) {
8414     prefix(REX_W);
8415     emit_int8((unsigned char)0xA9);
8416   } else {
8417     encode = prefixq_and_encode(encode);
8418     emit_int8((unsigned char)0xF7);
8419     emit_int8((unsigned char)(0xC0 | encode));
8420   }
8421   emit_int32(imm32);
8422 }
8423 
8424 void Assembler::testq(Register dst, Register src) {
8425   (void) prefixq_and_encode(dst->encoding(), src->encoding());
8426   emit_arith(0x85, 0xC0, dst, src);
8427 }
8428 
8429 void Assembler::xaddq(Address dst, Register src) {
8430   InstructionMark im(this);
8431   prefixq(dst, src);
8432   emit_int8(0x0F);
8433   emit_int8((unsigned char)0xC1);
8434   emit_operand(src, dst);
8435 }
8436 
8437 void Assembler::xchgq(Register dst, Address src) {
8438   InstructionMark im(this);
8439   prefixq(src, dst);
8440   emit_int8((unsigned char)0x87);
8441   emit_operand(dst, src);
8442 }
8443 
8444 void Assembler::xchgq(Register dst, Register src) {
8445   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
8446   emit_int8((unsigned char)0x87);
8447   emit_int8((unsigned char)(0xc0 | encode));
8448 }
8449 
8450 void Assembler::xorq(Register dst, Register src) {
8451   (void) prefixq_and_encode(dst->encoding(), src->encoding());
8452   emit_arith(0x33, 0xC0, dst, src);
8453 }
8454 
8455 void Assembler::xorq(Register dst, Address src) {
8456   InstructionMark im(this);
8457   prefixq(src, dst);
8458   emit_int8(0x33);
8459   emit_operand(dst, src);
8460 }
8461 
8462 #endif // !LP64