1 /*
   2  * Copyright (c) 1997, 2016, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #include "precompiled.hpp"
  26 #include "asm/assembler.hpp"
  27 #include "asm/assembler.inline.hpp"
  28 #include "gc/shared/cardTableModRefBS.hpp"
  29 #include "gc/shared/collectedHeap.inline.hpp"
  30 #include "interpreter/interpreter.hpp"
  31 #include "memory/resourceArea.hpp"
  32 #include "prims/methodHandles.hpp"
  33 #include "runtime/biasedLocking.hpp"
  34 #include "runtime/interfaceSupport.hpp"
  35 #include "runtime/objectMonitor.hpp"
  36 #include "runtime/os.hpp"
  37 #include "runtime/sharedRuntime.hpp"
  38 #include "runtime/stubRoutines.hpp"
  39 #include "utilities/macros.hpp"
  40 #if INCLUDE_ALL_GCS
  41 #include "gc/g1/g1CollectedHeap.inline.hpp"
  42 #include "gc/g1/g1SATBCardTableModRefBS.hpp"
  43 #include "gc/g1/heapRegion.hpp"
  44 #endif // INCLUDE_ALL_GCS
  45 
  46 #ifdef PRODUCT
  47 #define BLOCK_COMMENT(str) /* nothing */
  48 #define STOP(error) stop(error)
  49 #else
  50 #define BLOCK_COMMENT(str) block_comment(str)
  51 #define STOP(error) block_comment(error); stop(error)
  52 #endif
  53 
  54 #define BIND(label) bind(label); BLOCK_COMMENT(#label ":")
  55 // Implementation of AddressLiteral
  56 
  57 // A 2-D table for managing compressed displacement(disp8) on EVEX enabled platforms.
  58 unsigned char tuple_table[Assembler::EVEX_ETUP + 1][Assembler::AVX_512bit + 1] = {
  59   // -----------------Table 4.5 -------------------- //
  60   16, 32, 64,  // EVEX_FV(0)
  61   4,  4,  4,   // EVEX_FV(1) - with Evex.b
  62   16, 32, 64,  // EVEX_FV(2) - with Evex.w
  63   8,  8,  8,   // EVEX_FV(3) - with Evex.w and Evex.b
  64   8,  16, 32,  // EVEX_HV(0)
  65   4,  4,  4,   // EVEX_HV(1) - with Evex.b
  66   // -----------------Table 4.6 -------------------- //
  67   16, 32, 64,  // EVEX_FVM(0)
  68   1,  1,  1,   // EVEX_T1S(0)
  69   2,  2,  2,   // EVEX_T1S(1)
  70   4,  4,  4,   // EVEX_T1S(2)
  71   8,  8,  8,   // EVEX_T1S(3)
  72   4,  4,  4,   // EVEX_T1F(0)
  73   8,  8,  8,   // EVEX_T1F(1)
  74   8,  8,  8,   // EVEX_T2(0)
  75   0,  16, 16,  // EVEX_T2(1)
  76   0,  16, 16,  // EVEX_T4(0)
  77   0,  0,  32,  // EVEX_T4(1)
  78   0,  0,  32,  // EVEX_T8(0)
  79   8,  16, 32,  // EVEX_HVM(0)
  80   4,  8,  16,  // EVEX_QVM(0)
  81   2,  4,  8,   // EVEX_OVM(0)
  82   16, 16, 16,  // EVEX_M128(0)
  83   8,  32, 64,  // EVEX_DUP(0)
  84   0,  0,  0    // EVEX_NTUP
  85 };
  86 
  87 AddressLiteral::AddressLiteral(address target, relocInfo::relocType rtype) {
  88   _is_lval = false;
  89   _target = target;
  90   switch (rtype) {
  91   case relocInfo::oop_type:
  92   case relocInfo::metadata_type:
  93     // Oops are a special case. Normally they would be their own section
  94     // but in cases like icBuffer they are literals in the code stream that
  95     // we don't have a section for. We use none so that we get a literal address
  96     // which is always patchable.
  97     break;
  98   case relocInfo::external_word_type:
  99     _rspec = external_word_Relocation::spec(target);
 100     break;
 101   case relocInfo::internal_word_type:
 102     _rspec = internal_word_Relocation::spec(target);
 103     break;
 104   case relocInfo::opt_virtual_call_type:
 105     _rspec = opt_virtual_call_Relocation::spec();
 106     break;
 107   case relocInfo::static_call_type:
 108     _rspec = static_call_Relocation::spec();
 109     break;
 110   case relocInfo::runtime_call_type:
 111     _rspec = runtime_call_Relocation::spec();
 112     break;
 113   case relocInfo::poll_type:
 114   case relocInfo::poll_return_type:
 115     _rspec = Relocation::spec_simple(rtype);
 116     break;
 117   case relocInfo::none:
 118     break;
 119   default:
 120     ShouldNotReachHere();
 121     break;
 122   }
 123 }
 124 
 125 // Implementation of Address
 126 
 127 #ifdef _LP64
 128 
 129 Address Address::make_array(ArrayAddress adr) {
 130   // Not implementable on 64bit machines
 131   // Should have been handled higher up the call chain.
 132   ShouldNotReachHere();
 133   return Address();
 134 }
 135 
 136 // exceedingly dangerous constructor
 137 Address::Address(int disp, address loc, relocInfo::relocType rtype) {
 138   _base  = noreg;
 139   _index = noreg;
 140   _scale = no_scale;
 141   _disp  = disp;
 142   switch (rtype) {
 143     case relocInfo::external_word_type:
 144       _rspec = external_word_Relocation::spec(loc);
 145       break;
 146     case relocInfo::internal_word_type:
 147       _rspec = internal_word_Relocation::spec(loc);
 148       break;
 149     case relocInfo::runtime_call_type:
 150       // HMM
 151       _rspec = runtime_call_Relocation::spec();
 152       break;
 153     case relocInfo::poll_type:
 154     case relocInfo::poll_return_type:
 155       _rspec = Relocation::spec_simple(rtype);
 156       break;
 157     case relocInfo::none:
 158       break;
 159     default:
 160       ShouldNotReachHere();
 161   }
 162 }
 163 #else // LP64
 164 
 165 Address Address::make_array(ArrayAddress adr) {
 166   AddressLiteral base = adr.base();
 167   Address index = adr.index();
 168   assert(index._disp == 0, "must not have disp"); // maybe it can?
 169   Address array(index._base, index._index, index._scale, (intptr_t) base.target());
 170   array._rspec = base._rspec;
 171   return array;
 172 }
 173 
 174 // exceedingly dangerous constructor
 175 Address::Address(address loc, RelocationHolder spec) {
 176   _base  = noreg;
 177   _index = noreg;
 178   _scale = no_scale;
 179   _disp  = (intptr_t) loc;
 180   _rspec = spec;
 181 }
 182 
 183 #endif // _LP64
 184 
 185 
 186 
 187 // Convert the raw encoding form into the form expected by the constructor for
 188 // Address.  An index of 4 (rsp) corresponds to having no index, so convert
 189 // that to noreg for the Address constructor.
 190 Address Address::make_raw(int base, int index, int scale, int disp, relocInfo::relocType disp_reloc) {
 191   RelocationHolder rspec;
 192   if (disp_reloc != relocInfo::none) {
 193     rspec = Relocation::spec_simple(disp_reloc);
 194   }
 195   bool valid_index = index != rsp->encoding();
 196   if (valid_index) {
 197     Address madr(as_Register(base), as_Register(index), (Address::ScaleFactor)scale, in_ByteSize(disp));
 198     madr._rspec = rspec;
 199     return madr;
 200   } else {
 201     Address madr(as_Register(base), noreg, Address::no_scale, in_ByteSize(disp));
 202     madr._rspec = rspec;
 203     return madr;
 204   }
 205 }
 206 
 207 // Implementation of Assembler
 208 
 209 int AbstractAssembler::code_fill_byte() {
 210   return (u_char)'\xF4'; // hlt
 211 }
 212 
 213 // make this go away someday
 214 void Assembler::emit_data(jint data, relocInfo::relocType rtype, int format) {
 215   if (rtype == relocInfo::none)
 216     emit_int32(data);
 217   else
 218     emit_data(data, Relocation::spec_simple(rtype), format);
 219 }
 220 
 221 void Assembler::emit_data(jint data, RelocationHolder const& rspec, int format) {
 222   assert(imm_operand == 0, "default format must be immediate in this file");
 223   assert(inst_mark() != NULL, "must be inside InstructionMark");
 224   if (rspec.type() !=  relocInfo::none) {
 225     #ifdef ASSERT
 226       check_relocation(rspec, format);
 227     #endif
 228     // Do not use AbstractAssembler::relocate, which is not intended for
 229     // embedded words.  Instead, relocate to the enclosing instruction.
 230 
 231     // hack. call32 is too wide for mask so use disp32
 232     if (format == call32_operand)
 233       code_section()->relocate(inst_mark(), rspec, disp32_operand);
 234     else
 235       code_section()->relocate(inst_mark(), rspec, format);
 236   }
 237   emit_int32(data);
 238 }
 239 
 240 static int encode(Register r) {
 241   int enc = r->encoding();
 242   if (enc >= 8) {
 243     enc -= 8;
 244   }
 245   return enc;
 246 }
 247 
 248 void Assembler::emit_arith_b(int op1, int op2, Register dst, int imm8) {
 249   assert(dst->has_byte_register(), "must have byte register");
 250   assert(isByte(op1) && isByte(op2), "wrong opcode");
 251   assert(isByte(imm8), "not a byte");
 252   assert((op1 & 0x01) == 0, "should be 8bit operation");
 253   emit_int8(op1);
 254   emit_int8(op2 | encode(dst));
 255   emit_int8(imm8);
 256 }
 257 
 258 
 259 void Assembler::emit_arith(int op1, int op2, Register dst, int32_t imm32) {
 260   assert(isByte(op1) && isByte(op2), "wrong opcode");
 261   assert((op1 & 0x01) == 1, "should be 32bit operation");
 262   assert((op1 & 0x02) == 0, "sign-extension bit should not be set");
 263   if (is8bit(imm32)) {
 264     emit_int8(op1 | 0x02); // set sign bit
 265     emit_int8(op2 | encode(dst));
 266     emit_int8(imm32 & 0xFF);
 267   } else {
 268     emit_int8(op1);
 269     emit_int8(op2 | encode(dst));
 270     emit_int32(imm32);
 271   }
 272 }
 273 
 274 // Force generation of a 4 byte immediate value even if it fits into 8bit
 275 void Assembler::emit_arith_imm32(int op1, int op2, Register dst, int32_t imm32) {
 276   assert(isByte(op1) && isByte(op2), "wrong opcode");
 277   assert((op1 & 0x01) == 1, "should be 32bit operation");
 278   assert((op1 & 0x02) == 0, "sign-extension bit should not be set");
 279   emit_int8(op1);
 280   emit_int8(op2 | encode(dst));
 281   emit_int32(imm32);
 282 }
 283 
 284 // immediate-to-memory forms
 285 void Assembler::emit_arith_operand(int op1, Register rm, Address adr, int32_t imm32) {
 286   assert((op1 & 0x01) == 1, "should be 32bit operation");
 287   assert((op1 & 0x02) == 0, "sign-extension bit should not be set");
 288   if (is8bit(imm32)) {
 289     emit_int8(op1 | 0x02); // set sign bit
 290     emit_operand(rm, adr, 1);
 291     emit_int8(imm32 & 0xFF);
 292   } else {
 293     emit_int8(op1);
 294     emit_operand(rm, adr, 4);
 295     emit_int32(imm32);
 296   }
 297 }
 298 
 299 
 300 void Assembler::emit_arith(int op1, int op2, Register dst, Register src) {
 301   assert(isByte(op1) && isByte(op2), "wrong opcode");
 302   emit_int8(op1);
 303   emit_int8(op2 | encode(dst) << 3 | encode(src));
 304 }
 305 
 306 
 307 bool Assembler::query_compressed_disp_byte(int disp, bool is_evex_inst, int vector_len,
 308                                            int cur_tuple_type, int in_size_in_bits, int cur_encoding) {
 309   int mod_idx = 0;
 310   // We will test if the displacement fits the compressed format and if so
 311   // apply the compression to the displacment iff the result is8bit.
 312   if (VM_Version::supports_evex() && is_evex_inst) {
 313     switch (cur_tuple_type) {
 314     case EVEX_FV:
 315       if ((cur_encoding & VEX_W) == VEX_W) {
 316         mod_idx = ((cur_encoding & EVEX_Rb) == EVEX_Rb) ? 3 : 2;
 317       } else {
 318         mod_idx = ((cur_encoding & EVEX_Rb) == EVEX_Rb) ? 1 : 0;
 319       }
 320       break;
 321 
 322     case EVEX_HV:
 323       mod_idx = ((cur_encoding & EVEX_Rb) == EVEX_Rb) ? 1 : 0;
 324       break;
 325 
 326     case EVEX_FVM:
 327       break;
 328 
 329     case EVEX_T1S:
 330       switch (in_size_in_bits) {
 331       case EVEX_8bit:
 332         break;
 333 
 334       case EVEX_16bit:
 335         mod_idx = 1;
 336         break;
 337 
 338       case EVEX_32bit:
 339         mod_idx = 2;
 340         break;
 341 
 342       case EVEX_64bit:
 343         mod_idx = 3;
 344         break;
 345       }
 346       break;
 347 
 348     case EVEX_T1F:
 349     case EVEX_T2:
 350     case EVEX_T4:
 351       mod_idx = (in_size_in_bits == EVEX_64bit) ? 1 : 0;
 352       break;
 353 
 354     case EVEX_T8:
 355       break;
 356 
 357     case EVEX_HVM:
 358       break;
 359 
 360     case EVEX_QVM:
 361       break;
 362 
 363     case EVEX_OVM:
 364       break;
 365 
 366     case EVEX_M128:
 367       break;
 368 
 369     case EVEX_DUP:
 370       break;
 371 
 372     default:
 373       assert(0, "no valid evex tuple_table entry");
 374       break;
 375     }
 376 
 377     if (vector_len >= AVX_128bit && vector_len <= AVX_512bit) {
 378       int disp_factor = tuple_table[cur_tuple_type + mod_idx][vector_len];
 379       if ((disp % disp_factor) == 0) {
 380         int new_disp = disp / disp_factor;
 381         if ((-0x80 <= new_disp && new_disp < 0x80)) {
 382           disp = new_disp;
 383         }
 384       } else {
 385         return false;
 386       }
 387     }
 388   }
 389   return (-0x80 <= disp && disp < 0x80);
 390 }
 391 
 392 
 393 bool Assembler::emit_compressed_disp_byte(int &disp) {
 394   int mod_idx = 0;
 395   // We will test if the displacement fits the compressed format and if so
 396   // apply the compression to the displacment iff the result is8bit.
 397   if (VM_Version::supports_evex() && _attributes && _attributes->is_evex_instruction()) {
 398     int evex_encoding = _attributes->get_evex_encoding();
 399     int tuple_type = _attributes->get_tuple_type();
 400     switch (tuple_type) {
 401     case EVEX_FV:
 402       if ((evex_encoding & VEX_W) == VEX_W) {
 403         mod_idx = ((evex_encoding & EVEX_Rb) == EVEX_Rb) ? 3 : 2;
 404       } else {
 405         mod_idx = ((evex_encoding & EVEX_Rb) == EVEX_Rb) ? 1 : 0;
 406       }
 407       break;
 408 
 409     case EVEX_HV:
 410       mod_idx = ((evex_encoding & EVEX_Rb) == EVEX_Rb) ? 1 : 0;
 411       break;
 412 
 413     case EVEX_FVM:
 414       break;
 415 
 416     case EVEX_T1S:
 417       switch (_attributes->get_input_size()) {
 418       case EVEX_8bit:
 419         break;
 420 
 421       case EVEX_16bit:
 422         mod_idx = 1;
 423         break;
 424 
 425       case EVEX_32bit:
 426         mod_idx = 2;
 427         break;
 428 
 429       case EVEX_64bit:
 430         mod_idx = 3;
 431         break;
 432       }
 433       break;
 434 
 435     case EVEX_T1F:
 436     case EVEX_T2:
 437     case EVEX_T4:
 438       mod_idx = (_attributes->get_input_size() == EVEX_64bit) ? 1 : 0;
 439       break;
 440 
 441     case EVEX_T8:
 442       break;
 443 
 444     case EVEX_HVM:
 445       break;
 446 
 447     case EVEX_QVM:
 448       break;
 449 
 450     case EVEX_OVM:
 451       break;
 452 
 453     case EVEX_M128:
 454       break;
 455 
 456     case EVEX_DUP:
 457       break;
 458 
 459     default:
 460       assert(0, "no valid evex tuple_table entry");
 461       break;
 462     }
 463 
 464     int vector_len = _attributes->get_vector_len();
 465     if (vector_len >= AVX_128bit && vector_len <= AVX_512bit) {
 466       int disp_factor = tuple_table[tuple_type + mod_idx][vector_len];
 467       if ((disp % disp_factor) == 0) {
 468         int new_disp = disp / disp_factor;
 469         if (is8bit(new_disp)) {
 470           disp = new_disp;
 471         }
 472       } else {
 473         return false;
 474       }
 475     }
 476   }
 477   return is8bit(disp);
 478 }
 479 
 480 
 481 void Assembler::emit_operand(Register reg, Register base, Register index,
 482                              Address::ScaleFactor scale, int disp,
 483                              RelocationHolder const& rspec,
 484                              int rip_relative_correction) {
 485   relocInfo::relocType rtype = (relocInfo::relocType) rspec.type();
 486 
 487   // Encode the registers as needed in the fields they are used in
 488 
 489   int regenc = encode(reg) << 3;
 490   int indexenc = index->is_valid() ? encode(index) << 3 : 0;
 491   int baseenc = base->is_valid() ? encode(base) : 0;
 492 
 493   if (base->is_valid()) {
 494     if (index->is_valid()) {
 495       assert(scale != Address::no_scale, "inconsistent address");
 496       // [base + index*scale + disp]
 497       if (disp == 0 && rtype == relocInfo::none  &&
 498           base != rbp LP64_ONLY(&& base != r13)) {
 499         // [base + index*scale]
 500         // [00 reg 100][ss index base]
 501         assert(index != rsp, "illegal addressing mode");
 502         emit_int8(0x04 | regenc);
 503         emit_int8(scale << 6 | indexenc | baseenc);
 504       } else if (emit_compressed_disp_byte(disp) && rtype == relocInfo::none) {
 505         // [base + index*scale + imm8]
 506         // [01 reg 100][ss index base] imm8
 507         assert(index != rsp, "illegal addressing mode");
 508         emit_int8(0x44 | regenc);
 509         emit_int8(scale << 6 | indexenc | baseenc);
 510         emit_int8(disp & 0xFF);
 511       } else {
 512         // [base + index*scale + disp32]
 513         // [10 reg 100][ss index base] disp32
 514         assert(index != rsp, "illegal addressing mode");
 515         emit_int8(0x84 | regenc);
 516         emit_int8(scale << 6 | indexenc | baseenc);
 517         emit_data(disp, rspec, disp32_operand);
 518       }
 519     } else if (base == rsp LP64_ONLY(|| base == r12)) {
 520       // [rsp + disp]
 521       if (disp == 0 && rtype == relocInfo::none) {
 522         // [rsp]
 523         // [00 reg 100][00 100 100]
 524         emit_int8(0x04 | regenc);
 525         emit_int8(0x24);
 526       } else if (emit_compressed_disp_byte(disp) && rtype == relocInfo::none) {
 527         // [rsp + imm8]
 528         // [01 reg 100][00 100 100] disp8
 529         emit_int8(0x44 | regenc);
 530         emit_int8(0x24);
 531         emit_int8(disp & 0xFF);
 532       } else {
 533         // [rsp + imm32]
 534         // [10 reg 100][00 100 100] disp32
 535         emit_int8(0x84 | regenc);
 536         emit_int8(0x24);
 537         emit_data(disp, rspec, disp32_operand);
 538       }
 539     } else {
 540       // [base + disp]
 541       assert(base != rsp LP64_ONLY(&& base != r12), "illegal addressing mode");
 542       if (disp == 0 && rtype == relocInfo::none &&
 543           base != rbp LP64_ONLY(&& base != r13)) {
 544         // [base]
 545         // [00 reg base]
 546         emit_int8(0x00 | regenc | baseenc);
 547       } else if (emit_compressed_disp_byte(disp) && rtype == relocInfo::none) {
 548         // [base + disp8]
 549         // [01 reg base] disp8
 550         emit_int8(0x40 | regenc | baseenc);
 551         emit_int8(disp & 0xFF);
 552       } else {
 553         // [base + disp32]
 554         // [10 reg base] disp32
 555         emit_int8(0x80 | regenc | baseenc);
 556         emit_data(disp, rspec, disp32_operand);
 557       }
 558     }
 559   } else {
 560     if (index->is_valid()) {
 561       assert(scale != Address::no_scale, "inconsistent address");
 562       // [index*scale + disp]
 563       // [00 reg 100][ss index 101] disp32
 564       assert(index != rsp, "illegal addressing mode");
 565       emit_int8(0x04 | regenc);
 566       emit_int8(scale << 6 | indexenc | 0x05);
 567       emit_data(disp, rspec, disp32_operand);
 568     } else if (rtype != relocInfo::none ) {
 569       // [disp] (64bit) RIP-RELATIVE (32bit) abs
 570       // [00 000 101] disp32
 571 
 572       emit_int8(0x05 | regenc);
 573       // Note that the RIP-rel. correction applies to the generated
 574       // disp field, but _not_ to the target address in the rspec.
 575 
 576       // disp was created by converting the target address minus the pc
 577       // at the start of the instruction. That needs more correction here.
 578       // intptr_t disp = target - next_ip;
 579       assert(inst_mark() != NULL, "must be inside InstructionMark");
 580       address next_ip = pc() + sizeof(int32_t) + rip_relative_correction;
 581       int64_t adjusted = disp;
 582       // Do rip-rel adjustment for 64bit
 583       LP64_ONLY(adjusted -=  (next_ip - inst_mark()));
 584       assert(is_simm32(adjusted),
 585              "must be 32bit offset (RIP relative address)");
 586       emit_data((int32_t) adjusted, rspec, disp32_operand);
 587 
 588     } else {
 589       // 32bit never did this, did everything as the rip-rel/disp code above
 590       // [disp] ABSOLUTE
 591       // [00 reg 100][00 100 101] disp32
 592       emit_int8(0x04 | regenc);
 593       emit_int8(0x25);
 594       emit_data(disp, rspec, disp32_operand);
 595     }
 596   }
 597 }
 598 
 599 void Assembler::emit_operand(XMMRegister reg, Register base, Register index,
 600                              Address::ScaleFactor scale, int disp,
 601                              RelocationHolder const& rspec) {
 602   if (UseAVX > 2) {
 603     int xreg_enc = reg->encoding();
 604     if (xreg_enc > 15) {
 605       XMMRegister new_reg = as_XMMRegister(xreg_enc & 0xf);
 606       emit_operand((Register)new_reg, base, index, scale, disp, rspec);
 607       return;
 608     }
 609   }
 610   emit_operand((Register)reg, base, index, scale, disp, rspec);
 611 }
 612 
 613 // Secret local extension to Assembler::WhichOperand:
 614 #define end_pc_operand (_WhichOperand_limit)
 615 
 616 address Assembler::locate_operand(address inst, WhichOperand which) {
 617   // Decode the given instruction, and return the address of
 618   // an embedded 32-bit operand word.
 619 
 620   // If "which" is disp32_operand, selects the displacement portion
 621   // of an effective address specifier.
 622   // If "which" is imm64_operand, selects the trailing immediate constant.
 623   // If "which" is call32_operand, selects the displacement of a call or jump.
 624   // Caller is responsible for ensuring that there is such an operand,
 625   // and that it is 32/64 bits wide.
 626 
 627   // If "which" is end_pc_operand, find the end of the instruction.
 628 
 629   address ip = inst;
 630   bool is_64bit = false;
 631 
 632   debug_only(bool has_disp32 = false);
 633   int tail_size = 0; // other random bytes (#32, #16, etc.) at end of insn
 634 
 635   again_after_prefix:
 636   switch (0xFF & *ip++) {
 637 
 638   // These convenience macros generate groups of "case" labels for the switch.
 639 #define REP4(x) (x)+0: case (x)+1: case (x)+2: case (x)+3
 640 #define REP8(x) (x)+0: case (x)+1: case (x)+2: case (x)+3: \
 641              case (x)+4: case (x)+5: case (x)+6: case (x)+7
 642 #define REP16(x) REP8((x)+0): \
 643               case REP8((x)+8)
 644 
 645   case CS_segment:
 646   case SS_segment:
 647   case DS_segment:
 648   case ES_segment:
 649   case FS_segment:
 650   case GS_segment:
 651     // Seems dubious
 652     LP64_ONLY(assert(false, "shouldn't have that prefix"));
 653     assert(ip == inst+1, "only one prefix allowed");
 654     goto again_after_prefix;
 655 
 656   case 0x67:
 657   case REX:
 658   case REX_B:
 659   case REX_X:
 660   case REX_XB:
 661   case REX_R:
 662   case REX_RB:
 663   case REX_RX:
 664   case REX_RXB:
 665     NOT_LP64(assert(false, "64bit prefixes"));
 666     goto again_after_prefix;
 667 
 668   case REX_W:
 669   case REX_WB:
 670   case REX_WX:
 671   case REX_WXB:
 672   case REX_WR:
 673   case REX_WRB:
 674   case REX_WRX:
 675   case REX_WRXB:
 676     NOT_LP64(assert(false, "64bit prefixes"));
 677     is_64bit = true;
 678     goto again_after_prefix;
 679 
 680   case 0xFF: // pushq a; decl a; incl a; call a; jmp a
 681   case 0x88: // movb a, r
 682   case 0x89: // movl a, r
 683   case 0x8A: // movb r, a
 684   case 0x8B: // movl r, a
 685   case 0x8F: // popl a
 686     debug_only(has_disp32 = true);
 687     break;
 688 
 689   case 0x68: // pushq #32
 690     if (which == end_pc_operand) {
 691       return ip + 4;
 692     }
 693     assert(which == imm_operand && !is_64bit, "pushl has no disp32 or 64bit immediate");
 694     return ip;                  // not produced by emit_operand
 695 
 696   case 0x66: // movw ... (size prefix)
 697     again_after_size_prefix2:
 698     switch (0xFF & *ip++) {
 699     case REX:
 700     case REX_B:
 701     case REX_X:
 702     case REX_XB:
 703     case REX_R:
 704     case REX_RB:
 705     case REX_RX:
 706     case REX_RXB:
 707     case REX_W:
 708     case REX_WB:
 709     case REX_WX:
 710     case REX_WXB:
 711     case REX_WR:
 712     case REX_WRB:
 713     case REX_WRX:
 714     case REX_WRXB:
 715       NOT_LP64(assert(false, "64bit prefix found"));
 716       goto again_after_size_prefix2;
 717     case 0x8B: // movw r, a
 718     case 0x89: // movw a, r
 719       debug_only(has_disp32 = true);
 720       break;
 721     case 0xC7: // movw a, #16
 722       debug_only(has_disp32 = true);
 723       tail_size = 2;  // the imm16
 724       break;
 725     case 0x0F: // several SSE/SSE2 variants
 726       ip--;    // reparse the 0x0F
 727       goto again_after_prefix;
 728     default:
 729       ShouldNotReachHere();
 730     }
 731     break;
 732 
 733   case REP8(0xB8): // movl/q r, #32/#64(oop?)
 734     if (which == end_pc_operand)  return ip + (is_64bit ? 8 : 4);
 735     // these asserts are somewhat nonsensical
 736 #ifndef _LP64
 737     assert(which == imm_operand || which == disp32_operand,
 738            "which %d is_64_bit %d ip " INTPTR_FORMAT, which, is_64bit, p2i(ip));
 739 #else
 740     assert((which == call32_operand || which == imm_operand) && is_64bit ||
 741            which == narrow_oop_operand && !is_64bit,
 742            "which %d is_64_bit %d ip " INTPTR_FORMAT, which, is_64bit, p2i(ip));
 743 #endif // _LP64
 744     return ip;
 745 
 746   case 0x69: // imul r, a, #32
 747   case 0xC7: // movl a, #32(oop?)
 748     tail_size = 4;
 749     debug_only(has_disp32 = true); // has both kinds of operands!
 750     break;
 751 
 752   case 0x0F: // movx..., etc.
 753     switch (0xFF & *ip++) {
 754     case 0x3A: // pcmpestri
 755       tail_size = 1;
 756     case 0x38: // ptest, pmovzxbw
 757       ip++; // skip opcode
 758       debug_only(has_disp32 = true); // has both kinds of operands!
 759       break;
 760 
 761     case 0x70: // pshufd r, r/a, #8
 762       debug_only(has_disp32 = true); // has both kinds of operands!
 763     case 0x73: // psrldq r, #8
 764       tail_size = 1;
 765       break;
 766 
 767     case 0x12: // movlps
 768     case 0x28: // movaps
 769     case 0x2E: // ucomiss
 770     case 0x2F: // comiss
 771     case 0x54: // andps
 772     case 0x55: // andnps
 773     case 0x56: // orps
 774     case 0x57: // xorps
 775     case 0x58: // addpd
 776     case 0x59: // mulpd
 777     case 0x6E: // movd
 778     case 0x7E: // movd
 779     case 0xAE: // ldmxcsr, stmxcsr, fxrstor, fxsave, clflush
 780     case 0xFE: // paddd
 781       debug_only(has_disp32 = true);
 782       break;
 783 
 784     case 0xAD: // shrd r, a, %cl
 785     case 0xAF: // imul r, a
 786     case 0xBE: // movsbl r, a (movsxb)
 787     case 0xBF: // movswl r, a (movsxw)
 788     case 0xB6: // movzbl r, a (movzxb)
 789     case 0xB7: // movzwl r, a (movzxw)
 790     case REP16(0x40): // cmovl cc, r, a
 791     case 0xB0: // cmpxchgb
 792     case 0xB1: // cmpxchg
 793     case 0xC1: // xaddl
 794     case 0xC7: // cmpxchg8
 795     case REP16(0x90): // setcc a
 796       debug_only(has_disp32 = true);
 797       // fall out of the switch to decode the address
 798       break;
 799 
 800     case 0xC4: // pinsrw r, a, #8
 801       debug_only(has_disp32 = true);
 802     case 0xC5: // pextrw r, r, #8
 803       tail_size = 1;  // the imm8
 804       break;
 805 
 806     case 0xAC: // shrd r, a, #8
 807       debug_only(has_disp32 = true);
 808       tail_size = 1;  // the imm8
 809       break;
 810 
 811     case REP16(0x80): // jcc rdisp32
 812       if (which == end_pc_operand)  return ip + 4;
 813       assert(which == call32_operand, "jcc has no disp32 or imm");
 814       return ip;
 815     default:
 816       ShouldNotReachHere();
 817     }
 818     break;
 819 
 820   case 0x81: // addl a, #32; addl r, #32
 821     // also: orl, adcl, sbbl, andl, subl, xorl, cmpl
 822     // on 32bit in the case of cmpl, the imm might be an oop
 823     tail_size = 4;
 824     debug_only(has_disp32 = true); // has both kinds of operands!
 825     break;
 826 
 827   case 0x83: // addl a, #8; addl r, #8
 828     // also: orl, adcl, sbbl, andl, subl, xorl, cmpl
 829     debug_only(has_disp32 = true); // has both kinds of operands!
 830     tail_size = 1;
 831     break;
 832 
 833   case 0x9B:
 834     switch (0xFF & *ip++) {
 835     case 0xD9: // fnstcw a
 836       debug_only(has_disp32 = true);
 837       break;
 838     default:
 839       ShouldNotReachHere();
 840     }
 841     break;
 842 
 843   case REP4(0x00): // addb a, r; addl a, r; addb r, a; addl r, a
 844   case REP4(0x10): // adc...
 845   case REP4(0x20): // and...
 846   case REP4(0x30): // xor...
 847   case REP4(0x08): // or...
 848   case REP4(0x18): // sbb...
 849   case REP4(0x28): // sub...
 850   case 0xF7: // mull a
 851   case 0x8D: // lea r, a
 852   case 0x87: // xchg r, a
 853   case REP4(0x38): // cmp...
 854   case 0x85: // test r, a
 855     debug_only(has_disp32 = true); // has both kinds of operands!
 856     break;
 857 
 858   case 0xC1: // sal a, #8; sar a, #8; shl a, #8; shr a, #8
 859   case 0xC6: // movb a, #8
 860   case 0x80: // cmpb a, #8
 861   case 0x6B: // imul r, a, #8
 862     debug_only(has_disp32 = true); // has both kinds of operands!
 863     tail_size = 1; // the imm8
 864     break;
 865 
 866   case 0xC4: // VEX_3bytes
 867   case 0xC5: // VEX_2bytes
 868     assert((UseAVX > 0), "shouldn't have VEX prefix");
 869     assert(ip == inst+1, "no prefixes allowed");
 870     // C4 and C5 are also used as opcodes for PINSRW and PEXTRW instructions
 871     // but they have prefix 0x0F and processed when 0x0F processed above.
 872     //
 873     // In 32-bit mode the VEX first byte C4 and C5 alias onto LDS and LES
 874     // instructions (these instructions are not supported in 64-bit mode).
 875     // To distinguish them bits [7:6] are set in the VEX second byte since
 876     // ModRM byte can not be of the form 11xxxxxx in 32-bit mode. To set
 877     // those VEX bits REX and vvvv bits are inverted.
 878     //
 879     // Fortunately C2 doesn't generate these instructions so we don't need
 880     // to check for them in product version.
 881 
 882     // Check second byte
 883     NOT_LP64(assert((0xC0 & *ip) == 0xC0, "shouldn't have LDS and LES instructions"));
 884 
 885     int vex_opcode;
 886     // First byte
 887     if ((0xFF & *inst) == VEX_3bytes) {
 888       vex_opcode = VEX_OPCODE_MASK & *ip;
 889       ip++; // third byte
 890       is_64bit = ((VEX_W & *ip) == VEX_W);
 891     } else {
 892       vex_opcode = VEX_OPCODE_0F;
 893     }
 894     ip++; // opcode
 895     // To find the end of instruction (which == end_pc_operand).
 896     switch (vex_opcode) {
 897       case VEX_OPCODE_0F:
 898         switch (0xFF & *ip) {
 899         case 0x70: // pshufd r, r/a, #8
 900         case 0x71: // ps[rl|ra|ll]w r, #8
 901         case 0x72: // ps[rl|ra|ll]d r, #8
 902         case 0x73: // ps[rl|ra|ll]q r, #8
 903         case 0xC2: // cmp[ps|pd|ss|sd] r, r, r/a, #8
 904         case 0xC4: // pinsrw r, r, r/a, #8
 905         case 0xC5: // pextrw r/a, r, #8
 906         case 0xC6: // shufp[s|d] r, r, r/a, #8
 907           tail_size = 1;  // the imm8
 908           break;
 909         }
 910         break;
 911       case VEX_OPCODE_0F_3A:
 912         tail_size = 1;
 913         break;
 914     }
 915     ip++; // skip opcode
 916     debug_only(has_disp32 = true); // has both kinds of operands!
 917     break;
 918 
 919   case 0x62: // EVEX_4bytes
 920     assert((UseAVX > 0), "shouldn't have EVEX prefix");
 921     assert(ip == inst+1, "no prefixes allowed");
 922     // no EVEX collisions, all instructions that have 0x62 opcodes
 923     // have EVEX versions and are subopcodes of 0x66
 924     ip++; // skip P0 and exmaine W in P1
 925     is_64bit = ((VEX_W & *ip) == VEX_W);
 926     ip++; // move to P2
 927     ip++; // skip P2, move to opcode
 928     // To find the end of instruction (which == end_pc_operand).
 929     switch (0xFF & *ip) {
 930     case 0x22: // pinsrd r, r/a, #8
 931     case 0x61: // pcmpestri r, r/a, #8
 932     case 0x70: // pshufd r, r/a, #8
 933     case 0x73: // psrldq r, #8
 934       tail_size = 1;  // the imm8
 935       break;
 936     default:
 937       break;
 938     }
 939     ip++; // skip opcode
 940     debug_only(has_disp32 = true); // has both kinds of operands!
 941     break;
 942 
 943   case 0xD1: // sal a, 1; sar a, 1; shl a, 1; shr a, 1
 944   case 0xD3: // sal a, %cl; sar a, %cl; shl a, %cl; shr a, %cl
 945   case 0xD9: // fld_s a; fst_s a; fstp_s a; fldcw a
 946   case 0xDD: // fld_d a; fst_d a; fstp_d a
 947   case 0xDB: // fild_s a; fistp_s a; fld_x a; fstp_x a
 948   case 0xDF: // fild_d a; fistp_d a
 949   case 0xD8: // fadd_s a; fsubr_s a; fmul_s a; fdivr_s a; fcomp_s a
 950   case 0xDC: // fadd_d a; fsubr_d a; fmul_d a; fdivr_d a; fcomp_d a
 951   case 0xDE: // faddp_d a; fsubrp_d a; fmulp_d a; fdivrp_d a; fcompp_d a
 952     debug_only(has_disp32 = true);
 953     break;
 954 
 955   case 0xE8: // call rdisp32
 956   case 0xE9: // jmp  rdisp32
 957     if (which == end_pc_operand)  return ip + 4;
 958     assert(which == call32_operand, "call has no disp32 or imm");
 959     return ip;
 960 
 961   case 0xF0:                    // Lock
 962     assert(os::is_MP(), "only on MP");
 963     goto again_after_prefix;
 964 
 965   case 0xF3:                    // For SSE
 966   case 0xF2:                    // For SSE2
 967     switch (0xFF & *ip++) {
 968     case REX:
 969     case REX_B:
 970     case REX_X:
 971     case REX_XB:
 972     case REX_R:
 973     case REX_RB:
 974     case REX_RX:
 975     case REX_RXB:
 976     case REX_W:
 977     case REX_WB:
 978     case REX_WX:
 979     case REX_WXB:
 980     case REX_WR:
 981     case REX_WRB:
 982     case REX_WRX:
 983     case REX_WRXB:
 984       NOT_LP64(assert(false, "found 64bit prefix"));
 985       ip++;
 986     default:
 987       ip++;
 988     }
 989     debug_only(has_disp32 = true); // has both kinds of operands!
 990     break;
 991 
 992   default:
 993     ShouldNotReachHere();
 994 
 995 #undef REP8
 996 #undef REP16
 997   }
 998 
 999   assert(which != call32_operand, "instruction is not a call, jmp, or jcc");
1000 #ifdef _LP64
1001   assert(which != imm_operand, "instruction is not a movq reg, imm64");
1002 #else
1003   // assert(which != imm_operand || has_imm32, "instruction has no imm32 field");
1004   assert(which != imm_operand || has_disp32, "instruction has no imm32 field");
1005 #endif // LP64
1006   assert(which != disp32_operand || has_disp32, "instruction has no disp32 field");
1007 
1008   // parse the output of emit_operand
1009   int op2 = 0xFF & *ip++;
1010   int base = op2 & 0x07;
1011   int op3 = -1;
1012   const int b100 = 4;
1013   const int b101 = 5;
1014   if (base == b100 && (op2 >> 6) != 3) {
1015     op3 = 0xFF & *ip++;
1016     base = op3 & 0x07;   // refetch the base
1017   }
1018   // now ip points at the disp (if any)
1019 
1020   switch (op2 >> 6) {
1021   case 0:
1022     // [00 reg  100][ss index base]
1023     // [00 reg  100][00   100  esp]
1024     // [00 reg base]
1025     // [00 reg  100][ss index  101][disp32]
1026     // [00 reg  101]               [disp32]
1027 
1028     if (base == b101) {
1029       if (which == disp32_operand)
1030         return ip;              // caller wants the disp32
1031       ip += 4;                  // skip the disp32
1032     }
1033     break;
1034 
1035   case 1:
1036     // [01 reg  100][ss index base][disp8]
1037     // [01 reg  100][00   100  esp][disp8]
1038     // [01 reg base]               [disp8]
1039     ip += 1;                    // skip the disp8
1040     break;
1041 
1042   case 2:
1043     // [10 reg  100][ss index base][disp32]
1044     // [10 reg  100][00   100  esp][disp32]
1045     // [10 reg base]               [disp32]
1046     if (which == disp32_operand)
1047       return ip;                // caller wants the disp32
1048     ip += 4;                    // skip the disp32
1049     break;
1050 
1051   case 3:
1052     // [11 reg base]  (not a memory addressing mode)
1053     break;
1054   }
1055 
1056   if (which == end_pc_operand) {
1057     return ip + tail_size;
1058   }
1059 
1060 #ifdef _LP64
1061   assert(which == narrow_oop_operand && !is_64bit, "instruction is not a movl adr, imm32");
1062 #else
1063   assert(which == imm_operand, "instruction has only an imm field");
1064 #endif // LP64
1065   return ip;
1066 }
1067 
1068 address Assembler::locate_next_instruction(address inst) {
1069   // Secretly share code with locate_operand:
1070   return locate_operand(inst, end_pc_operand);
1071 }
1072 
1073 
1074 #ifdef ASSERT
1075 void Assembler::check_relocation(RelocationHolder const& rspec, int format) {
1076   address inst = inst_mark();
1077   assert(inst != NULL && inst < pc(), "must point to beginning of instruction");
1078   address opnd;
1079 
1080   Relocation* r = rspec.reloc();
1081   if (r->type() == relocInfo::none) {
1082     return;
1083   } else if (r->is_call() || format == call32_operand) {
1084     // assert(format == imm32_operand, "cannot specify a nonzero format");
1085     opnd = locate_operand(inst, call32_operand);
1086   } else if (r->is_data()) {
1087     assert(format == imm_operand || format == disp32_operand
1088            LP64_ONLY(|| format == narrow_oop_operand), "format ok");
1089     opnd = locate_operand(inst, (WhichOperand)format);
1090   } else {
1091     assert(format == imm_operand, "cannot specify a format");
1092     return;
1093   }
1094   assert(opnd == pc(), "must put operand where relocs can find it");
1095 }
1096 #endif // ASSERT
1097 
1098 void Assembler::emit_operand32(Register reg, Address adr) {
1099   assert(reg->encoding() < 8, "no extended registers");
1100   assert(!adr.base_needs_rex() && !adr.index_needs_rex(), "no extended registers");
1101   emit_operand(reg, adr._base, adr._index, adr._scale, adr._disp,
1102                adr._rspec);
1103 }
1104 
1105 void Assembler::emit_operand(Register reg, Address adr,
1106                              int rip_relative_correction) {
1107   emit_operand(reg, adr._base, adr._index, adr._scale, adr._disp,
1108                adr._rspec,
1109                rip_relative_correction);
1110 }
1111 
1112 void Assembler::emit_operand(XMMRegister reg, Address adr) {
1113   emit_operand(reg, adr._base, adr._index, adr._scale, adr._disp,
1114                adr._rspec);
1115 }
1116 
1117 // MMX operations
1118 void Assembler::emit_operand(MMXRegister reg, Address adr) {
1119   assert(!adr.base_needs_rex() && !adr.index_needs_rex(), "no extended registers");
1120   emit_operand((Register)reg, adr._base, adr._index, adr._scale, adr._disp, adr._rspec);
1121 }
1122 
1123 // work around gcc (3.2.1-7a) bug
1124 void Assembler::emit_operand(Address adr, MMXRegister reg) {
1125   assert(!adr.base_needs_rex() && !adr.index_needs_rex(), "no extended registers");
1126   emit_operand((Register)reg, adr._base, adr._index, adr._scale, adr._disp, adr._rspec);
1127 }
1128 
1129 
1130 void Assembler::emit_farith(int b1, int b2, int i) {
1131   assert(isByte(b1) && isByte(b2), "wrong opcode");
1132   assert(0 <= i &&  i < 8, "illegal stack offset");
1133   emit_int8(b1);
1134   emit_int8(b2 + i);
1135 }
1136 
1137 
1138 // Now the Assembler instructions (identical for 32/64 bits)
1139 
1140 void Assembler::adcl(Address dst, int32_t imm32) {
1141   InstructionMark im(this);
1142   prefix(dst);
1143   emit_arith_operand(0x81, rdx, dst, imm32);
1144 }
1145 
1146 void Assembler::adcl(Address dst, Register src) {
1147   InstructionMark im(this);
1148   prefix(dst, src);
1149   emit_int8(0x11);
1150   emit_operand(src, dst);
1151 }
1152 
1153 void Assembler::adcl(Register dst, int32_t imm32) {
1154   prefix(dst);
1155   emit_arith(0x81, 0xD0, dst, imm32);
1156 }
1157 
1158 void Assembler::adcl(Register dst, Address src) {
1159   InstructionMark im(this);
1160   prefix(src, dst);
1161   emit_int8(0x13);
1162   emit_operand(dst, src);
1163 }
1164 
1165 void Assembler::adcl(Register dst, Register src) {
1166   (void) prefix_and_encode(dst->encoding(), src->encoding());
1167   emit_arith(0x13, 0xC0, dst, src);
1168 }
1169 
1170 void Assembler::addl(Address dst, int32_t imm32) {
1171   InstructionMark im(this);
1172   prefix(dst);
1173   emit_arith_operand(0x81, rax, dst, imm32);
1174 }
1175 
1176 void Assembler::addl(Address dst, Register src) {
1177   InstructionMark im(this);
1178   prefix(dst, src);
1179   emit_int8(0x01);
1180   emit_operand(src, dst);
1181 }
1182 
1183 void Assembler::addl(Register dst, int32_t imm32) {
1184   prefix(dst);
1185   emit_arith(0x81, 0xC0, dst, imm32);
1186 }
1187 
1188 void Assembler::addl(Register dst, Address src) {
1189   InstructionMark im(this);
1190   prefix(src, dst);
1191   emit_int8(0x03);
1192   emit_operand(dst, src);
1193 }
1194 
1195 void Assembler::addl(Register dst, Register src) {
1196   (void) prefix_and_encode(dst->encoding(), src->encoding());
1197   emit_arith(0x03, 0xC0, dst, src);
1198 }
1199 
1200 void Assembler::addr_nop_4() {
1201   assert(UseAddressNop, "no CPU support");
1202   // 4 bytes: NOP DWORD PTR [EAX+0]
1203   emit_int8(0x0F);
1204   emit_int8(0x1F);
1205   emit_int8(0x40); // emit_rm(cbuf, 0x1, EAX_enc, EAX_enc);
1206   emit_int8(0);    // 8-bits offset (1 byte)
1207 }
1208 
1209 void Assembler::addr_nop_5() {
1210   assert(UseAddressNop, "no CPU support");
1211   // 5 bytes: NOP DWORD PTR [EAX+EAX*0+0] 8-bits offset
1212   emit_int8(0x0F);
1213   emit_int8(0x1F);
1214   emit_int8(0x44); // emit_rm(cbuf, 0x1, EAX_enc, 0x4);
1215   emit_int8(0x00); // emit_rm(cbuf, 0x0, EAX_enc, EAX_enc);
1216   emit_int8(0);    // 8-bits offset (1 byte)
1217 }
1218 
1219 void Assembler::addr_nop_7() {
1220   assert(UseAddressNop, "no CPU support");
1221   // 7 bytes: NOP DWORD PTR [EAX+0] 32-bits offset
1222   emit_int8(0x0F);
1223   emit_int8(0x1F);
1224   emit_int8((unsigned char)0x80);
1225                    // emit_rm(cbuf, 0x2, EAX_enc, EAX_enc);
1226   emit_int32(0);   // 32-bits offset (4 bytes)
1227 }
1228 
1229 void Assembler::addr_nop_8() {
1230   assert(UseAddressNop, "no CPU support");
1231   // 8 bytes: NOP DWORD PTR [EAX+EAX*0+0] 32-bits offset
1232   emit_int8(0x0F);
1233   emit_int8(0x1F);
1234   emit_int8((unsigned char)0x84);
1235                    // emit_rm(cbuf, 0x2, EAX_enc, 0x4);
1236   emit_int8(0x00); // emit_rm(cbuf, 0x0, EAX_enc, EAX_enc);
1237   emit_int32(0);   // 32-bits offset (4 bytes)
1238 }
1239 
1240 void Assembler::addsd(XMMRegister dst, XMMRegister src) {
1241   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1242   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
1243   attributes.set_rex_vex_w_reverted();
1244   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
1245   emit_int8(0x58);
1246   emit_int8((unsigned char)(0xC0 | encode));
1247 }
1248 
1249 void Assembler::addsd(XMMRegister dst, Address src) {
1250   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1251   InstructionMark im(this);
1252   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
1253   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
1254   attributes.set_rex_vex_w_reverted();
1255   simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
1256   emit_int8(0x58);
1257   emit_operand(dst, src);
1258 }
1259 
1260 void Assembler::addss(XMMRegister dst, XMMRegister src) {
1261   NOT_LP64(assert(VM_Version::supports_sse(), ""));
1262   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
1263   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
1264   emit_int8(0x58);
1265   emit_int8((unsigned char)(0xC0 | encode));
1266 }
1267 
1268 void Assembler::addss(XMMRegister dst, Address src) {
1269   NOT_LP64(assert(VM_Version::supports_sse(), ""));
1270   InstructionMark im(this);
1271   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
1272   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
1273   simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
1274   emit_int8(0x58);
1275   emit_operand(dst, src);
1276 }
1277 
1278 void Assembler::aesdec(XMMRegister dst, Address src) {
1279   assert(VM_Version::supports_aes(), "");
1280   InstructionMark im(this);
1281   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
1282   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
1283   emit_int8((unsigned char)0xDE);
1284   emit_operand(dst, src);
1285 }
1286 
1287 void Assembler::aesdec(XMMRegister dst, XMMRegister src) {
1288   assert(VM_Version::supports_aes(), "");
1289   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
1290   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
1291   emit_int8((unsigned char)0xDE);
1292   emit_int8(0xC0 | encode);
1293 }
1294 
1295 void Assembler::aesdeclast(XMMRegister dst, Address src) {
1296   assert(VM_Version::supports_aes(), "");
1297   InstructionMark im(this);
1298   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
1299   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
1300   emit_int8((unsigned char)0xDF);
1301   emit_operand(dst, src);
1302 }
1303 
1304 void Assembler::aesdeclast(XMMRegister dst, XMMRegister src) {
1305   assert(VM_Version::supports_aes(), "");
1306   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
1307   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
1308   emit_int8((unsigned char)0xDF);
1309   emit_int8((unsigned char)(0xC0 | encode));
1310 }
1311 
1312 void Assembler::aesenc(XMMRegister dst, Address src) {
1313   assert(VM_Version::supports_aes(), "");
1314   InstructionMark im(this);
1315   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
1316   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
1317   emit_int8((unsigned char)0xDC);
1318   emit_operand(dst, src);
1319 }
1320 
1321 void Assembler::aesenc(XMMRegister dst, XMMRegister src) {
1322   assert(VM_Version::supports_aes(), "");
1323   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
1324   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
1325   emit_int8((unsigned char)0xDC);
1326   emit_int8(0xC0 | encode);
1327 }
1328 
1329 void Assembler::aesenclast(XMMRegister dst, Address src) {
1330   assert(VM_Version::supports_aes(), "");
1331   InstructionMark im(this);
1332   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
1333   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
1334   emit_int8((unsigned char)0xDD);
1335   emit_operand(dst, src);
1336 }
1337 
1338 void Assembler::aesenclast(XMMRegister dst, XMMRegister src) {
1339   assert(VM_Version::supports_aes(), "");
1340   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
1341   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
1342   emit_int8((unsigned char)0xDD);
1343   emit_int8((unsigned char)(0xC0 | encode));
1344 }
1345 
1346 void Assembler::andl(Address dst, int32_t imm32) {
1347   InstructionMark im(this);
1348   prefix(dst);
1349   emit_int8((unsigned char)0x81);
1350   emit_operand(rsp, dst, 4);
1351   emit_int32(imm32);
1352 }
1353 
1354 void Assembler::andl(Register dst, int32_t imm32) {
1355   prefix(dst);
1356   emit_arith(0x81, 0xE0, dst, imm32);
1357 }
1358 
1359 void Assembler::andl(Register dst, Address src) {
1360   InstructionMark im(this);
1361   prefix(src, dst);
1362   emit_int8(0x23);
1363   emit_operand(dst, src);
1364 }
1365 
1366 void Assembler::andl(Register dst, Register src) {
1367   (void) prefix_and_encode(dst->encoding(), src->encoding());
1368   emit_arith(0x23, 0xC0, dst, src);
1369 }
1370 
1371 void Assembler::andnl(Register dst, Register src1, Register src2) {
1372   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1373   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
1374   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
1375   emit_int8((unsigned char)0xF2);
1376   emit_int8((unsigned char)(0xC0 | encode));
1377 }
1378 
1379 void Assembler::andnl(Register dst, Register src1, Address src2) {
1380   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1381   InstructionMark im(this);
1382   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
1383   vex_prefix(src2, src1->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
1384   emit_int8((unsigned char)0xF2);
1385   emit_operand(dst, src2);
1386 }
1387 
1388 void Assembler::bsfl(Register dst, Register src) {
1389   int encode = prefix_and_encode(dst->encoding(), src->encoding());
1390   emit_int8(0x0F);
1391   emit_int8((unsigned char)0xBC);
1392   emit_int8((unsigned char)(0xC0 | encode));
1393 }
1394 
1395 void Assembler::bsrl(Register dst, Register src) {
1396   int encode = prefix_and_encode(dst->encoding(), src->encoding());
1397   emit_int8(0x0F);
1398   emit_int8((unsigned char)0xBD);
1399   emit_int8((unsigned char)(0xC0 | encode));
1400 }
1401 
1402 void Assembler::bswapl(Register reg) { // bswap
1403   int encode = prefix_and_encode(reg->encoding());
1404   emit_int8(0x0F);
1405   emit_int8((unsigned char)(0xC8 | encode));
1406 }
1407 
1408 void Assembler::blsil(Register dst, Register src) {
1409   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1410   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
1411   int encode = vex_prefix_and_encode(rbx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
1412   emit_int8((unsigned char)0xF3);
1413   emit_int8((unsigned char)(0xC0 | encode));
1414 }
1415 
1416 void Assembler::blsil(Register dst, Address src) {
1417   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1418   InstructionMark im(this);
1419   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
1420   vex_prefix(src, dst->encoding(), rbx->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
1421   emit_int8((unsigned char)0xF3);
1422   emit_operand(rbx, src);
1423 }
1424 
1425 void Assembler::blsmskl(Register dst, Register src) {
1426   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1427   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
1428   int encode = vex_prefix_and_encode(rdx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
1429   emit_int8((unsigned char)0xF3);
1430   emit_int8((unsigned char)(0xC0 | encode));
1431 }
1432 
1433 void Assembler::blsmskl(Register dst, Address src) {
1434   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1435   InstructionMark im(this);
1436   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
1437   vex_prefix(src, dst->encoding(), rdx->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
1438   emit_int8((unsigned char)0xF3);
1439   emit_operand(rdx, src);
1440 }
1441 
1442 void Assembler::blsrl(Register dst, Register src) {
1443   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1444   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
1445   int encode = vex_prefix_and_encode(rcx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
1446   emit_int8((unsigned char)0xF3);
1447   emit_int8((unsigned char)(0xC0 | encode));
1448 }
1449 
1450 void Assembler::blsrl(Register dst, Address src) {
1451   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1452   InstructionMark im(this);
1453   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
1454   vex_prefix(src, dst->encoding(), rcx->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
1455   emit_int8((unsigned char)0xF3);
1456   emit_operand(rcx, src);
1457 }
1458 
1459 void Assembler::call(Label& L, relocInfo::relocType rtype) {
1460   // suspect disp32 is always good
1461   int operand = LP64_ONLY(disp32_operand) NOT_LP64(imm_operand);
1462 
1463   if (L.is_bound()) {
1464     const int long_size = 5;
1465     int offs = (int)( target(L) - pc() );
1466     assert(offs <= 0, "assembler error");
1467     InstructionMark im(this);
1468     // 1110 1000 #32-bit disp
1469     emit_int8((unsigned char)0xE8);
1470     emit_data(offs - long_size, rtype, operand);
1471   } else {
1472     InstructionMark im(this);
1473     // 1110 1000 #32-bit disp
1474     L.add_patch_at(code(), locator());
1475 
1476     emit_int8((unsigned char)0xE8);
1477     emit_data(int(0), rtype, operand);
1478   }
1479 }
1480 
1481 void Assembler::call(Register dst) {
1482   int encode = prefix_and_encode(dst->encoding());
1483   emit_int8((unsigned char)0xFF);
1484   emit_int8((unsigned char)(0xD0 | encode));
1485 }
1486 
1487 
1488 void Assembler::call(Address adr) {
1489   InstructionMark im(this);
1490   prefix(adr);
1491   emit_int8((unsigned char)0xFF);
1492   emit_operand(rdx, adr);
1493 }
1494 
1495 void Assembler::call_literal(address entry, RelocationHolder const& rspec) {
1496   assert(entry != NULL, "call most probably wrong");
1497   InstructionMark im(this);
1498   emit_int8((unsigned char)0xE8);
1499   intptr_t disp = entry - (pc() + sizeof(int32_t));
1500   assert(is_simm32(disp), "must be 32bit offset (call2)");
1501   // Technically, should use call32_operand, but this format is
1502   // implied by the fact that we're emitting a call instruction.
1503 
1504   int operand = LP64_ONLY(disp32_operand) NOT_LP64(call32_operand);
1505   emit_data((int) disp, rspec, operand);
1506 }
1507 
1508 void Assembler::cdql() {
1509   emit_int8((unsigned char)0x99);
1510 }
1511 
1512 void Assembler::cld() {
1513   emit_int8((unsigned char)0xFC);
1514 }
1515 
1516 void Assembler::cmovl(Condition cc, Register dst, Register src) {
1517   NOT_LP64(guarantee(VM_Version::supports_cmov(), "illegal instruction"));
1518   int encode = prefix_and_encode(dst->encoding(), src->encoding());
1519   emit_int8(0x0F);
1520   emit_int8(0x40 | cc);
1521   emit_int8((unsigned char)(0xC0 | encode));
1522 }
1523 
1524 
1525 void Assembler::cmovl(Condition cc, Register dst, Address src) {
1526   NOT_LP64(guarantee(VM_Version::supports_cmov(), "illegal instruction"));
1527   prefix(src, dst);
1528   emit_int8(0x0F);
1529   emit_int8(0x40 | cc);
1530   emit_operand(dst, src);
1531 }
1532 
1533 void Assembler::cmpb(Address dst, int imm8) {
1534   InstructionMark im(this);
1535   prefix(dst);
1536   emit_int8((unsigned char)0x80);
1537   emit_operand(rdi, dst, 1);
1538   emit_int8(imm8);
1539 }
1540 
1541 void Assembler::cmpl(Address dst, int32_t imm32) {
1542   InstructionMark im(this);
1543   prefix(dst);
1544   emit_int8((unsigned char)0x81);
1545   emit_operand(rdi, dst, 4);
1546   emit_int32(imm32);
1547 }
1548 
1549 void Assembler::cmpl(Register dst, int32_t imm32) {
1550   prefix(dst);
1551   emit_arith(0x81, 0xF8, dst, imm32);
1552 }
1553 
1554 void Assembler::cmpl(Register dst, Register src) {
1555   (void) prefix_and_encode(dst->encoding(), src->encoding());
1556   emit_arith(0x3B, 0xC0, dst, src);
1557 }
1558 
1559 void Assembler::cmpl(Register dst, Address  src) {
1560   InstructionMark im(this);
1561   prefix(src, dst);
1562   emit_int8((unsigned char)0x3B);
1563   emit_operand(dst, src);
1564 }
1565 
1566 void Assembler::cmpw(Address dst, int imm16) {
1567   InstructionMark im(this);
1568   assert(!dst.base_needs_rex() && !dst.index_needs_rex(), "no extended registers");
1569   emit_int8(0x66);
1570   emit_int8((unsigned char)0x81);
1571   emit_operand(rdi, dst, 2);
1572   emit_int16(imm16);
1573 }
1574 
1575 // The 32-bit cmpxchg compares the value at adr with the contents of rax,
1576 // and stores reg into adr if so; otherwise, the value at adr is loaded into rax,.
1577 // The ZF is set if the compared values were equal, and cleared otherwise.
1578 void Assembler::cmpxchgl(Register reg, Address adr) { // cmpxchg
1579   InstructionMark im(this);
1580   prefix(adr, reg);
1581   emit_int8(0x0F);
1582   emit_int8((unsigned char)0xB1);
1583   emit_operand(reg, adr);
1584 }
1585 
1586 // The 8-bit cmpxchg compares the value at adr with the contents of rax,
1587 // and stores reg into adr if so; otherwise, the value at adr is loaded into rax,.
1588 // The ZF is set if the compared values were equal, and cleared otherwise.
1589 void Assembler::cmpxchgb(Register reg, Address adr) { // cmpxchg
1590   InstructionMark im(this);
1591   prefix(adr, reg, true);
1592   emit_int8(0x0F);
1593   emit_int8((unsigned char)0xB0);
1594   emit_operand(reg, adr);
1595 }
1596 
1597 void Assembler::comisd(XMMRegister dst, Address src) {
1598   // NOTE: dbx seems to decode this as comiss even though the
1599   // 0x66 is there. Strangly ucomisd comes out correct
1600   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1601   InstructionMark im(this);
1602   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);;
1603   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
1604   attributes.set_rex_vex_w_reverted();
1605   simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
1606   emit_int8(0x2F);
1607   emit_operand(dst, src);
1608 }
1609 
1610 void Assembler::comisd(XMMRegister dst, XMMRegister src) {
1611   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1612   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1613   attributes.set_rex_vex_w_reverted();
1614   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
1615   emit_int8(0x2F);
1616   emit_int8((unsigned char)(0xC0 | encode));
1617 }
1618 
1619 void Assembler::comiss(XMMRegister dst, Address src) {
1620   NOT_LP64(assert(VM_Version::supports_sse(), ""));
1621   InstructionMark im(this);
1622   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1623   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
1624   simd_prefix(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
1625   emit_int8(0x2F);
1626   emit_operand(dst, src);
1627 }
1628 
1629 void Assembler::comiss(XMMRegister dst, XMMRegister src) {
1630   NOT_LP64(assert(VM_Version::supports_sse(), ""));
1631   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1632   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
1633   emit_int8(0x2F);
1634   emit_int8((unsigned char)(0xC0 | encode));
1635 }
1636 
1637 void Assembler::cpuid() {
1638   emit_int8(0x0F);
1639   emit_int8((unsigned char)0xA2);
1640 }
1641 
1642 // Opcode / Instruction                      Op /  En  64 - Bit Mode     Compat / Leg Mode Description                  Implemented
1643 // F2 0F 38 F0 / r       CRC32 r32, r / m8   RM        Valid             Valid             Accumulate CRC32 on r / m8.  v
1644 // F2 REX 0F 38 F0 / r   CRC32 r32, r / m8*  RM        Valid             N.E.              Accumulate CRC32 on r / m8.  -
1645 // F2 REX.W 0F 38 F0 / r CRC32 r64, r / m8   RM        Valid             N.E.              Accumulate CRC32 on r / m8.  -
1646 //
1647 // F2 0F 38 F1 / r       CRC32 r32, r / m16  RM        Valid             Valid             Accumulate CRC32 on r / m16. v
1648 //
1649 // F2 0F 38 F1 / r       CRC32 r32, r / m32  RM        Valid             Valid             Accumulate CRC32 on r / m32. v
1650 //
1651 // F2 REX.W 0F 38 F1 / r CRC32 r64, r / m64  RM        Valid             N.E.              Accumulate CRC32 on r / m64. v
1652 void Assembler::crc32(Register crc, Register v, int8_t sizeInBytes) {
1653   assert(VM_Version::supports_sse4_2(), "");
1654   int8_t w = 0x01;
1655   Prefix p = Prefix_EMPTY;
1656 
1657   emit_int8((int8_t)0xF2);
1658   switch (sizeInBytes) {
1659   case 1:
1660     w = 0;
1661     break;
1662   case 2:
1663   case 4:
1664     break;
1665   LP64_ONLY(case 8:)
1666     // This instruction is not valid in 32 bits
1667     // Note:
1668     // http://www.intel.com/content/dam/www/public/us/en/documents/manuals/64-ia-32-architectures-software-developer-instruction-set-reference-manual-325383.pdf
1669     //
1670     // Page B - 72   Vol. 2C says
1671     // qwreg2 to qwreg            1111 0010 : 0100 1R0B : 0000 1111 : 0011 1000 : 1111 0000 : 11 qwreg1 qwreg2
1672     // mem64 to qwreg             1111 0010 : 0100 1R0B : 0000 1111 : 0011 1000 : 1111 0000 : mod qwreg r / m
1673     //                                                                            F0!!!
1674     // while 3 - 208 Vol. 2A
1675     // F2 REX.W 0F 38 F1 / r       CRC32 r64, r / m64             RM         Valid      N.E.Accumulate CRC32 on r / m64.
1676     //
1677     // the 0 on a last bit is reserved for a different flavor of this instruction :
1678     // F2 REX.W 0F 38 F0 / r       CRC32 r64, r / m8              RM         Valid      N.E.Accumulate CRC32 on r / m8.
1679     p = REX_W;
1680     break;
1681   default:
1682     assert(0, "Unsupported value for a sizeInBytes argument");
1683     break;
1684   }
1685   LP64_ONLY(prefix(crc, v, p);)
1686   emit_int8((int8_t)0x0F);
1687   emit_int8(0x38);
1688   emit_int8((int8_t)(0xF0 | w));
1689   emit_int8(0xC0 | ((crc->encoding() & 0x7) << 3) | (v->encoding() & 7));
1690 }
1691 
1692 void Assembler::crc32(Register crc, Address adr, int8_t sizeInBytes) {
1693   assert(VM_Version::supports_sse4_2(), "");
1694   InstructionMark im(this);
1695   int8_t w = 0x01;
1696   Prefix p = Prefix_EMPTY;
1697 
1698   emit_int8((int8_t)0xF2);
1699   switch (sizeInBytes) {
1700   case 1:
1701     w = 0;
1702     break;
1703   case 2:
1704   case 4:
1705     break;
1706   LP64_ONLY(case 8:)
1707     // This instruction is not valid in 32 bits
1708     p = REX_W;
1709     break;
1710   default:
1711     assert(0, "Unsupported value for a sizeInBytes argument");
1712     break;
1713   }
1714   LP64_ONLY(prefix(crc, adr, p);)
1715   emit_int8((int8_t)0x0F);
1716   emit_int8(0x38);
1717   emit_int8((int8_t)(0xF0 | w));
1718   emit_operand(crc, adr);
1719 }
1720 
1721 void Assembler::cvtdq2pd(XMMRegister dst, XMMRegister src) {
1722   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1723   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
1724   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
1725   emit_int8((unsigned char)0xE6);
1726   emit_int8((unsigned char)(0xC0 | encode));
1727 }
1728 
1729 void Assembler::cvtdq2ps(XMMRegister dst, XMMRegister src) {
1730   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1731   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
1732   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
1733   emit_int8(0x5B);
1734   emit_int8((unsigned char)(0xC0 | encode));
1735 }
1736 
1737 void Assembler::cvtsd2ss(XMMRegister dst, XMMRegister src) {
1738   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1739   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
1740   attributes.set_rex_vex_w_reverted();
1741   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
1742   emit_int8(0x5A);
1743   emit_int8((unsigned char)(0xC0 | encode));
1744 }
1745 
1746 void Assembler::cvtsd2ss(XMMRegister dst, Address src) {
1747   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1748   InstructionMark im(this);
1749   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
1750   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
1751   attributes.set_rex_vex_w_reverted();
1752   simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
1753   emit_int8(0x5A);
1754   emit_operand(dst, src);
1755 }
1756 
1757 void Assembler::cvtsi2sdl(XMMRegister dst, Register src) {
1758   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1759   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1760   int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
1761   emit_int8(0x2A);
1762   emit_int8((unsigned char)(0xC0 | encode));
1763 }
1764 
1765 void Assembler::cvtsi2sdl(XMMRegister dst, Address src) {
1766   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1767   InstructionMark im(this);
1768   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1769   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
1770   simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
1771   emit_int8(0x2A);
1772   emit_operand(dst, src);
1773 }
1774 
1775 void Assembler::cvtsi2ssl(XMMRegister dst, Register src) {
1776   NOT_LP64(assert(VM_Version::supports_sse(), ""));
1777   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1778   int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
1779   emit_int8(0x2A);
1780   emit_int8((unsigned char)(0xC0 | encode));
1781 }
1782 
1783 void Assembler::cvtsi2ssl(XMMRegister dst, Address src) {
1784   NOT_LP64(assert(VM_Version::supports_sse(), ""));
1785   InstructionMark im(this);
1786   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1787   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
1788   simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
1789   emit_int8(0x2A);
1790   emit_operand(dst, src);
1791 }
1792 
1793 void Assembler::cvtsi2ssq(XMMRegister dst, Register src) {
1794   NOT_LP64(assert(VM_Version::supports_sse(), ""));
1795   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1796   int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
1797   emit_int8(0x2A);
1798   emit_int8((unsigned char)(0xC0 | encode));
1799 }
1800 
1801 void Assembler::cvtss2sd(XMMRegister dst, XMMRegister src) {
1802   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1803   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
1804   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
1805   emit_int8(0x5A);
1806   emit_int8((unsigned char)(0xC0 | encode));
1807 }
1808 
1809 void Assembler::cvtss2sd(XMMRegister dst, Address src) {
1810   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1811   InstructionMark im(this);
1812   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
1813   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
1814   simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
1815   emit_int8(0x5A);
1816   emit_operand(dst, src);
1817 }
1818 
1819 
1820 void Assembler::cvttsd2sil(Register dst, XMMRegister src) {
1821   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1822   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1823   int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
1824   emit_int8(0x2C);
1825   emit_int8((unsigned char)(0xC0 | encode));
1826 }
1827 
1828 void Assembler::cvttss2sil(Register dst, XMMRegister src) {
1829   NOT_LP64(assert(VM_Version::supports_sse(), ""));
1830   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1831   int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
1832   emit_int8(0x2C);
1833   emit_int8((unsigned char)(0xC0 | encode));
1834 }
1835 
1836 void Assembler::decl(Address dst) {
1837   // Don't use it directly. Use MacroAssembler::decrement() instead.
1838   InstructionMark im(this);
1839   prefix(dst);
1840   emit_int8((unsigned char)0xFF);
1841   emit_operand(rcx, dst);
1842 }
1843 
1844 void Assembler::divsd(XMMRegister dst, Address src) {
1845   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1846   InstructionMark im(this);
1847   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
1848   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
1849   attributes.set_rex_vex_w_reverted();
1850   simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
1851   emit_int8(0x5E);
1852   emit_operand(dst, src);
1853 }
1854 
1855 void Assembler::divsd(XMMRegister dst, XMMRegister src) {
1856   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1857   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
1858   attributes.set_rex_vex_w_reverted();
1859   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
1860   emit_int8(0x5E);
1861   emit_int8((unsigned char)(0xC0 | encode));
1862 }
1863 
1864 void Assembler::divss(XMMRegister dst, Address src) {
1865   NOT_LP64(assert(VM_Version::supports_sse(), ""));
1866   InstructionMark im(this);
1867   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
1868   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
1869   simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
1870   emit_int8(0x5E);
1871   emit_operand(dst, src);
1872 }
1873 
1874 void Assembler::divss(XMMRegister dst, XMMRegister src) {
1875   NOT_LP64(assert(VM_Version::supports_sse(), ""));
1876   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
1877   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
1878   emit_int8(0x5E);
1879   emit_int8((unsigned char)(0xC0 | encode));
1880 }
1881 
1882 void Assembler::emms() {
1883   NOT_LP64(assert(VM_Version::supports_mmx(), ""));
1884   emit_int8(0x0F);
1885   emit_int8(0x77);
1886 }
1887 
1888 void Assembler::hlt() {
1889   emit_int8((unsigned char)0xF4);
1890 }
1891 
1892 void Assembler::idivl(Register src) {
1893   int encode = prefix_and_encode(src->encoding());
1894   emit_int8((unsigned char)0xF7);
1895   emit_int8((unsigned char)(0xF8 | encode));
1896 }
1897 
1898 void Assembler::divl(Register src) { // Unsigned
1899   int encode = prefix_and_encode(src->encoding());
1900   emit_int8((unsigned char)0xF7);
1901   emit_int8((unsigned char)(0xF0 | encode));
1902 }
1903 
1904 void Assembler::imull(Register src) {
1905   int encode = prefix_and_encode(src->encoding());
1906   emit_int8((unsigned char)0xF7);
1907   emit_int8((unsigned char)(0xE8 | encode));
1908 }
1909 
1910 void Assembler::imull(Register dst, Register src) {
1911   int encode = prefix_and_encode(dst->encoding(), src->encoding());
1912   emit_int8(0x0F);
1913   emit_int8((unsigned char)0xAF);
1914   emit_int8((unsigned char)(0xC0 | encode));
1915 }
1916 
1917 
1918 void Assembler::imull(Register dst, Register src, int value) {
1919   int encode = prefix_and_encode(dst->encoding(), src->encoding());
1920   if (is8bit(value)) {
1921     emit_int8(0x6B);
1922     emit_int8((unsigned char)(0xC0 | encode));
1923     emit_int8(value & 0xFF);
1924   } else {
1925     emit_int8(0x69);
1926     emit_int8((unsigned char)(0xC0 | encode));
1927     emit_int32(value);
1928   }
1929 }
1930 
1931 void Assembler::imull(Register dst, Address src) {
1932   InstructionMark im(this);
1933   prefix(src, dst);
1934   emit_int8(0x0F);
1935   emit_int8((unsigned char) 0xAF);
1936   emit_operand(dst, src);
1937 }
1938 
1939 
1940 void Assembler::incl(Address dst) {
1941   // Don't use it directly. Use MacroAssembler::increment() instead.
1942   InstructionMark im(this);
1943   prefix(dst);
1944   emit_int8((unsigned char)0xFF);
1945   emit_operand(rax, dst);
1946 }
1947 
1948 void Assembler::jcc(Condition cc, Label& L, bool maybe_short) {
1949   InstructionMark im(this);
1950   assert((0 <= cc) && (cc < 16), "illegal cc");
1951   if (L.is_bound()) {
1952     address dst = target(L);
1953     assert(dst != NULL, "jcc most probably wrong");
1954 
1955     const int short_size = 2;
1956     const int long_size = 6;
1957     intptr_t offs = (intptr_t)dst - (intptr_t)pc();
1958     if (maybe_short && is8bit(offs - short_size)) {
1959       // 0111 tttn #8-bit disp
1960       emit_int8(0x70 | cc);
1961       emit_int8((offs - short_size) & 0xFF);
1962     } else {
1963       // 0000 1111 1000 tttn #32-bit disp
1964       assert(is_simm32(offs - long_size),
1965              "must be 32bit offset (call4)");
1966       emit_int8(0x0F);
1967       emit_int8((unsigned char)(0x80 | cc));
1968       emit_int32(offs - long_size);
1969     }
1970   } else {
1971     // Note: could eliminate cond. jumps to this jump if condition
1972     //       is the same however, seems to be rather unlikely case.
1973     // Note: use jccb() if label to be bound is very close to get
1974     //       an 8-bit displacement
1975     L.add_patch_at(code(), locator());
1976     emit_int8(0x0F);
1977     emit_int8((unsigned char)(0x80 | cc));
1978     emit_int32(0);
1979   }
1980 }
1981 
1982 void Assembler::jccb(Condition cc, Label& L) {
1983   if (L.is_bound()) {
1984     const int short_size = 2;
1985     address entry = target(L);
1986 #ifdef ASSERT
1987     intptr_t dist = (intptr_t)entry - ((intptr_t)pc() + short_size);
1988     intptr_t delta = short_branch_delta();
1989     if (delta != 0) {
1990       dist += (dist < 0 ? (-delta) :delta);
1991     }
1992     assert(is8bit(dist), "Dispacement too large for a short jmp");
1993 #endif
1994     intptr_t offs = (intptr_t)entry - (intptr_t)pc();
1995     // 0111 tttn #8-bit disp
1996     emit_int8(0x70 | cc);
1997     emit_int8((offs - short_size) & 0xFF);
1998   } else {
1999     InstructionMark im(this);
2000     L.add_patch_at(code(), locator());
2001     emit_int8(0x70 | cc);
2002     emit_int8(0);
2003   }
2004 }
2005 
2006 void Assembler::jmp(Address adr) {
2007   InstructionMark im(this);
2008   prefix(adr);
2009   emit_int8((unsigned char)0xFF);
2010   emit_operand(rsp, adr);
2011 }
2012 
2013 void Assembler::jmp(Label& L, bool maybe_short) {
2014   if (L.is_bound()) {
2015     address entry = target(L);
2016     assert(entry != NULL, "jmp most probably wrong");
2017     InstructionMark im(this);
2018     const int short_size = 2;
2019     const int long_size = 5;
2020     intptr_t offs = entry - pc();
2021     if (maybe_short && is8bit(offs - short_size)) {
2022       emit_int8((unsigned char)0xEB);
2023       emit_int8((offs - short_size) & 0xFF);
2024     } else {
2025       emit_int8((unsigned char)0xE9);
2026       emit_int32(offs - long_size);
2027     }
2028   } else {
2029     // By default, forward jumps are always 32-bit displacements, since
2030     // we can't yet know where the label will be bound.  If you're sure that
2031     // the forward jump will not run beyond 256 bytes, use jmpb to
2032     // force an 8-bit displacement.
2033     InstructionMark im(this);
2034     L.add_patch_at(code(), locator());
2035     emit_int8((unsigned char)0xE9);
2036     emit_int32(0);
2037   }
2038 }
2039 
2040 void Assembler::jmp(Register entry) {
2041   int encode = prefix_and_encode(entry->encoding());
2042   emit_int8((unsigned char)0xFF);
2043   emit_int8((unsigned char)(0xE0 | encode));
2044 }
2045 
2046 void Assembler::jmp_literal(address dest, RelocationHolder const& rspec) {
2047   InstructionMark im(this);
2048   emit_int8((unsigned char)0xE9);
2049   assert(dest != NULL, "must have a target");
2050   intptr_t disp = dest - (pc() + sizeof(int32_t));
2051   assert(is_simm32(disp), "must be 32bit offset (jmp)");
2052   emit_data(disp, rspec.reloc(), call32_operand);
2053 }
2054 
2055 void Assembler::jmpb(Label& L) {
2056   if (L.is_bound()) {
2057     const int short_size = 2;
2058     address entry = target(L);
2059     assert(entry != NULL, "jmp most probably wrong");
2060 #ifdef ASSERT
2061     intptr_t dist = (intptr_t)entry - ((intptr_t)pc() + short_size);
2062     intptr_t delta = short_branch_delta();
2063     if (delta != 0) {
2064       dist += (dist < 0 ? (-delta) :delta);
2065     }
2066     assert(is8bit(dist), "Dispacement too large for a short jmp");
2067 #endif
2068     intptr_t offs = entry - pc();
2069     emit_int8((unsigned char)0xEB);
2070     emit_int8((offs - short_size) & 0xFF);
2071   } else {
2072     InstructionMark im(this);
2073     L.add_patch_at(code(), locator());
2074     emit_int8((unsigned char)0xEB);
2075     emit_int8(0);
2076   }
2077 }
2078 
2079 void Assembler::ldmxcsr( Address src) {
2080   NOT_LP64(assert(VM_Version::supports_sse(), ""));
2081   InstructionMark im(this);
2082   prefix(src);
2083   emit_int8(0x0F);
2084   emit_int8((unsigned char)0xAE);
2085   emit_operand(as_Register(2), src);
2086 }
2087 
2088 void Assembler::leal(Register dst, Address src) {
2089   InstructionMark im(this);
2090 #ifdef _LP64
2091   emit_int8(0x67); // addr32
2092   prefix(src, dst);
2093 #endif // LP64
2094   emit_int8((unsigned char)0x8D);
2095   emit_operand(dst, src);
2096 }
2097 
2098 void Assembler::lfence() {
2099   emit_int8(0x0F);
2100   emit_int8((unsigned char)0xAE);
2101   emit_int8((unsigned char)0xE8);
2102 }
2103 
2104 void Assembler::lock() {
2105   emit_int8((unsigned char)0xF0);
2106 }
2107 
2108 void Assembler::lzcntl(Register dst, Register src) {
2109   assert(VM_Version::supports_lzcnt(), "encoding is treated as BSR");
2110   emit_int8((unsigned char)0xF3);
2111   int encode = prefix_and_encode(dst->encoding(), src->encoding());
2112   emit_int8(0x0F);
2113   emit_int8((unsigned char)0xBD);
2114   emit_int8((unsigned char)(0xC0 | encode));
2115 }
2116 
2117 // Emit mfence instruction
2118 void Assembler::mfence() {
2119   NOT_LP64(assert(VM_Version::supports_sse2(), "unsupported");)
2120   emit_int8(0x0F);
2121   emit_int8((unsigned char)0xAE);
2122   emit_int8((unsigned char)0xF0);
2123 }
2124 
2125 void Assembler::mov(Register dst, Register src) {
2126   LP64_ONLY(movq(dst, src)) NOT_LP64(movl(dst, src));
2127 }
2128 
2129 void Assembler::movapd(XMMRegister dst, XMMRegister src) {
2130   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2131   int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_128bit;
2132   InstructionAttr attributes(vector_len, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
2133   attributes.set_rex_vex_w_reverted();
2134   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2135   emit_int8(0x28);
2136   emit_int8((unsigned char)(0xC0 | encode));
2137 }
2138 
2139 void Assembler::movaps(XMMRegister dst, XMMRegister src) {
2140   NOT_LP64(assert(VM_Version::supports_sse(), ""));
2141   int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_128bit;
2142   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
2143   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2144   emit_int8(0x28);
2145   emit_int8((unsigned char)(0xC0 | encode));
2146 }
2147 
2148 void Assembler::movlhps(XMMRegister dst, XMMRegister src) {
2149   NOT_LP64(assert(VM_Version::supports_sse(), ""));
2150   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2151   int encode = simd_prefix_and_encode(dst, src, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2152   emit_int8(0x16);
2153   emit_int8((unsigned char)(0xC0 | encode));
2154 }
2155 
2156 void Assembler::movb(Register dst, Address src) {
2157   NOT_LP64(assert(dst->has_byte_register(), "must have byte register"));
2158   InstructionMark im(this);
2159   prefix(src, dst, true);
2160   emit_int8((unsigned char)0x8A);
2161   emit_operand(dst, src);
2162 }
2163 
2164 void Assembler::movddup(XMMRegister dst, XMMRegister src) {
2165   NOT_LP64(assert(VM_Version::supports_sse3(), ""));
2166   int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_128bit;
2167   InstructionAttr attributes(vector_len, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2168   attributes.set_rex_vex_w_reverted();
2169   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2170   emit_int8(0x12);
2171   emit_int8(0xC0 | encode);
2172 }
2173 
2174 void Assembler::kmovbl(KRegister dst, Register src) {
2175   assert(VM_Version::supports_avx512dq(), "");
2176   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2177   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2178   emit_int8((unsigned char)0x92);
2179   emit_int8((unsigned char)(0xC0 | encode));
2180 }
2181 
2182 void Assembler::kmovbl(Register dst, KRegister src) {
2183   assert(VM_Version::supports_avx512dq(), "");
2184   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2185   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2186   emit_int8((unsigned char)0x93);
2187   emit_int8((unsigned char)(0xC0 | encode));
2188 }
2189 
2190 void Assembler::kmovwl(KRegister dst, Register src) {
2191   assert(VM_Version::supports_evex(), "");
2192   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2193   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2194   emit_int8((unsigned char)0x92);
2195   emit_int8((unsigned char)(0xC0 | encode));
2196 }
2197 
2198 void Assembler::kmovwl(Register dst, KRegister src) {
2199   assert(VM_Version::supports_evex(), "");
2200   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2201   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2202   emit_int8((unsigned char)0x93);
2203   emit_int8((unsigned char)(0xC0 | encode));
2204 }
2205 
2206 void Assembler::kmovwl(KRegister dst, Address src) {
2207   assert(VM_Version::supports_evex(), "");
2208   InstructionMark im(this);
2209   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2210   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2211   emit_int8((unsigned char)0x90);
2212   emit_operand((Register)dst, src);
2213 }
2214 
2215 void Assembler::kmovdl(KRegister dst, Register src) {
2216   assert(VM_Version::supports_avx512bw(), "");
2217   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2218   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2219   emit_int8((unsigned char)0x92);
2220   emit_int8((unsigned char)(0xC0 | encode));
2221 }
2222 
2223 void Assembler::kmovdl(Register dst, KRegister src) {
2224   assert(VM_Version::supports_avx512bw(), "");
2225   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2226   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2227   emit_int8((unsigned char)0x93);
2228   emit_int8((unsigned char)(0xC0 | encode));
2229 }
2230 
2231 void Assembler::kmovql(KRegister dst, KRegister src) {
2232   assert(VM_Version::supports_avx512bw(), "");
2233   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2234   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2235   emit_int8((unsigned char)0x90);
2236   emit_int8((unsigned char)(0xC0 | encode));
2237 }
2238 
2239 void Assembler::kmovql(KRegister dst, Address src) {
2240   assert(VM_Version::supports_avx512bw(), "");
2241   InstructionMark im(this);
2242   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2243   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2244   emit_int8((unsigned char)0x90);
2245   emit_operand((Register)dst, src);
2246 }
2247 
2248 void Assembler::kmovql(Address dst, KRegister src) {
2249   assert(VM_Version::supports_avx512bw(), "");
2250   InstructionMark im(this);
2251   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2252   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2253   emit_int8((unsigned char)0x90);
2254   emit_operand((Register)src, dst);
2255 }
2256 
2257 void Assembler::kmovql(KRegister dst, Register src) {
2258   assert(VM_Version::supports_avx512bw(), "");
2259   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2260   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2261   emit_int8((unsigned char)0x92);
2262   emit_int8((unsigned char)(0xC0 | encode));
2263 }
2264 
2265 void Assembler::kmovql(Register dst, KRegister src) {
2266   assert(VM_Version::supports_avx512bw(), "");
2267   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2268   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2269   emit_int8((unsigned char)0x93);
2270   emit_int8((unsigned char)(0xC0 | encode));
2271 }
2272 
2273 // This instruction produces ZF or CF flags
2274 void Assembler::kortestbl(KRegister src1, KRegister src2) {
2275   assert(VM_Version::supports_avx512dq(), "");
2276   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2277   int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2278   emit_int8((unsigned char)0x98);
2279   emit_int8((unsigned char)(0xC0 | encode));
2280 }
2281 
2282 // This instruction produces ZF or CF flags
2283 void Assembler::kortestwl(KRegister src1, KRegister src2) {
2284   assert(VM_Version::supports_evex(), "");
2285   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2286   int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2287   emit_int8((unsigned char)0x98);
2288   emit_int8((unsigned char)(0xC0 | encode));
2289 }
2290 
2291 // This instruction produces ZF or CF flags
2292 void Assembler::kortestdl(KRegister src1, KRegister src2) {
2293   assert(VM_Version::supports_avx512bw(), "");
2294   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2295   int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2296   emit_int8((unsigned char)0x98);
2297   emit_int8((unsigned char)(0xC0 | encode));
2298 }
2299 
2300 // This instruction produces ZF or CF flags
2301 void Assembler::kortestql(KRegister src1, KRegister src2) {
2302   assert(VM_Version::supports_avx512bw(), "");
2303   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2304   int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2305   emit_int8((unsigned char)0x98);
2306   emit_int8((unsigned char)(0xC0 | encode));
2307 }
2308 
2309 void Assembler::movb(Address dst, int imm8) {
2310   InstructionMark im(this);
2311    prefix(dst);
2312   emit_int8((unsigned char)0xC6);
2313   emit_operand(rax, dst, 1);
2314   emit_int8(imm8);
2315 }
2316 
2317 
2318 void Assembler::movb(Address dst, Register src) {
2319   assert(src->has_byte_register(), "must have byte register");
2320   InstructionMark im(this);
2321   prefix(dst, src, true);
2322   emit_int8((unsigned char)0x88);
2323   emit_operand(src, dst);
2324 }
2325 
2326 void Assembler::movdl(XMMRegister dst, Register src) {
2327   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2328   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2329   int encode = simd_prefix_and_encode(dst, xnoreg, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2330   emit_int8(0x6E);
2331   emit_int8((unsigned char)(0xC0 | encode));
2332 }
2333 
2334 void Assembler::movdl(Register dst, XMMRegister src) {
2335   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2336   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2337   // swap src/dst to get correct prefix
2338   int encode = simd_prefix_and_encode(src, xnoreg, as_XMMRegister(dst->encoding()), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2339   emit_int8(0x7E);
2340   emit_int8((unsigned char)(0xC0 | encode));
2341 }
2342 
2343 void Assembler::movdl(XMMRegister dst, Address src) {
2344   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2345   InstructionMark im(this);
2346   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2347   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
2348   simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2349   emit_int8(0x6E);
2350   emit_operand(dst, src);
2351 }
2352 
2353 void Assembler::movdl(Address dst, XMMRegister src) {
2354   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2355   InstructionMark im(this);
2356   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2357   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
2358   simd_prefix(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2359   emit_int8(0x7E);
2360   emit_operand(src, dst);
2361 }
2362 
2363 void Assembler::movdqa(XMMRegister dst, XMMRegister src) {
2364   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2365   int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_128bit;
2366   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2367   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2368   emit_int8(0x6F);
2369   emit_int8((unsigned char)(0xC0 | encode));
2370 }
2371 
2372 void Assembler::movdqa(XMMRegister dst, Address src) {
2373   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2374   InstructionMark im(this);
2375   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2376   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2377   simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2378   emit_int8(0x6F);
2379   emit_operand(dst, src);
2380 }
2381 
2382 void Assembler::movdqu(XMMRegister dst, Address src) {
2383   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2384   InstructionMark im(this);
2385   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2386   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2387   simd_prefix(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2388   emit_int8(0x6F);
2389   emit_operand(dst, src);
2390 }
2391 
2392 void Assembler::movdqu(XMMRegister dst, XMMRegister src) {
2393   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2394   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2395   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2396   emit_int8(0x6F);
2397   emit_int8((unsigned char)(0xC0 | encode));
2398 }
2399 
2400 void Assembler::movdqu(Address dst, XMMRegister src) {
2401   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2402   InstructionMark im(this);
2403   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2404   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2405   simd_prefix(src, xnoreg, dst, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2406   emit_int8(0x7F);
2407   emit_operand(src, dst);
2408 }
2409 
2410 // Move Unaligned 256bit Vector
2411 void Assembler::vmovdqu(XMMRegister dst, XMMRegister src) {
2412   assert(UseAVX > 0, "");
2413   InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2414   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2415   emit_int8(0x6F);
2416   emit_int8((unsigned char)(0xC0 | encode));
2417 }
2418 
2419 void Assembler::vmovdqu(XMMRegister dst, Address src) {
2420   assert(UseAVX > 0, "");
2421   InstructionMark im(this);
2422   InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2423   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2424   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2425   emit_int8(0x6F);
2426   emit_operand(dst, src);
2427 }
2428 
2429 void Assembler::vmovdqu(Address dst, XMMRegister src) {
2430   assert(UseAVX > 0, "");
2431   InstructionMark im(this);
2432   InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2433   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2434   // swap src<->dst for encoding
2435   assert(src != xnoreg, "sanity");
2436   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2437   emit_int8(0x7F);
2438   emit_operand(src, dst);
2439 }
2440 
2441 // Move Unaligned EVEX enabled Vector (programmable : 8,16,32,64)
2442 void Assembler::evmovdqub(XMMRegister dst, XMMRegister src, int vector_len) {
2443   assert(VM_Version::supports_evex(), "");
2444   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
2445   attributes.set_is_evex_instruction();
2446   int prefix = (_legacy_mode_bw) ? VEX_SIMD_F2 : VEX_SIMD_F3;
2447   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), (Assembler::VexSimdPrefix)prefix, VEX_OPCODE_0F, &attributes);
2448   emit_int8(0x6F);
2449   emit_int8((unsigned char)(0xC0 | encode));
2450 }
2451 
2452 void Assembler::evmovdqub(XMMRegister dst, Address src, int vector_len) {
2453   assert(VM_Version::supports_evex(), "");
2454   InstructionMark im(this);
2455   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
2456   int prefix = (_legacy_mode_bw) ? VEX_SIMD_F2 : VEX_SIMD_F3;
2457   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2458   attributes.set_is_evex_instruction();
2459   vex_prefix(src, 0, dst->encoding(), (Assembler::VexSimdPrefix)prefix, VEX_OPCODE_0F, &attributes);
2460   emit_int8(0x6F);
2461   emit_operand(dst, src);
2462 }
2463 
2464 void Assembler::evmovdqub(Address dst, XMMRegister src, int vector_len) {
2465   assert(VM_Version::supports_evex(), "");
2466   assert(src != xnoreg, "sanity");
2467   InstructionMark im(this);
2468   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
2469   int prefix = (_legacy_mode_bw) ? VEX_SIMD_F2 : VEX_SIMD_F3;
2470   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2471   attributes.set_is_evex_instruction();
2472   vex_prefix(dst, 0, src->encoding(), (Assembler::VexSimdPrefix)prefix, VEX_OPCODE_0F, &attributes);
2473   emit_int8(0x7F);
2474   emit_operand(src, dst);
2475 }
2476 
2477 void Assembler::evmovdquw(XMMRegister dst, XMMRegister src, int vector_len) {
2478   assert(VM_Version::supports_evex(), "");
2479   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
2480   attributes.set_is_evex_instruction();
2481   int prefix = (_legacy_mode_bw) ? VEX_SIMD_F2 : VEX_SIMD_F3;
2482   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), (Assembler::VexSimdPrefix)prefix, VEX_OPCODE_0F, &attributes);
2483   emit_int8(0x6F);
2484   emit_int8((unsigned char)(0xC0 | encode));
2485 }
2486 
2487 void Assembler::evmovdquw(XMMRegister dst, Address src, int vector_len) {
2488   assert(VM_Version::supports_evex(), "");
2489   InstructionMark im(this);
2490   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
2491   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2492   attributes.set_is_evex_instruction();
2493   int prefix = (_legacy_mode_bw) ? VEX_SIMD_F2 : VEX_SIMD_F3;
2494   vex_prefix(src, 0, dst->encoding(), (Assembler::VexSimdPrefix)prefix, VEX_OPCODE_0F, &attributes);
2495   emit_int8(0x6F);
2496   emit_operand(dst, src);
2497 }
2498 
2499 void Assembler::evmovdquw(Address dst, XMMRegister src, int vector_len) {
2500   assert(VM_Version::supports_evex(), "");
2501   assert(src != xnoreg, "sanity");
2502   InstructionMark im(this);
2503   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
2504   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2505   attributes.set_is_evex_instruction();
2506   int prefix = (_legacy_mode_bw) ? VEX_SIMD_F2 : VEX_SIMD_F3;
2507   vex_prefix(dst, 0, src->encoding(), (Assembler::VexSimdPrefix)prefix, VEX_OPCODE_0F, &attributes);
2508   emit_int8(0x7F);
2509   emit_operand(src, dst);
2510 }
2511 
2512 void Assembler::evmovdqul(XMMRegister dst, XMMRegister src, int vector_len) {
2513   assert(VM_Version::supports_evex(), "");
2514   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2515   attributes.set_is_evex_instruction();
2516   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2517   emit_int8(0x6F);
2518   emit_int8((unsigned char)(0xC0 | encode));
2519 }
2520 
2521 void Assembler::evmovdqul(XMMRegister dst, Address src, int vector_len) {
2522   assert(VM_Version::supports_evex(), "");
2523   InstructionMark im(this);
2524   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false , /* uses_vl */ true);
2525   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2526   attributes.set_is_evex_instruction();
2527   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2528   emit_int8(0x6F);
2529   emit_operand(dst, src);
2530 }
2531 
2532 void Assembler::evmovdqul(Address dst, XMMRegister src, int vector_len) {
2533   assert(VM_Version::supports_evex(), "");
2534   assert(src != xnoreg, "sanity");
2535   InstructionMark im(this);
2536   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2537   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2538   attributes.set_is_evex_instruction();
2539   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2540   emit_int8(0x7F);
2541   emit_operand(src, dst);
2542 }
2543 
2544 void Assembler::evmovdquq(XMMRegister dst, XMMRegister src, int vector_len) {
2545   assert(VM_Version::supports_evex(), "");
2546   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2547   attributes.set_is_evex_instruction();
2548   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2549   emit_int8(0x6F);
2550   emit_int8((unsigned char)(0xC0 | encode));
2551 }
2552 
2553 void Assembler::evmovdquq(XMMRegister dst, Address src, int vector_len) {
2554   assert(VM_Version::supports_evex(), "");
2555   InstructionMark im(this);
2556   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2557   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2558   attributes.set_is_evex_instruction();
2559   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2560   emit_int8(0x6F);
2561   emit_operand(dst, src);
2562 }
2563 
2564 void Assembler::evmovdquq(Address dst, XMMRegister src, int vector_len) {
2565   assert(VM_Version::supports_evex(), "");
2566   assert(src != xnoreg, "sanity");
2567   InstructionMark im(this);
2568   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2569   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2570   attributes.set_is_evex_instruction();
2571   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2572   emit_int8(0x7F);
2573   emit_operand(src, dst);
2574 }
2575 
2576 // Uses zero extension on 64bit
2577 
2578 void Assembler::movl(Register dst, int32_t imm32) {
2579   int encode = prefix_and_encode(dst->encoding());
2580   emit_int8((unsigned char)(0xB8 | encode));
2581   emit_int32(imm32);
2582 }
2583 
2584 void Assembler::movl(Register dst, Register src) {
2585   int encode = prefix_and_encode(dst->encoding(), src->encoding());
2586   emit_int8((unsigned char)0x8B);
2587   emit_int8((unsigned char)(0xC0 | encode));
2588 }
2589 
2590 void Assembler::movl(Register dst, Address src) {
2591   InstructionMark im(this);
2592   prefix(src, dst);
2593   emit_int8((unsigned char)0x8B);
2594   emit_operand(dst, src);
2595 }
2596 
2597 void Assembler::movl(Address dst, int32_t imm32) {
2598   InstructionMark im(this);
2599   prefix(dst);
2600   emit_int8((unsigned char)0xC7);
2601   emit_operand(rax, dst, 4);
2602   emit_int32(imm32);
2603 }
2604 
2605 void Assembler::movl(Address dst, Register src) {
2606   InstructionMark im(this);
2607   prefix(dst, src);
2608   emit_int8((unsigned char)0x89);
2609   emit_operand(src, dst);
2610 }
2611 
2612 // New cpus require to use movsd and movss to avoid partial register stall
2613 // when loading from memory. But for old Opteron use movlpd instead of movsd.
2614 // The selection is done in MacroAssembler::movdbl() and movflt().
2615 void Assembler::movlpd(XMMRegister dst, Address src) {
2616   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2617   InstructionMark im(this);
2618   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2619   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
2620   attributes.set_rex_vex_w_reverted();
2621   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2622   emit_int8(0x12);
2623   emit_operand(dst, src);
2624 }
2625 
2626 void Assembler::movq( MMXRegister dst, Address src ) {
2627   assert( VM_Version::supports_mmx(), "" );
2628   emit_int8(0x0F);
2629   emit_int8(0x6F);
2630   emit_operand(dst, src);
2631 }
2632 
2633 void Assembler::movq( Address dst, MMXRegister src ) {
2634   assert( VM_Version::supports_mmx(), "" );
2635   emit_int8(0x0F);
2636   emit_int8(0x7F);
2637   // workaround gcc (3.2.1-7a) bug
2638   // In that version of gcc with only an emit_operand(MMX, Address)
2639   // gcc will tail jump and try and reverse the parameters completely
2640   // obliterating dst in the process. By having a version available
2641   // that doesn't need to swap the args at the tail jump the bug is
2642   // avoided.
2643   emit_operand(dst, src);
2644 }
2645 
2646 void Assembler::movq(XMMRegister dst, Address src) {
2647   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2648   InstructionMark im(this);
2649   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2650   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
2651   attributes.set_rex_vex_w_reverted();
2652   simd_prefix(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2653   emit_int8(0x7E);
2654   emit_operand(dst, src);
2655 }
2656 
2657 void Assembler::movq(Address dst, XMMRegister src) {
2658   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2659   InstructionMark im(this);
2660   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2661   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
2662   attributes.set_rex_vex_w_reverted();
2663   simd_prefix(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2664   emit_int8((unsigned char)0xD6);
2665   emit_operand(src, dst);
2666 }
2667 
2668 void Assembler::movsbl(Register dst, Address src) { // movsxb
2669   InstructionMark im(this);
2670   prefix(src, dst);
2671   emit_int8(0x0F);
2672   emit_int8((unsigned char)0xBE);
2673   emit_operand(dst, src);
2674 }
2675 
2676 void Assembler::movsbl(Register dst, Register src) { // movsxb
2677   NOT_LP64(assert(src->has_byte_register(), "must have byte register"));
2678   int encode = prefix_and_encode(dst->encoding(), false, src->encoding(), true);
2679   emit_int8(0x0F);
2680   emit_int8((unsigned char)0xBE);
2681   emit_int8((unsigned char)(0xC0 | encode));
2682 }
2683 
2684 void Assembler::movsd(XMMRegister dst, XMMRegister src) {
2685   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2686   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
2687   attributes.set_rex_vex_w_reverted();
2688   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2689   emit_int8(0x10);
2690   emit_int8((unsigned char)(0xC0 | encode));
2691 }
2692 
2693 void Assembler::movsd(XMMRegister dst, Address src) {
2694   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2695   InstructionMark im(this);
2696   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
2697   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
2698   attributes.set_rex_vex_w_reverted();
2699   simd_prefix(dst, xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2700   emit_int8(0x10);
2701   emit_operand(dst, src);
2702 }
2703 
2704 void Assembler::movsd(Address dst, XMMRegister src) {
2705   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2706   InstructionMark im(this);
2707   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
2708   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
2709   attributes.set_rex_vex_w_reverted();
2710   simd_prefix(src, xnoreg, dst, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2711   emit_int8(0x11);
2712   emit_operand(src, dst);
2713 }
2714 
2715 void Assembler::movss(XMMRegister dst, XMMRegister src) {
2716   NOT_LP64(assert(VM_Version::supports_sse(), ""));
2717   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
2718   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2719   emit_int8(0x10);
2720   emit_int8((unsigned char)(0xC0 | encode));
2721 }
2722 
2723 void Assembler::movss(XMMRegister dst, Address src) {
2724   NOT_LP64(assert(VM_Version::supports_sse(), ""));
2725   InstructionMark im(this);
2726   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
2727   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
2728   simd_prefix(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2729   emit_int8(0x10);
2730   emit_operand(dst, src);
2731 }
2732 
2733 void Assembler::movss(Address dst, XMMRegister src) {
2734   NOT_LP64(assert(VM_Version::supports_sse(), ""));
2735   InstructionMark im(this);
2736   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
2737   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
2738   simd_prefix(src, xnoreg, dst, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2739   emit_int8(0x11);
2740   emit_operand(src, dst);
2741 }
2742 
2743 void Assembler::movswl(Register dst, Address src) { // movsxw
2744   InstructionMark im(this);
2745   prefix(src, dst);
2746   emit_int8(0x0F);
2747   emit_int8((unsigned char)0xBF);
2748   emit_operand(dst, src);
2749 }
2750 
2751 void Assembler::movswl(Register dst, Register src) { // movsxw
2752   int encode = prefix_and_encode(dst->encoding(), src->encoding());
2753   emit_int8(0x0F);
2754   emit_int8((unsigned char)0xBF);
2755   emit_int8((unsigned char)(0xC0 | encode));
2756 }
2757 
2758 void Assembler::movw(Address dst, int imm16) {
2759   InstructionMark im(this);
2760 
2761   emit_int8(0x66); // switch to 16-bit mode
2762   prefix(dst);
2763   emit_int8((unsigned char)0xC7);
2764   emit_operand(rax, dst, 2);
2765   emit_int16(imm16);
2766 }
2767 
2768 void Assembler::movw(Register dst, Address src) {
2769   InstructionMark im(this);
2770   emit_int8(0x66);
2771   prefix(src, dst);
2772   emit_int8((unsigned char)0x8B);
2773   emit_operand(dst, src);
2774 }
2775 
2776 void Assembler::movw(Address dst, Register src) {
2777   InstructionMark im(this);
2778   emit_int8(0x66);
2779   prefix(dst, src);
2780   emit_int8((unsigned char)0x89);
2781   emit_operand(src, dst);
2782 }
2783 
2784 void Assembler::movzbl(Register dst, Address src) { // movzxb
2785   InstructionMark im(this);
2786   prefix(src, dst);
2787   emit_int8(0x0F);
2788   emit_int8((unsigned char)0xB6);
2789   emit_operand(dst, src);
2790 }
2791 
2792 void Assembler::movzbl(Register dst, Register src) { // movzxb
2793   NOT_LP64(assert(src->has_byte_register(), "must have byte register"));
2794   int encode = prefix_and_encode(dst->encoding(), false, src->encoding(), true);
2795   emit_int8(0x0F);
2796   emit_int8((unsigned char)0xB6);
2797   emit_int8(0xC0 | encode);
2798 }
2799 
2800 void Assembler::movzwl(Register dst, Address src) { // movzxw
2801   InstructionMark im(this);
2802   prefix(src, dst);
2803   emit_int8(0x0F);
2804   emit_int8((unsigned char)0xB7);
2805   emit_operand(dst, src);
2806 }
2807 
2808 void Assembler::movzwl(Register dst, Register src) { // movzxw
2809   int encode = prefix_and_encode(dst->encoding(), src->encoding());
2810   emit_int8(0x0F);
2811   emit_int8((unsigned char)0xB7);
2812   emit_int8(0xC0 | encode);
2813 }
2814 
2815 void Assembler::mull(Address src) {
2816   InstructionMark im(this);
2817   prefix(src);
2818   emit_int8((unsigned char)0xF7);
2819   emit_operand(rsp, src);
2820 }
2821 
2822 void Assembler::mull(Register src) {
2823   int encode = prefix_and_encode(src->encoding());
2824   emit_int8((unsigned char)0xF7);
2825   emit_int8((unsigned char)(0xE0 | encode));
2826 }
2827 
2828 void Assembler::mulsd(XMMRegister dst, Address src) {
2829   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2830   InstructionMark im(this);
2831   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
2832   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
2833   attributes.set_rex_vex_w_reverted();
2834   simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2835   emit_int8(0x59);
2836   emit_operand(dst, src);
2837 }
2838 
2839 void Assembler::mulsd(XMMRegister dst, XMMRegister src) {
2840   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2841   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
2842   attributes.set_rex_vex_w_reverted();
2843   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2844   emit_int8(0x59);
2845   emit_int8((unsigned char)(0xC0 | encode));
2846 }
2847 
2848 void Assembler::mulss(XMMRegister dst, Address src) {
2849   NOT_LP64(assert(VM_Version::supports_sse(), ""));
2850   InstructionMark im(this);
2851   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
2852   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
2853   simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2854   emit_int8(0x59);
2855   emit_operand(dst, src);
2856 }
2857 
2858 void Assembler::mulss(XMMRegister dst, XMMRegister src) {
2859   NOT_LP64(assert(VM_Version::supports_sse(), ""));
2860   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
2861   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2862   emit_int8(0x59);
2863   emit_int8((unsigned char)(0xC0 | encode));
2864 }
2865 
2866 void Assembler::negl(Register dst) {
2867   int encode = prefix_and_encode(dst->encoding());
2868   emit_int8((unsigned char)0xF7);
2869   emit_int8((unsigned char)(0xD8 | encode));
2870 }
2871 
2872 void Assembler::nop(int i) {
2873 #ifdef ASSERT
2874   assert(i > 0, " ");
2875   // The fancy nops aren't currently recognized by debuggers making it a
2876   // pain to disassemble code while debugging. If asserts are on clearly
2877   // speed is not an issue so simply use the single byte traditional nop
2878   // to do alignment.
2879 
2880   for (; i > 0 ; i--) emit_int8((unsigned char)0x90);
2881   return;
2882 
2883 #endif // ASSERT
2884 
2885   if (UseAddressNop && VM_Version::is_intel()) {
2886     //
2887     // Using multi-bytes nops "0x0F 0x1F [address]" for Intel
2888     //  1: 0x90
2889     //  2: 0x66 0x90
2890     //  3: 0x66 0x66 0x90 (don't use "0x0F 0x1F 0x00" - need patching safe padding)
2891     //  4: 0x0F 0x1F 0x40 0x00
2892     //  5: 0x0F 0x1F 0x44 0x00 0x00
2893     //  6: 0x66 0x0F 0x1F 0x44 0x00 0x00
2894     //  7: 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00
2895     //  8: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
2896     //  9: 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
2897     // 10: 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
2898     // 11: 0x66 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
2899 
2900     // The rest coding is Intel specific - don't use consecutive address nops
2901 
2902     // 12: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
2903     // 13: 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
2904     // 14: 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
2905     // 15: 0x66 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
2906 
2907     while(i >= 15) {
2908       // For Intel don't generate consecutive addess nops (mix with regular nops)
2909       i -= 15;
2910       emit_int8(0x66);   // size prefix
2911       emit_int8(0x66);   // size prefix
2912       emit_int8(0x66);   // size prefix
2913       addr_nop_8();
2914       emit_int8(0x66);   // size prefix
2915       emit_int8(0x66);   // size prefix
2916       emit_int8(0x66);   // size prefix
2917       emit_int8((unsigned char)0x90);
2918                          // nop
2919     }
2920     switch (i) {
2921       case 14:
2922         emit_int8(0x66); // size prefix
2923       case 13:
2924         emit_int8(0x66); // size prefix
2925       case 12:
2926         addr_nop_8();
2927         emit_int8(0x66); // size prefix
2928         emit_int8(0x66); // size prefix
2929         emit_int8(0x66); // size prefix
2930         emit_int8((unsigned char)0x90);
2931                          // nop
2932         break;
2933       case 11:
2934         emit_int8(0x66); // size prefix
2935       case 10:
2936         emit_int8(0x66); // size prefix
2937       case 9:
2938         emit_int8(0x66); // size prefix
2939       case 8:
2940         addr_nop_8();
2941         break;
2942       case 7:
2943         addr_nop_7();
2944         break;
2945       case 6:
2946         emit_int8(0x66); // size prefix
2947       case 5:
2948         addr_nop_5();
2949         break;
2950       case 4:
2951         addr_nop_4();
2952         break;
2953       case 3:
2954         // Don't use "0x0F 0x1F 0x00" - need patching safe padding
2955         emit_int8(0x66); // size prefix
2956       case 2:
2957         emit_int8(0x66); // size prefix
2958       case 1:
2959         emit_int8((unsigned char)0x90);
2960                          // nop
2961         break;
2962       default:
2963         assert(i == 0, " ");
2964     }
2965     return;
2966   }
2967   if (UseAddressNop && VM_Version::is_amd()) {
2968     //
2969     // Using multi-bytes nops "0x0F 0x1F [address]" for AMD.
2970     //  1: 0x90
2971     //  2: 0x66 0x90
2972     //  3: 0x66 0x66 0x90 (don't use "0x0F 0x1F 0x00" - need patching safe padding)
2973     //  4: 0x0F 0x1F 0x40 0x00
2974     //  5: 0x0F 0x1F 0x44 0x00 0x00
2975     //  6: 0x66 0x0F 0x1F 0x44 0x00 0x00
2976     //  7: 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00
2977     //  8: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
2978     //  9: 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
2979     // 10: 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
2980     // 11: 0x66 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
2981 
2982     // The rest coding is AMD specific - use consecutive address nops
2983 
2984     // 12: 0x66 0x0F 0x1F 0x44 0x00 0x00 0x66 0x0F 0x1F 0x44 0x00 0x00
2985     // 13: 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00 0x66 0x0F 0x1F 0x44 0x00 0x00
2986     // 14: 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00
2987     // 15: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00
2988     // 16: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
2989     //     Size prefixes (0x66) are added for larger sizes
2990 
2991     while(i >= 22) {
2992       i -= 11;
2993       emit_int8(0x66); // size prefix
2994       emit_int8(0x66); // size prefix
2995       emit_int8(0x66); // size prefix
2996       addr_nop_8();
2997     }
2998     // Generate first nop for size between 21-12
2999     switch (i) {
3000       case 21:
3001         i -= 1;
3002         emit_int8(0x66); // size prefix
3003       case 20:
3004       case 19:
3005         i -= 1;
3006         emit_int8(0x66); // size prefix
3007       case 18:
3008       case 17:
3009         i -= 1;
3010         emit_int8(0x66); // size prefix
3011       case 16:
3012       case 15:
3013         i -= 8;
3014         addr_nop_8();
3015         break;
3016       case 14:
3017       case 13:
3018         i -= 7;
3019         addr_nop_7();
3020         break;
3021       case 12:
3022         i -= 6;
3023         emit_int8(0x66); // size prefix
3024         addr_nop_5();
3025         break;
3026       default:
3027         assert(i < 12, " ");
3028     }
3029 
3030     // Generate second nop for size between 11-1
3031     switch (i) {
3032       case 11:
3033         emit_int8(0x66); // size prefix
3034       case 10:
3035         emit_int8(0x66); // size prefix
3036       case 9:
3037         emit_int8(0x66); // size prefix
3038       case 8:
3039         addr_nop_8();
3040         break;
3041       case 7:
3042         addr_nop_7();
3043         break;
3044       case 6:
3045         emit_int8(0x66); // size prefix
3046       case 5:
3047         addr_nop_5();
3048         break;
3049       case 4:
3050         addr_nop_4();
3051         break;
3052       case 3:
3053         // Don't use "0x0F 0x1F 0x00" - need patching safe padding
3054         emit_int8(0x66); // size prefix
3055       case 2:
3056         emit_int8(0x66); // size prefix
3057       case 1:
3058         emit_int8((unsigned char)0x90);
3059                          // nop
3060         break;
3061       default:
3062         assert(i == 0, " ");
3063     }
3064     return;
3065   }
3066 
3067   // Using nops with size prefixes "0x66 0x90".
3068   // From AMD Optimization Guide:
3069   //  1: 0x90
3070   //  2: 0x66 0x90
3071   //  3: 0x66 0x66 0x90
3072   //  4: 0x66 0x66 0x66 0x90
3073   //  5: 0x66 0x66 0x90 0x66 0x90
3074   //  6: 0x66 0x66 0x90 0x66 0x66 0x90
3075   //  7: 0x66 0x66 0x66 0x90 0x66 0x66 0x90
3076   //  8: 0x66 0x66 0x66 0x90 0x66 0x66 0x66 0x90
3077   //  9: 0x66 0x66 0x90 0x66 0x66 0x90 0x66 0x66 0x90
3078   // 10: 0x66 0x66 0x66 0x90 0x66 0x66 0x90 0x66 0x66 0x90
3079   //
3080   while(i > 12) {
3081     i -= 4;
3082     emit_int8(0x66); // size prefix
3083     emit_int8(0x66);
3084     emit_int8(0x66);
3085     emit_int8((unsigned char)0x90);
3086                      // nop
3087   }
3088   // 1 - 12 nops
3089   if(i > 8) {
3090     if(i > 9) {
3091       i -= 1;
3092       emit_int8(0x66);
3093     }
3094     i -= 3;
3095     emit_int8(0x66);
3096     emit_int8(0x66);
3097     emit_int8((unsigned char)0x90);
3098   }
3099   // 1 - 8 nops
3100   if(i > 4) {
3101     if(i > 6) {
3102       i -= 1;
3103       emit_int8(0x66);
3104     }
3105     i -= 3;
3106     emit_int8(0x66);
3107     emit_int8(0x66);
3108     emit_int8((unsigned char)0x90);
3109   }
3110   switch (i) {
3111     case 4:
3112       emit_int8(0x66);
3113     case 3:
3114       emit_int8(0x66);
3115     case 2:
3116       emit_int8(0x66);
3117     case 1:
3118       emit_int8((unsigned char)0x90);
3119       break;
3120     default:
3121       assert(i == 0, " ");
3122   }
3123 }
3124 
3125 void Assembler::notl(Register dst) {
3126   int encode = prefix_and_encode(dst->encoding());
3127   emit_int8((unsigned char)0xF7);
3128   emit_int8((unsigned char)(0xD0 | encode));
3129 }
3130 
3131 void Assembler::orl(Address dst, int32_t imm32) {
3132   InstructionMark im(this);
3133   prefix(dst);
3134   emit_arith_operand(0x81, rcx, dst, imm32);
3135 }
3136 
3137 void Assembler::orl(Register dst, int32_t imm32) {
3138   prefix(dst);
3139   emit_arith(0x81, 0xC8, dst, imm32);
3140 }
3141 
3142 void Assembler::orl(Register dst, Address src) {
3143   InstructionMark im(this);
3144   prefix(src, dst);
3145   emit_int8(0x0B);
3146   emit_operand(dst, src);
3147 }
3148 
3149 void Assembler::orl(Register dst, Register src) {
3150   (void) prefix_and_encode(dst->encoding(), src->encoding());
3151   emit_arith(0x0B, 0xC0, dst, src);
3152 }
3153 
3154 void Assembler::orl(Address dst, Register src) {
3155   InstructionMark im(this);
3156   prefix(dst, src);
3157   emit_int8(0x09);
3158   emit_operand(src, dst);
3159 }
3160 
3161 void Assembler::packuswb(XMMRegister dst, Address src) {
3162   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3163   assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
3164   InstructionMark im(this);
3165   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
3166   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
3167   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3168   emit_int8(0x67);
3169   emit_operand(dst, src);
3170 }
3171 
3172 void Assembler::packuswb(XMMRegister dst, XMMRegister src) {
3173   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3174   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
3175   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3176   emit_int8(0x67);
3177   emit_int8((unsigned char)(0xC0 | encode));
3178 }
3179 
3180 void Assembler::vpackuswb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
3181   assert(UseAVX > 0, "some form of AVX must be enabled");
3182   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
3183   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3184   emit_int8(0x67);
3185   emit_int8((unsigned char)(0xC0 | encode));
3186 }
3187 
3188 void Assembler::vpermq(XMMRegister dst, XMMRegister src, int imm8, int vector_len) {
3189   assert(VM_Version::supports_avx2(), "");
3190   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
3191   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3192   emit_int8(0x00);
3193   emit_int8(0xC0 | encode);
3194   emit_int8(imm8);
3195 }
3196 
3197 void Assembler::pause() {
3198   emit_int8((unsigned char)0xF3);
3199   emit_int8((unsigned char)0x90);
3200 }
3201 
3202 void Assembler::pcmpestri(XMMRegister dst, Address src, int imm8) {
3203   assert(VM_Version::supports_sse4_2(), "");
3204   InstructionMark im(this);
3205   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3206   simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3207   emit_int8(0x61);
3208   emit_operand(dst, src);
3209   emit_int8(imm8);
3210 }
3211 
3212 void Assembler::pcmpestri(XMMRegister dst, XMMRegister src, int imm8) {
3213   assert(VM_Version::supports_sse4_2(), "");
3214   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3215   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3216   emit_int8(0x61);
3217   emit_int8((unsigned char)(0xC0 | encode));
3218   emit_int8(imm8);
3219 }
3220 
3221 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
3222 void Assembler::pcmpeqb(XMMRegister dst, XMMRegister src) {
3223   assert(VM_Version::supports_sse2(), "");
3224   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3225   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3226   emit_int8(0x74);
3227   emit_int8((unsigned char)(0xC0 | encode));
3228 }
3229 
3230 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
3231 void Assembler::vpcmpeqb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
3232   assert(VM_Version::supports_avx(), "");
3233   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3234   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3235   emit_int8(0x74);
3236   emit_int8((unsigned char)(0xC0 | encode));
3237 }
3238 
3239 // In this context, kdst is written the mask used to process the equal components
3240 void Assembler::evpcmpeqb(KRegister kdst, XMMRegister nds, XMMRegister src, int vector_len) {
3241   assert(VM_Version::supports_avx512bw(), "");
3242   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
3243   attributes.set_is_evex_instruction();
3244   int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3245   emit_int8(0x74);
3246   emit_int8((unsigned char)(0xC0 | encode));
3247 }
3248 
3249 void Assembler::evpcmpeqb(KRegister kdst, XMMRegister nds, Address src, int vector_len) {
3250   assert(VM_Version::supports_avx512bw(), "");
3251   InstructionMark im(this);
3252   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
3253   attributes.set_is_evex_instruction();
3254   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
3255   int dst_enc = kdst->encoding();
3256   vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3257   emit_int8(0x74);
3258   emit_operand(as_Register(dst_enc), src);
3259 }
3260 
3261 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
3262 void Assembler::pcmpeqw(XMMRegister dst, XMMRegister src) {
3263   assert(VM_Version::supports_sse2(), "");
3264   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3265   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3266   emit_int8(0x75);
3267   emit_int8((unsigned char)(0xC0 | encode));
3268 }
3269 
3270 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
3271 void Assembler::vpcmpeqw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
3272   assert(VM_Version::supports_avx(), "");
3273   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3274   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3275   emit_int8(0x75);
3276   emit_int8((unsigned char)(0xC0 | encode));
3277 }
3278 
3279 // In this context, kdst is written the mask used to process the equal components
3280 void Assembler::evpcmpeqw(KRegister kdst, XMMRegister nds, XMMRegister src, int vector_len) {
3281   assert(VM_Version::supports_avx512bw(), "");
3282   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
3283   attributes.set_is_evex_instruction();
3284   int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3285   emit_int8(0x75);
3286   emit_int8((unsigned char)(0xC0 | encode));
3287 }
3288 
3289 void Assembler::evpcmpeqw(KRegister kdst, XMMRegister nds, Address src, int vector_len) {
3290   assert(VM_Version::supports_avx512bw(), "");
3291   InstructionMark im(this);
3292   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
3293   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
3294   attributes.set_is_evex_instruction();
3295   int dst_enc = kdst->encoding();
3296   vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3297   emit_int8(0x75);
3298   emit_operand(as_Register(dst_enc), src);
3299 }
3300 
3301 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
3302 void Assembler::pcmpeqd(XMMRegister dst, XMMRegister src) {
3303   assert(VM_Version::supports_sse2(), "");
3304   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3305   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3306   emit_int8(0x76);
3307   emit_int8((unsigned char)(0xC0 | encode));
3308 }
3309 
3310 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
3311 void Assembler::vpcmpeqd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
3312   assert(VM_Version::supports_avx(), "");
3313   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3314   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3315   emit_int8(0x76);
3316   emit_int8((unsigned char)(0xC0 | encode));
3317 }
3318 
3319 // In this context, kdst is written the mask used to process the equal components
3320 void Assembler::evpcmpeqd(KRegister kdst, XMMRegister nds, XMMRegister src, int vector_len) {
3321   assert(VM_Version::supports_evex(), "");
3322   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
3323   attributes.set_is_evex_instruction();
3324   int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3325   emit_int8(0x76);
3326   emit_int8((unsigned char)(0xC0 | encode));
3327 }
3328 
3329 void Assembler::evpcmpeqd(KRegister kdst, XMMRegister nds, Address src, int vector_len) {
3330   assert(VM_Version::supports_evex(), "");
3331   InstructionMark im(this);
3332   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
3333   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
3334   attributes.set_is_evex_instruction();
3335   int dst_enc = kdst->encoding();
3336   vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3337   emit_int8(0x76);
3338   emit_operand(as_Register(dst_enc), src);
3339 }
3340 
3341 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
3342 void Assembler::pcmpeqq(XMMRegister dst, XMMRegister src) {
3343   assert(VM_Version::supports_sse4_1(), "");
3344   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3345   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3346   emit_int8(0x29);
3347   emit_int8((unsigned char)(0xC0 | encode));
3348 }
3349 
3350 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
3351 void Assembler::vpcmpeqq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
3352   assert(VM_Version::supports_avx(), "");
3353   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3354   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3355   emit_int8(0x29);
3356   emit_int8((unsigned char)(0xC0 | encode));
3357 }
3358 
3359 // In this context, kdst is written the mask used to process the equal components
3360 void Assembler::evpcmpeqq(KRegister kdst, XMMRegister nds, XMMRegister src, int vector_len) {
3361   assert(VM_Version::supports_evex(), "");
3362   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
3363   attributes.set_is_evex_instruction();
3364   int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3365   emit_int8(0x29);
3366   emit_int8((unsigned char)(0xC0 | encode));
3367 }
3368 
3369 // In this context, kdst is written the mask used to process the equal components
3370 void Assembler::evpcmpeqq(KRegister kdst, XMMRegister nds, Address src, int vector_len) {
3371   assert(VM_Version::supports_evex(), "");
3372   InstructionMark im(this);
3373   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
3374   attributes.set_is_evex_instruction();
3375   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
3376   int dst_enc = kdst->encoding();
3377   vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3378   emit_int8(0x29);
3379   emit_operand(as_Register(dst_enc), src);
3380 }
3381 
3382 void Assembler::pmovmskb(Register dst, XMMRegister src) {
3383   assert(VM_Version::supports_sse2(), "");
3384   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3385   int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3386   emit_int8((unsigned char)0xD7);
3387   emit_int8((unsigned char)(0xC0 | encode));
3388 }
3389 
3390 void Assembler::vpmovmskb(Register dst, XMMRegister src) {
3391   assert(VM_Version::supports_avx2(), "");
3392   InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3393   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3394   emit_int8((unsigned char)0xD7);
3395   emit_int8((unsigned char)(0xC0 | encode));
3396 }
3397 
3398 void Assembler::pextrd(Register dst, XMMRegister src, int imm8) {
3399   assert(VM_Version::supports_sse4_1(), "");
3400   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
3401   int encode = simd_prefix_and_encode(src, xnoreg, as_XMMRegister(dst->encoding()), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3402   emit_int8(0x16);
3403   emit_int8((unsigned char)(0xC0 | encode));
3404   emit_int8(imm8);
3405 }
3406 
3407 void Assembler::pextrd(Address dst, XMMRegister src, int imm8) {
3408   assert(VM_Version::supports_sse4_1(), "");
3409   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
3410   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
3411   simd_prefix(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3412   emit_int8(0x16);
3413   emit_operand(src, dst);
3414   emit_int8(imm8);
3415 }
3416 
3417 void Assembler::pextrq(Register dst, XMMRegister src, int imm8) {
3418   assert(VM_Version::supports_sse4_1(), "");
3419   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
3420   int encode = simd_prefix_and_encode(src, xnoreg, as_XMMRegister(dst->encoding()), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3421   emit_int8(0x16);
3422   emit_int8((unsigned char)(0xC0 | encode));
3423   emit_int8(imm8);
3424 }
3425 
3426 void Assembler::pextrq(Address dst, XMMRegister src, int imm8) {
3427   assert(VM_Version::supports_sse4_1(), "");
3428   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
3429   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
3430   simd_prefix(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3431   emit_int8(0x16);
3432   emit_operand(src, dst);
3433   emit_int8(imm8);
3434 }
3435 
3436 void Assembler::pextrw(Register dst, XMMRegister src, int imm8) {
3437   assert(VM_Version::supports_sse2(), "");
3438   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
3439   int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3440   emit_int8((unsigned char)0xC5);
3441   emit_int8((unsigned char)(0xC0 | encode));
3442   emit_int8(imm8);
3443 }
3444 
3445 void Assembler::pextrw(Address dst, XMMRegister src, int imm8) {
3446   assert(VM_Version::supports_sse4_1(), "");
3447   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
3448   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_16bit);
3449   simd_prefix(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3450   emit_int8((unsigned char)0x15);
3451   emit_operand(src, dst);
3452   emit_int8(imm8);
3453 }
3454 
3455 void Assembler::pextrb(Address dst, XMMRegister src, int imm8) {
3456   assert(VM_Version::supports_sse4_1(), "");
3457   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
3458   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_8bit);
3459   simd_prefix(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3460   emit_int8(0x14);
3461   emit_operand(src, dst);
3462   emit_int8(imm8);
3463 }
3464 
3465 void Assembler::pinsrd(XMMRegister dst, Register src, int imm8) {
3466   assert(VM_Version::supports_sse4_1(), "");
3467   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
3468   int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3469   emit_int8(0x22);
3470   emit_int8((unsigned char)(0xC0 | encode));
3471   emit_int8(imm8);
3472 }
3473 
3474 void Assembler::pinsrd(XMMRegister dst, Address src, int imm8) {
3475   assert(VM_Version::supports_sse4_1(), "");
3476   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
3477   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
3478   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3479   emit_int8(0x22);
3480   emit_operand(dst,src);
3481   emit_int8(imm8);
3482 }
3483 
3484 void Assembler::pinsrq(XMMRegister dst, Register src, int imm8) {
3485   assert(VM_Version::supports_sse4_1(), "");
3486   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
3487   int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3488   emit_int8(0x22);
3489   emit_int8((unsigned char)(0xC0 | encode));
3490   emit_int8(imm8);
3491 }
3492 
3493 void Assembler::pinsrq(XMMRegister dst, Address src, int imm8) {
3494   assert(VM_Version::supports_sse4_1(), "");
3495   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
3496   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
3497   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3498   emit_int8(0x22);
3499   emit_operand(dst, src);
3500   emit_int8(imm8);
3501 }
3502 
3503 void Assembler::pinsrw(XMMRegister dst, Register src, int imm8) {
3504   assert(VM_Version::supports_sse2(), "");
3505   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
3506   int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3507   emit_int8((unsigned char)0xC4);
3508   emit_int8((unsigned char)(0xC0 | encode));
3509   emit_int8(imm8);
3510 }
3511 
3512 void Assembler::pinsrw(XMMRegister dst, Address src, int imm8) {
3513   assert(VM_Version::supports_sse2(), "");
3514   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
3515   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_16bit);
3516   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3517   emit_int8((unsigned char)0xC4);
3518   emit_operand(dst, src);
3519   emit_int8(imm8);
3520 }
3521 
3522 void Assembler::pinsrb(XMMRegister dst, Address src, int imm8) {
3523   assert(VM_Version::supports_sse4_1(), "");
3524   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
3525   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_8bit);
3526   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3527   emit_int8(0x20);
3528   emit_operand(dst, src);
3529   emit_int8(imm8);
3530 }
3531 
3532 void Assembler::pmovzxbw(XMMRegister dst, Address src) {
3533   assert(VM_Version::supports_sse4_1(), "");
3534   InstructionMark im(this);
3535   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
3536   attributes.set_address_attributes(/* tuple_type */ EVEX_HVM, /* input_size_in_bits */ EVEX_NObit);
3537   simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3538   emit_int8(0x30);
3539   emit_operand(dst, src);
3540 }
3541 
3542 void Assembler::pmovzxbw(XMMRegister dst, XMMRegister src) {
3543   assert(VM_Version::supports_sse4_1(), "");
3544   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
3545   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3546   emit_int8(0x30);
3547   emit_int8((unsigned char)(0xC0 | encode));
3548 }
3549 
3550 void Assembler::vpmovzxbw(XMMRegister dst, Address src, int vector_len) {
3551   assert(VM_Version::supports_avx(), "");
3552   InstructionMark im(this);
3553   assert(dst != xnoreg, "sanity");
3554   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
3555   attributes.set_address_attributes(/* tuple_type */ EVEX_HVM, /* input_size_in_bits */ EVEX_NObit);
3556   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3557   emit_int8(0x30);
3558   emit_operand(dst, src);
3559 }
3560 
3561 // generic
3562 void Assembler::pop(Register dst) {
3563   int encode = prefix_and_encode(dst->encoding());
3564   emit_int8(0x58 | encode);
3565 }
3566 
3567 void Assembler::popcntl(Register dst, Address src) {
3568   assert(VM_Version::supports_popcnt(), "must support");
3569   InstructionMark im(this);
3570   emit_int8((unsigned char)0xF3);
3571   prefix(src, dst);
3572   emit_int8(0x0F);
3573   emit_int8((unsigned char)0xB8);
3574   emit_operand(dst, src);
3575 }
3576 
3577 void Assembler::popcntl(Register dst, Register src) {
3578   assert(VM_Version::supports_popcnt(), "must support");
3579   emit_int8((unsigned char)0xF3);
3580   int encode = prefix_and_encode(dst->encoding(), src->encoding());
3581   emit_int8(0x0F);
3582   emit_int8((unsigned char)0xB8);
3583   emit_int8((unsigned char)(0xC0 | encode));
3584 }
3585 
3586 void Assembler::popf() {
3587   emit_int8((unsigned char)0x9D);
3588 }
3589 
3590 #ifndef _LP64 // no 32bit push/pop on amd64
3591 void Assembler::popl(Address dst) {
3592   // NOTE: this will adjust stack by 8byte on 64bits
3593   InstructionMark im(this);
3594   prefix(dst);
3595   emit_int8((unsigned char)0x8F);
3596   emit_operand(rax, dst);
3597 }
3598 #endif
3599 
3600 void Assembler::prefetch_prefix(Address src) {
3601   prefix(src);
3602   emit_int8(0x0F);
3603 }
3604 
3605 void Assembler::prefetchnta(Address src) {
3606   NOT_LP64(assert(VM_Version::supports_sse(), "must support"));
3607   InstructionMark im(this);
3608   prefetch_prefix(src);
3609   emit_int8(0x18);
3610   emit_operand(rax, src); // 0, src
3611 }
3612 
3613 void Assembler::prefetchr(Address src) {
3614   assert(VM_Version::supports_3dnow_prefetch(), "must support");
3615   InstructionMark im(this);
3616   prefetch_prefix(src);
3617   emit_int8(0x0D);
3618   emit_operand(rax, src); // 0, src
3619 }
3620 
3621 void Assembler::prefetcht0(Address src) {
3622   NOT_LP64(assert(VM_Version::supports_sse(), "must support"));
3623   InstructionMark im(this);
3624   prefetch_prefix(src);
3625   emit_int8(0x18);
3626   emit_operand(rcx, src); // 1, src
3627 }
3628 
3629 void Assembler::prefetcht1(Address src) {
3630   NOT_LP64(assert(VM_Version::supports_sse(), "must support"));
3631   InstructionMark im(this);
3632   prefetch_prefix(src);
3633   emit_int8(0x18);
3634   emit_operand(rdx, src); // 2, src
3635 }
3636 
3637 void Assembler::prefetcht2(Address src) {
3638   NOT_LP64(assert(VM_Version::supports_sse(), "must support"));
3639   InstructionMark im(this);
3640   prefetch_prefix(src);
3641   emit_int8(0x18);
3642   emit_operand(rbx, src); // 3, src
3643 }
3644 
3645 void Assembler::prefetchw(Address src) {
3646   assert(VM_Version::supports_3dnow_prefetch(), "must support");
3647   InstructionMark im(this);
3648   prefetch_prefix(src);
3649   emit_int8(0x0D);
3650   emit_operand(rcx, src); // 1, src
3651 }
3652 
3653 void Assembler::prefix(Prefix p) {
3654   emit_int8(p);
3655 }
3656 
3657 void Assembler::pshufb(XMMRegister dst, XMMRegister src) {
3658   assert(VM_Version::supports_ssse3(), "");
3659   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
3660   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3661   emit_int8(0x00);
3662   emit_int8((unsigned char)(0xC0 | encode));
3663 }
3664 
3665 void Assembler::pshufb(XMMRegister dst, Address src) {
3666   assert(VM_Version::supports_ssse3(), "");
3667   InstructionMark im(this);
3668   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
3669   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
3670   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3671   emit_int8(0x00);
3672   emit_operand(dst, src);
3673 }
3674 
3675 void Assembler::pshufd(XMMRegister dst, XMMRegister src, int mode) {
3676   assert(isByte(mode), "invalid value");
3677   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3678   int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_128bit;
3679   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
3680   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3681   emit_int8(0x70);
3682   emit_int8((unsigned char)(0xC0 | encode));
3683   emit_int8(mode & 0xFF);
3684 }
3685 
3686 void Assembler::pshufd(XMMRegister dst, Address src, int mode) {
3687   assert(isByte(mode), "invalid value");
3688   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3689   assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
3690   InstructionMark im(this);
3691   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
3692   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
3693   simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3694   emit_int8(0x70);
3695   emit_operand(dst, src);
3696   emit_int8(mode & 0xFF);
3697 }
3698 
3699 void Assembler::pshuflw(XMMRegister dst, XMMRegister src, int mode) {
3700   assert(isByte(mode), "invalid value");
3701   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3702   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
3703   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
3704   emit_int8(0x70);
3705   emit_int8((unsigned char)(0xC0 | encode));
3706   emit_int8(mode & 0xFF);
3707 }
3708 
3709 void Assembler::pshuflw(XMMRegister dst, Address src, int mode) {
3710   assert(isByte(mode), "invalid value");
3711   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3712   assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
3713   InstructionMark im(this);
3714   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
3715   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
3716   simd_prefix(dst, xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
3717   emit_int8(0x70);
3718   emit_operand(dst, src);
3719   emit_int8(mode & 0xFF);
3720 }
3721 
3722 void Assembler::psrldq(XMMRegister dst, int shift) {
3723   // Shift left 128 bit value in dst XMMRegister by shift number of bytes.
3724   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3725   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
3726   // XMM3 is for /3 encoding: 66 0F 73 /3 ib
3727   int encode = simd_prefix_and_encode(xmm3, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3728   emit_int8(0x73);
3729   emit_int8((unsigned char)(0xC0 | encode));
3730   emit_int8(shift);
3731 }
3732 
3733 void Assembler::pslldq(XMMRegister dst, int shift) {
3734   // Shift left 128 bit value in dst XMMRegister by shift number of bytes.
3735   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3736   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
3737   // XMM7 is for /7 encoding: 66 0F 73 /7 ib
3738   int encode = simd_prefix_and_encode(xmm7, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3739   emit_int8(0x73);
3740   emit_int8((unsigned char)(0xC0 | encode));
3741   emit_int8(shift);
3742 }
3743 
3744 void Assembler::ptest(XMMRegister dst, Address src) {
3745   assert(VM_Version::supports_sse4_1(), "");
3746   assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
3747   InstructionMark im(this);
3748   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3749   simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3750   emit_int8(0x17);
3751   emit_operand(dst, src);
3752 }
3753 
3754 void Assembler::ptest(XMMRegister dst, XMMRegister src) {
3755   assert(VM_Version::supports_sse4_1(), "");
3756   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3757   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3758   emit_int8(0x17);
3759   emit_int8((unsigned char)(0xC0 | encode));
3760 }
3761 
3762 void Assembler::vptest(XMMRegister dst, Address src) {
3763   assert(VM_Version::supports_avx(), "");
3764   InstructionMark im(this);
3765   InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3766   assert(dst != xnoreg, "sanity");
3767   // swap src<->dst for encoding
3768   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3769   emit_int8(0x17);
3770   emit_operand(dst, src);
3771 }
3772 
3773 void Assembler::vptest(XMMRegister dst, XMMRegister src) {
3774   assert(VM_Version::supports_avx(), "");
3775   InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3776   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3777   emit_int8(0x17);
3778   emit_int8((unsigned char)(0xC0 | encode));
3779 }
3780 
3781 void Assembler::punpcklbw(XMMRegister dst, Address src) {
3782   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3783   assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
3784   InstructionMark im(this);
3785   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_vlbw, /* no_mask_reg */ false, /* uses_vl */ true);
3786   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
3787   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3788   emit_int8(0x60);
3789   emit_operand(dst, src);
3790 }
3791 
3792 void Assembler::punpcklbw(XMMRegister dst, XMMRegister src) {
3793   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3794   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_vlbw, /* no_mask_reg */ false, /* uses_vl */ true);
3795   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3796   emit_int8(0x60);
3797   emit_int8((unsigned char)(0xC0 | encode));
3798 }
3799 
3800 void Assembler::punpckldq(XMMRegister dst, Address src) {
3801   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3802   assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
3803   InstructionMark im(this);
3804   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
3805   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
3806   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3807   emit_int8(0x62);
3808   emit_operand(dst, src);
3809 }
3810 
3811 void Assembler::punpckldq(XMMRegister dst, XMMRegister src) {
3812   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3813   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
3814   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3815   emit_int8(0x62);
3816   emit_int8((unsigned char)(0xC0 | encode));
3817 }
3818 
3819 void Assembler::punpcklqdq(XMMRegister dst, XMMRegister src) {
3820   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3821   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
3822   attributes.set_rex_vex_w_reverted();
3823   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3824   emit_int8(0x6C);
3825   emit_int8((unsigned char)(0xC0 | encode));
3826 }
3827 
3828 void Assembler::push(int32_t imm32) {
3829   // in 64bits we push 64bits onto the stack but only
3830   // take a 32bit immediate
3831   emit_int8(0x68);
3832   emit_int32(imm32);
3833 }
3834 
3835 void Assembler::push(Register src) {
3836   int encode = prefix_and_encode(src->encoding());
3837 
3838   emit_int8(0x50 | encode);
3839 }
3840 
3841 void Assembler::pushf() {
3842   emit_int8((unsigned char)0x9C);
3843 }
3844 
3845 #ifndef _LP64 // no 32bit push/pop on amd64
3846 void Assembler::pushl(Address src) {
3847   // Note this will push 64bit on 64bit
3848   InstructionMark im(this);
3849   prefix(src);
3850   emit_int8((unsigned char)0xFF);
3851   emit_operand(rsi, src);
3852 }
3853 #endif
3854 
3855 void Assembler::rcll(Register dst, int imm8) {
3856   assert(isShiftCount(imm8), "illegal shift count");
3857   int encode = prefix_and_encode(dst->encoding());
3858   if (imm8 == 1) {
3859     emit_int8((unsigned char)0xD1);
3860     emit_int8((unsigned char)(0xD0 | encode));
3861   } else {
3862     emit_int8((unsigned char)0xC1);
3863     emit_int8((unsigned char)0xD0 | encode);
3864     emit_int8(imm8);
3865   }
3866 }
3867 
3868 void Assembler::rcpps(XMMRegister dst, XMMRegister src) {
3869   NOT_LP64(assert(VM_Version::supports_sse(), ""));
3870   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3871   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
3872   emit_int8(0x53);
3873   emit_int8((unsigned char)(0xC0 | encode));
3874 }
3875 
3876 void Assembler::rcpss(XMMRegister dst, XMMRegister src) {
3877   NOT_LP64(assert(VM_Version::supports_sse(), ""));
3878   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3879   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
3880   emit_int8(0x53);
3881   emit_int8((unsigned char)(0xC0 | encode));
3882 }
3883 
3884 void Assembler::rdtsc() {
3885   emit_int8((unsigned char)0x0F);
3886   emit_int8((unsigned char)0x31);
3887 }
3888 
3889 // copies data from [esi] to [edi] using rcx pointer sized words
3890 // generic
3891 void Assembler::rep_mov() {
3892   emit_int8((unsigned char)0xF3);
3893   // MOVSQ
3894   LP64_ONLY(prefix(REX_W));
3895   emit_int8((unsigned char)0xA5);
3896 }
3897 
3898 // sets rcx bytes with rax, value at [edi]
3899 void Assembler::rep_stosb() {
3900   emit_int8((unsigned char)0xF3); // REP
3901   LP64_ONLY(prefix(REX_W));
3902   emit_int8((unsigned char)0xAA); // STOSB
3903 }
3904 
3905 // sets rcx pointer sized words with rax, value at [edi]
3906 // generic
3907 void Assembler::rep_stos() {
3908   emit_int8((unsigned char)0xF3); // REP
3909   LP64_ONLY(prefix(REX_W));       // LP64:STOSQ, LP32:STOSD
3910   emit_int8((unsigned char)0xAB);
3911 }
3912 
3913 // scans rcx pointer sized words at [edi] for occurance of rax,
3914 // generic
3915 void Assembler::repne_scan() { // repne_scan
3916   emit_int8((unsigned char)0xF2);
3917   // SCASQ
3918   LP64_ONLY(prefix(REX_W));
3919   emit_int8((unsigned char)0xAF);
3920 }
3921 
3922 #ifdef _LP64
3923 // scans rcx 4 byte words at [edi] for occurance of rax,
3924 // generic
3925 void Assembler::repne_scanl() { // repne_scan
3926   emit_int8((unsigned char)0xF2);
3927   // SCASL
3928   emit_int8((unsigned char)0xAF);
3929 }
3930 #endif
3931 
3932 void Assembler::ret(int imm16) {
3933   if (imm16 == 0) {
3934     emit_int8((unsigned char)0xC3);
3935   } else {
3936     emit_int8((unsigned char)0xC2);
3937     emit_int16(imm16);
3938   }
3939 }
3940 
3941 void Assembler::sahf() {
3942 #ifdef _LP64
3943   // Not supported in 64bit mode
3944   ShouldNotReachHere();
3945 #endif
3946   emit_int8((unsigned char)0x9E);
3947 }
3948 
3949 void Assembler::sarl(Register dst, int imm8) {
3950   int encode = prefix_and_encode(dst->encoding());
3951   assert(isShiftCount(imm8), "illegal shift count");
3952   if (imm8 == 1) {
3953     emit_int8((unsigned char)0xD1);
3954     emit_int8((unsigned char)(0xF8 | encode));
3955   } else {
3956     emit_int8((unsigned char)0xC1);
3957     emit_int8((unsigned char)(0xF8 | encode));
3958     emit_int8(imm8);
3959   }
3960 }
3961 
3962 void Assembler::sarl(Register dst) {
3963   int encode = prefix_and_encode(dst->encoding());
3964   emit_int8((unsigned char)0xD3);
3965   emit_int8((unsigned char)(0xF8 | encode));
3966 }
3967 
3968 void Assembler::sbbl(Address dst, int32_t imm32) {
3969   InstructionMark im(this);
3970   prefix(dst);
3971   emit_arith_operand(0x81, rbx, dst, imm32);
3972 }
3973 
3974 void Assembler::sbbl(Register dst, int32_t imm32) {
3975   prefix(dst);
3976   emit_arith(0x81, 0xD8, dst, imm32);
3977 }
3978 
3979 
3980 void Assembler::sbbl(Register dst, Address src) {
3981   InstructionMark im(this);
3982   prefix(src, dst);
3983   emit_int8(0x1B);
3984   emit_operand(dst, src);
3985 }
3986 
3987 void Assembler::sbbl(Register dst, Register src) {
3988   (void) prefix_and_encode(dst->encoding(), src->encoding());
3989   emit_arith(0x1B, 0xC0, dst, src);
3990 }
3991 
3992 void Assembler::setb(Condition cc, Register dst) {
3993   assert(0 <= cc && cc < 16, "illegal cc");
3994   int encode = prefix_and_encode(dst->encoding(), true);
3995   emit_int8(0x0F);
3996   emit_int8((unsigned char)0x90 | cc);
3997   emit_int8((unsigned char)(0xC0 | encode));
3998 }
3999 
4000 void Assembler::palignr(XMMRegister dst, XMMRegister src, int imm8) {
4001   assert(VM_Version::supports_ssse3(), "");
4002   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ false);
4003   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
4004   emit_int8((unsigned char)0x0F);
4005   emit_int8((unsigned char)(0xC0 | encode));
4006   emit_int8(imm8);
4007 }
4008 
4009 void Assembler::pblendw(XMMRegister dst, XMMRegister src, int imm8) {
4010   assert(VM_Version::supports_sse4_1(), "");
4011   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
4012   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
4013   emit_int8((unsigned char)0x0E);
4014   emit_int8((unsigned char)(0xC0 | encode));
4015   emit_int8(imm8);
4016 }
4017 
4018 void Assembler::sha1rnds4(XMMRegister dst, XMMRegister src, int imm8) {
4019   assert(VM_Version::supports_sha(), "");
4020   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
4021   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F_3A, &attributes);
4022   emit_int8((unsigned char)0xCC);
4023   emit_int8((unsigned char)(0xC0 | encode));
4024   emit_int8((unsigned char)imm8);
4025 }
4026 
4027 void Assembler::sha1nexte(XMMRegister dst, XMMRegister src) {
4028   assert(VM_Version::supports_sha(), "");
4029   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
4030   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
4031   emit_int8((unsigned char)0xC8);
4032   emit_int8((unsigned char)(0xC0 | encode));
4033 }
4034 
4035 void Assembler::sha1msg1(XMMRegister dst, XMMRegister src) {
4036   assert(VM_Version::supports_sha(), "");
4037   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
4038   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
4039   emit_int8((unsigned char)0xC9);
4040   emit_int8((unsigned char)(0xC0 | encode));
4041 }
4042 
4043 void Assembler::sha1msg2(XMMRegister dst, XMMRegister src) {
4044   assert(VM_Version::supports_sha(), "");
4045   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
4046   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
4047   emit_int8((unsigned char)0xCA);
4048   emit_int8((unsigned char)(0xC0 | encode));
4049 }
4050 
4051 // xmm0 is implicit additional source to this instruction.
4052 void Assembler::sha256rnds2(XMMRegister dst, XMMRegister src) {
4053   assert(VM_Version::supports_sha(), "");
4054   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
4055   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
4056   emit_int8((unsigned char)0xCB);
4057   emit_int8((unsigned char)(0xC0 | encode));
4058 }
4059 
4060 void Assembler::sha256msg1(XMMRegister dst, XMMRegister src) {
4061   assert(VM_Version::supports_sha(), "");
4062   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
4063   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
4064   emit_int8((unsigned char)0xCC);
4065   emit_int8((unsigned char)(0xC0 | encode));
4066 }
4067 
4068 void Assembler::sha256msg2(XMMRegister dst, XMMRegister src) {
4069   assert(VM_Version::supports_sha(), "");
4070   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
4071   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
4072   emit_int8((unsigned char)0xCD);
4073   emit_int8((unsigned char)(0xC0 | encode));
4074 }
4075 
4076 
4077 void Assembler::shll(Register dst, int imm8) {
4078   assert(isShiftCount(imm8), "illegal shift count");
4079   int encode = prefix_and_encode(dst->encoding());
4080   if (imm8 == 1 ) {
4081     emit_int8((unsigned char)0xD1);
4082     emit_int8((unsigned char)(0xE0 | encode));
4083   } else {
4084     emit_int8((unsigned char)0xC1);
4085     emit_int8((unsigned char)(0xE0 | encode));
4086     emit_int8(imm8);
4087   }
4088 }
4089 
4090 void Assembler::shll(Register dst) {
4091   int encode = prefix_and_encode(dst->encoding());
4092   emit_int8((unsigned char)0xD3);
4093   emit_int8((unsigned char)(0xE0 | encode));
4094 }
4095 
4096 void Assembler::shrl(Register dst, int imm8) {
4097   assert(isShiftCount(imm8), "illegal shift count");
4098   int encode = prefix_and_encode(dst->encoding());
4099   emit_int8((unsigned char)0xC1);
4100   emit_int8((unsigned char)(0xE8 | encode));
4101   emit_int8(imm8);
4102 }
4103 
4104 void Assembler::shrl(Register dst) {
4105   int encode = prefix_and_encode(dst->encoding());
4106   emit_int8((unsigned char)0xD3);
4107   emit_int8((unsigned char)(0xE8 | encode));
4108 }
4109 
4110 // copies a single word from [esi] to [edi]
4111 void Assembler::smovl() {
4112   emit_int8((unsigned char)0xA5);
4113 }
4114 
4115 void Assembler::sqrtsd(XMMRegister dst, XMMRegister src) {
4116   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4117   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
4118   attributes.set_rex_vex_w_reverted();
4119   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
4120   emit_int8(0x51);
4121   emit_int8((unsigned char)(0xC0 | encode));
4122 }
4123 
4124 void Assembler::sqrtsd(XMMRegister dst, Address src) {
4125   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4126   InstructionMark im(this);
4127   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
4128   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
4129   attributes.set_rex_vex_w_reverted();
4130   simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
4131   emit_int8(0x51);
4132   emit_operand(dst, src);
4133 }
4134 
4135 void Assembler::sqrtss(XMMRegister dst, XMMRegister src) {
4136   NOT_LP64(assert(VM_Version::supports_sse(), ""));
4137   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
4138   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
4139   emit_int8(0x51);
4140   emit_int8((unsigned char)(0xC0 | encode));
4141 }
4142 
4143 void Assembler::std() {
4144   emit_int8((unsigned char)0xFD);
4145 }
4146 
4147 void Assembler::sqrtss(XMMRegister dst, Address src) {
4148   NOT_LP64(assert(VM_Version::supports_sse(), ""));
4149   InstructionMark im(this);
4150   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
4151   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
4152   simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
4153   emit_int8(0x51);
4154   emit_operand(dst, src);
4155 }
4156 
4157 void Assembler::stmxcsr( Address dst) {
4158   NOT_LP64(assert(VM_Version::supports_sse(), ""));
4159   InstructionMark im(this);
4160   prefix(dst);
4161   emit_int8(0x0F);
4162   emit_int8((unsigned char)0xAE);
4163   emit_operand(as_Register(3), dst);
4164 }
4165 
4166 void Assembler::subl(Address dst, int32_t imm32) {
4167   InstructionMark im(this);
4168   prefix(dst);
4169   emit_arith_operand(0x81, rbp, dst, imm32);
4170 }
4171 
4172 void Assembler::subl(Address dst, Register src) {
4173   InstructionMark im(this);
4174   prefix(dst, src);
4175   emit_int8(0x29);
4176   emit_operand(src, dst);
4177 }
4178 
4179 void Assembler::subl(Register dst, int32_t imm32) {
4180   prefix(dst);
4181   emit_arith(0x81, 0xE8, dst, imm32);
4182 }
4183 
4184 // Force generation of a 4 byte immediate value even if it fits into 8bit
4185 void Assembler::subl_imm32(Register dst, int32_t imm32) {
4186   prefix(dst);
4187   emit_arith_imm32(0x81, 0xE8, dst, imm32);
4188 }
4189 
4190 void Assembler::subl(Register dst, Address src) {
4191   InstructionMark im(this);
4192   prefix(src, dst);
4193   emit_int8(0x2B);
4194   emit_operand(dst, src);
4195 }
4196 
4197 void Assembler::subl(Register dst, Register src) {
4198   (void) prefix_and_encode(dst->encoding(), src->encoding());
4199   emit_arith(0x2B, 0xC0, dst, src);
4200 }
4201 
4202 void Assembler::subsd(XMMRegister dst, XMMRegister src) {
4203   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4204   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
4205   attributes.set_rex_vex_w_reverted();
4206   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
4207   emit_int8(0x5C);
4208   emit_int8((unsigned char)(0xC0 | encode));
4209 }
4210 
4211 void Assembler::subsd(XMMRegister dst, Address src) {
4212   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4213   InstructionMark im(this);
4214   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
4215   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
4216   attributes.set_rex_vex_w_reverted();
4217   simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
4218   emit_int8(0x5C);
4219   emit_operand(dst, src);
4220 }
4221 
4222 void Assembler::subss(XMMRegister dst, XMMRegister src) {
4223   NOT_LP64(assert(VM_Version::supports_sse(), ""));
4224   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false , /* uses_vl */ false);
4225   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
4226   emit_int8(0x5C);
4227   emit_int8((unsigned char)(0xC0 | encode));
4228 }
4229 
4230 void Assembler::subss(XMMRegister dst, Address src) {
4231   NOT_LP64(assert(VM_Version::supports_sse(), ""));
4232   InstructionMark im(this);
4233   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
4234   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
4235   simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
4236   emit_int8(0x5C);
4237   emit_operand(dst, src);
4238 }
4239 
4240 void Assembler::testb(Register dst, int imm8) {
4241   NOT_LP64(assert(dst->has_byte_register(), "must have byte register"));
4242   (void) prefix_and_encode(dst->encoding(), true);
4243   emit_arith_b(0xF6, 0xC0, dst, imm8);
4244 }
4245 
4246 void Assembler::testb(Address dst, int imm8) {
4247   InstructionMark im(this);
4248   prefix(dst);
4249   emit_int8((unsigned char)0xF6);
4250   emit_operand(rax, dst, 1);
4251   emit_int8(imm8);
4252 }
4253 
4254 void Assembler::testl(Register dst, int32_t imm32) {
4255   // not using emit_arith because test
4256   // doesn't support sign-extension of
4257   // 8bit operands
4258   int encode = dst->encoding();
4259   if (encode == 0) {
4260     emit_int8((unsigned char)0xA9);
4261   } else {
4262     encode = prefix_and_encode(encode);
4263     emit_int8((unsigned char)0xF7);
4264     emit_int8((unsigned char)(0xC0 | encode));
4265   }
4266   emit_int32(imm32);
4267 }
4268 
4269 void Assembler::testl(Register dst, Register src) {
4270   (void) prefix_and_encode(dst->encoding(), src->encoding());
4271   emit_arith(0x85, 0xC0, dst, src);
4272 }
4273 
4274 void Assembler::testl(Register dst, Address src) {
4275   InstructionMark im(this);
4276   prefix(src, dst);
4277   emit_int8((unsigned char)0x85);
4278   emit_operand(dst, src);
4279 }
4280 
4281 void Assembler::tzcntl(Register dst, Register src) {
4282   assert(VM_Version::supports_bmi1(), "tzcnt instruction not supported");
4283   emit_int8((unsigned char)0xF3);
4284   int encode = prefix_and_encode(dst->encoding(), src->encoding());
4285   emit_int8(0x0F);
4286   emit_int8((unsigned char)0xBC);
4287   emit_int8((unsigned char)0xC0 | encode);
4288 }
4289 
4290 void Assembler::tzcntq(Register dst, Register src) {
4291   assert(VM_Version::supports_bmi1(), "tzcnt instruction not supported");
4292   emit_int8((unsigned char)0xF3);
4293   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
4294   emit_int8(0x0F);
4295   emit_int8((unsigned char)0xBC);
4296   emit_int8((unsigned char)(0xC0 | encode));
4297 }
4298 
4299 void Assembler::ucomisd(XMMRegister dst, Address src) {
4300   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4301   InstructionMark im(this);
4302   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4303   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
4304   attributes.set_rex_vex_w_reverted();
4305   simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4306   emit_int8(0x2E);
4307   emit_operand(dst, src);
4308 }
4309 
4310 void Assembler::ucomisd(XMMRegister dst, XMMRegister src) {
4311   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4312   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4313   attributes.set_rex_vex_w_reverted();
4314   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4315   emit_int8(0x2E);
4316   emit_int8((unsigned char)(0xC0 | encode));
4317 }
4318 
4319 void Assembler::ucomiss(XMMRegister dst, Address src) {
4320   NOT_LP64(assert(VM_Version::supports_sse(), ""));
4321   InstructionMark im(this);
4322   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4323   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
4324   simd_prefix(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
4325   emit_int8(0x2E);
4326   emit_operand(dst, src);
4327 }
4328 
4329 void Assembler::ucomiss(XMMRegister dst, XMMRegister src) {
4330   NOT_LP64(assert(VM_Version::supports_sse(), ""));
4331   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4332   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
4333   emit_int8(0x2E);
4334   emit_int8((unsigned char)(0xC0 | encode));
4335 }
4336 
4337 void Assembler::xabort(int8_t imm8) {
4338   emit_int8((unsigned char)0xC6);
4339   emit_int8((unsigned char)0xF8);
4340   emit_int8((unsigned char)(imm8 & 0xFF));
4341 }
4342 
4343 void Assembler::xaddl(Address dst, Register src) {
4344   InstructionMark im(this);
4345   prefix(dst, src);
4346   emit_int8(0x0F);
4347   emit_int8((unsigned char)0xC1);
4348   emit_operand(src, dst);
4349 }
4350 
4351 void Assembler::xbegin(Label& abort, relocInfo::relocType rtype) {
4352   InstructionMark im(this);
4353   relocate(rtype);
4354   if (abort.is_bound()) {
4355     address entry = target(abort);
4356     assert(entry != NULL, "abort entry NULL");
4357     intptr_t offset = entry - pc();
4358     emit_int8((unsigned char)0xC7);
4359     emit_int8((unsigned char)0xF8);
4360     emit_int32(offset - 6); // 2 opcode + 4 address
4361   } else {
4362     abort.add_patch_at(code(), locator());
4363     emit_int8((unsigned char)0xC7);
4364     emit_int8((unsigned char)0xF8);
4365     emit_int32(0);
4366   }
4367 }
4368 
4369 void Assembler::xchgl(Register dst, Address src) { // xchg
4370   InstructionMark im(this);
4371   prefix(src, dst);
4372   emit_int8((unsigned char)0x87);
4373   emit_operand(dst, src);
4374 }
4375 
4376 void Assembler::xchgl(Register dst, Register src) {
4377   int encode = prefix_and_encode(dst->encoding(), src->encoding());
4378   emit_int8((unsigned char)0x87);
4379   emit_int8((unsigned char)(0xC0 | encode));
4380 }
4381 
4382 void Assembler::xend() {
4383   emit_int8((unsigned char)0x0F);
4384   emit_int8((unsigned char)0x01);
4385   emit_int8((unsigned char)0xD5);
4386 }
4387 
4388 void Assembler::xgetbv() {
4389   emit_int8(0x0F);
4390   emit_int8(0x01);
4391   emit_int8((unsigned char)0xD0);
4392 }
4393 
4394 void Assembler::xorl(Register dst, int32_t imm32) {
4395   prefix(dst);
4396   emit_arith(0x81, 0xF0, dst, imm32);
4397 }
4398 
4399 void Assembler::xorl(Register dst, Address src) {
4400   InstructionMark im(this);
4401   prefix(src, dst);
4402   emit_int8(0x33);
4403   emit_operand(dst, src);
4404 }
4405 
4406 void Assembler::xorl(Register dst, Register src) {
4407   (void) prefix_and_encode(dst->encoding(), src->encoding());
4408   emit_arith(0x33, 0xC0, dst, src);
4409 }
4410 
4411 void Assembler::xorb(Register dst, Address src) {
4412   InstructionMark im(this);
4413   prefix(src, dst);
4414   emit_int8(0x32);
4415   emit_operand(dst, src);
4416 }
4417 
4418 // AVX 3-operands scalar float-point arithmetic instructions
4419 
4420 void Assembler::vaddsd(XMMRegister dst, XMMRegister nds, Address src) {
4421   assert(VM_Version::supports_avx(), "");
4422   InstructionMark im(this);
4423   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
4424   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
4425   attributes.set_rex_vex_w_reverted();
4426   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
4427   emit_int8(0x58);
4428   emit_operand(dst, src);
4429 }
4430 
4431 void Assembler::vaddsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
4432   assert(VM_Version::supports_avx(), "");
4433   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
4434   attributes.set_rex_vex_w_reverted();
4435   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
4436   emit_int8(0x58);
4437   emit_int8((unsigned char)(0xC0 | encode));
4438 }
4439 
4440 void Assembler::vaddss(XMMRegister dst, XMMRegister nds, Address src) {
4441   assert(VM_Version::supports_avx(), "");
4442   InstructionMark im(this);
4443   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
4444   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
4445   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
4446   emit_int8(0x58);
4447   emit_operand(dst, src);
4448 }
4449 
4450 void Assembler::vaddss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
4451   assert(VM_Version::supports_avx(), "");
4452   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
4453   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
4454   emit_int8(0x58);
4455   emit_int8((unsigned char)(0xC0 | encode));
4456 }
4457 
4458 void Assembler::vdivsd(XMMRegister dst, XMMRegister nds, Address src) {
4459   assert(VM_Version::supports_avx(), "");
4460   InstructionMark im(this);
4461   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
4462   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
4463   attributes.set_rex_vex_w_reverted();
4464   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
4465   emit_int8(0x5E);
4466   emit_operand(dst, src);
4467 }
4468 
4469 void Assembler::vdivsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
4470   assert(VM_Version::supports_avx(), "");
4471   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
4472   attributes.set_rex_vex_w_reverted();
4473   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
4474   emit_int8(0x5E);
4475   emit_int8((unsigned char)(0xC0 | encode));
4476 }
4477 
4478 void Assembler::vdivss(XMMRegister dst, XMMRegister nds, Address src) {
4479   assert(VM_Version::supports_avx(), "");
4480   InstructionMark im(this);
4481   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
4482   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
4483   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
4484   emit_int8(0x5E);
4485   emit_operand(dst, src);
4486 }
4487 
4488 void Assembler::vdivss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
4489   assert(VM_Version::supports_avx(), "");
4490   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
4491   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
4492   emit_int8(0x5E);
4493   emit_int8((unsigned char)(0xC0 | encode));
4494 }
4495 
4496 void Assembler::vmulsd(XMMRegister dst, XMMRegister nds, Address src) {
4497   assert(VM_Version::supports_avx(), "");
4498   InstructionMark im(this);
4499   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
4500   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
4501   attributes.set_rex_vex_w_reverted();
4502   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
4503   emit_int8(0x59);
4504   emit_operand(dst, src);
4505 }
4506 
4507 void Assembler::vmulsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
4508   assert(VM_Version::supports_avx(), "");
4509   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
4510   attributes.set_rex_vex_w_reverted();
4511   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
4512   emit_int8(0x59);
4513   emit_int8((unsigned char)(0xC0 | encode));
4514 }
4515 
4516 void Assembler::vmulss(XMMRegister dst, XMMRegister nds, Address src) {
4517   assert(VM_Version::supports_avx(), "");
4518   InstructionMark im(this);
4519   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
4520   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
4521   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
4522   emit_int8(0x59);
4523   emit_operand(dst, src);
4524 }
4525 
4526 void Assembler::vmulss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
4527   assert(VM_Version::supports_avx(), "");
4528   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
4529   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
4530   emit_int8(0x59);
4531   emit_int8((unsigned char)(0xC0 | encode));
4532 }
4533 
4534 void Assembler::vsubsd(XMMRegister dst, XMMRegister nds, Address src) {
4535   assert(VM_Version::supports_avx(), "");
4536   InstructionMark im(this);
4537   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
4538   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
4539   attributes.set_rex_vex_w_reverted();
4540   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
4541   emit_int8(0x5C);
4542   emit_operand(dst, src);
4543 }
4544 
4545 void Assembler::vsubsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
4546   assert(VM_Version::supports_avx(), "");
4547   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
4548   attributes.set_rex_vex_w_reverted();
4549   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
4550   emit_int8(0x5C);
4551   emit_int8((unsigned char)(0xC0 | encode));
4552 }
4553 
4554 void Assembler::vsubss(XMMRegister dst, XMMRegister nds, Address src) {
4555   assert(VM_Version::supports_avx(), "");
4556   InstructionMark im(this);
4557   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
4558   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
4559   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
4560   emit_int8(0x5C);
4561   emit_operand(dst, src);
4562 }
4563 
4564 void Assembler::vsubss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
4565   assert(VM_Version::supports_avx(), "");
4566   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
4567   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
4568   emit_int8(0x5C);
4569   emit_int8((unsigned char)(0xC0 | encode));
4570 }
4571 
4572 //====================VECTOR ARITHMETIC=====================================
4573 
4574 // Float-point vector arithmetic
4575 
4576 void Assembler::addpd(XMMRegister dst, XMMRegister src) {
4577   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4578   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4579   attributes.set_rex_vex_w_reverted();
4580   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4581   emit_int8(0x58);
4582   emit_int8((unsigned char)(0xC0 | encode));
4583 }
4584 
4585 void Assembler::addpd(XMMRegister dst, Address src) {
4586   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4587   InstructionMark im(this);
4588   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4589   attributes.set_rex_vex_w_reverted();
4590   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
4591   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4592   emit_int8(0x58);
4593   emit_operand(dst, src);
4594 }
4595 
4596 
4597 void Assembler::addps(XMMRegister dst, XMMRegister src) {
4598   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4599   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4600   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
4601   emit_int8(0x58);
4602   emit_int8((unsigned char)(0xC0 | encode));
4603 }
4604 
4605 void Assembler::vaddpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4606   assert(VM_Version::supports_avx(), "");
4607   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4608   attributes.set_rex_vex_w_reverted();
4609   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4610   emit_int8(0x58);
4611   emit_int8((unsigned char)(0xC0 | encode));
4612 }
4613 
4614 void Assembler::vaddps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4615   assert(VM_Version::supports_avx(), "");
4616   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4617   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
4618   emit_int8(0x58);
4619   emit_int8((unsigned char)(0xC0 | encode));
4620 }
4621 
4622 void Assembler::vaddpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4623   assert(VM_Version::supports_avx(), "");
4624   InstructionMark im(this);
4625   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4626   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
4627   attributes.set_rex_vex_w_reverted();
4628   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4629   emit_int8(0x58);
4630   emit_operand(dst, src);
4631 }
4632 
4633 void Assembler::vaddps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4634   assert(VM_Version::supports_avx(), "");
4635   InstructionMark im(this);
4636   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4637   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
4638   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
4639   emit_int8(0x58);
4640   emit_operand(dst, src);
4641 }
4642 
4643 void Assembler::subpd(XMMRegister dst, XMMRegister src) {
4644   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4645   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4646   attributes.set_rex_vex_w_reverted();
4647   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4648   emit_int8(0x5C);
4649   emit_int8((unsigned char)(0xC0 | encode));
4650 }
4651 
4652 void Assembler::subps(XMMRegister dst, XMMRegister src) {
4653   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4654   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4655   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
4656   emit_int8(0x5C);
4657   emit_int8((unsigned char)(0xC0 | encode));
4658 }
4659 
4660 void Assembler::vsubpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4661   assert(VM_Version::supports_avx(), "");
4662   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4663   attributes.set_rex_vex_w_reverted();
4664   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4665   emit_int8(0x5C);
4666   emit_int8((unsigned char)(0xC0 | encode));
4667 }
4668 
4669 void Assembler::vsubps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4670   assert(VM_Version::supports_avx(), "");
4671   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4672   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
4673   emit_int8(0x5C);
4674   emit_int8((unsigned char)(0xC0 | encode));
4675 }
4676 
4677 void Assembler::vsubpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4678   assert(VM_Version::supports_avx(), "");
4679   InstructionMark im(this);
4680   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4681   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
4682   attributes.set_rex_vex_w_reverted();
4683   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4684   emit_int8(0x5C);
4685   emit_operand(dst, src);
4686 }
4687 
4688 void Assembler::vsubps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4689   assert(VM_Version::supports_avx(), "");
4690   InstructionMark im(this);
4691   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4692   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
4693   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
4694   emit_int8(0x5C);
4695   emit_operand(dst, src);
4696 }
4697 
4698 void Assembler::mulpd(XMMRegister dst, XMMRegister src) {
4699   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4700   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4701   attributes.set_rex_vex_w_reverted();
4702   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4703   emit_int8(0x59);
4704   emit_int8((unsigned char)(0xC0 | encode));
4705 }
4706 
4707 void Assembler::mulpd(XMMRegister dst, Address src) {
4708   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4709   InstructionMark im(this);
4710   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4711   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
4712   attributes.set_rex_vex_w_reverted();
4713   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4714   emit_int8(0x59);
4715   emit_operand(dst, src);
4716 }
4717 
4718 void Assembler::mulps(XMMRegister dst, XMMRegister src) {
4719   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4720   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4721   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
4722   emit_int8(0x59);
4723   emit_int8((unsigned char)(0xC0 | encode));
4724 }
4725 
4726 void Assembler::vmulpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4727   assert(VM_Version::supports_avx(), "");
4728   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4729   attributes.set_rex_vex_w_reverted();
4730   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4731   emit_int8(0x59);
4732   emit_int8((unsigned char)(0xC0 | encode));
4733 }
4734 
4735 void Assembler::vmulps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4736   assert(VM_Version::supports_avx(), "");
4737   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4738   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
4739   emit_int8(0x59);
4740   emit_int8((unsigned char)(0xC0 | encode));
4741 }
4742 
4743 void Assembler::vmulpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4744   assert(VM_Version::supports_avx(), "");
4745   InstructionMark im(this);
4746   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4747   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
4748   attributes.set_rex_vex_w_reverted();
4749   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4750   emit_int8(0x59);
4751   emit_operand(dst, src);
4752 }
4753 
4754 void Assembler::vmulps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4755   assert(VM_Version::supports_avx(), "");
4756   InstructionMark im(this);
4757   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4758   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
4759   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
4760   emit_int8(0x59);
4761   emit_operand(dst, src);
4762 }
4763 
4764 void Assembler::divpd(XMMRegister dst, XMMRegister src) {
4765   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4766   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4767   attributes.set_rex_vex_w_reverted();
4768   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4769   emit_int8(0x5E);
4770   emit_int8((unsigned char)(0xC0 | encode));
4771 }
4772 
4773 void Assembler::divps(XMMRegister dst, XMMRegister src) {
4774   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4775   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4776   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
4777   emit_int8(0x5E);
4778   emit_int8((unsigned char)(0xC0 | encode));
4779 }
4780 
4781 void Assembler::vdivpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4782   assert(VM_Version::supports_avx(), "");
4783   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4784   attributes.set_rex_vex_w_reverted();
4785   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4786   emit_int8(0x5E);
4787   emit_int8((unsigned char)(0xC0 | encode));
4788 }
4789 
4790 void Assembler::vdivps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4791   assert(VM_Version::supports_avx(), "");
4792   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4793   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
4794   emit_int8(0x5E);
4795   emit_int8((unsigned char)(0xC0 | encode));
4796 }
4797 
4798 void Assembler::vdivpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4799   assert(VM_Version::supports_avx(), "");
4800   InstructionMark im(this);
4801   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4802   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
4803   attributes.set_rex_vex_w_reverted();
4804   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4805   emit_int8(0x5E);
4806   emit_operand(dst, src);
4807 }
4808 
4809 void Assembler::vdivps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4810   assert(VM_Version::supports_avx(), "");
4811   InstructionMark im(this);
4812   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4813   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
4814   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
4815   emit_int8(0x5E);
4816   emit_operand(dst, src);
4817 }
4818 
4819 void Assembler::vsqrtpd(XMMRegister dst, XMMRegister src, int vector_len) {
4820   assert(VM_Version::supports_avx(), "");
4821   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4822   attributes.set_rex_vex_w_reverted();
4823   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4824   emit_int8(0x51);
4825   emit_int8((unsigned char)(0xC0 | encode));
4826 }
4827 
4828 void Assembler::vsqrtpd(XMMRegister dst, Address src, int vector_len) {
4829   assert(VM_Version::supports_avx(), "");
4830   InstructionMark im(this);
4831   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4832   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
4833   attributes.set_rex_vex_w_reverted();
4834   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4835   emit_int8(0x51);
4836   emit_operand(dst, src);
4837 }
4838 
4839 void Assembler::andpd(XMMRegister dst, XMMRegister src) {
4840   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4841   InstructionAttr attributes(AVX_128bit, /* rex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
4842   attributes.set_rex_vex_w_reverted();
4843   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4844   emit_int8(0x54);
4845   emit_int8((unsigned char)(0xC0 | encode));
4846 }
4847 
4848 void Assembler::andps(XMMRegister dst, XMMRegister src) {
4849   NOT_LP64(assert(VM_Version::supports_sse(), ""));
4850   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
4851   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
4852   emit_int8(0x54);
4853   emit_int8((unsigned char)(0xC0 | encode));
4854 }
4855 
4856 void Assembler::andps(XMMRegister dst, Address src) {
4857   NOT_LP64(assert(VM_Version::supports_sse(), ""));
4858   InstructionMark im(this);
4859   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
4860   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
4861   simd_prefix(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
4862   emit_int8(0x54);
4863   emit_operand(dst, src);
4864 }
4865 
4866 void Assembler::andpd(XMMRegister dst, Address src) {
4867   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4868   InstructionMark im(this);
4869   InstructionAttr attributes(AVX_128bit, /* rex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
4870   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
4871   attributes.set_rex_vex_w_reverted();
4872   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4873   emit_int8(0x54);
4874   emit_operand(dst, src);
4875 }
4876 
4877 void Assembler::vandpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4878   assert(VM_Version::supports_avx(), "");
4879   InstructionAttr attributes(vector_len, /* vex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
4880   attributes.set_rex_vex_w_reverted();
4881   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4882   emit_int8(0x54);
4883   emit_int8((unsigned char)(0xC0 | encode));
4884 }
4885 
4886 void Assembler::vandps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4887   assert(VM_Version::supports_avx(), "");
4888   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
4889   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
4890   emit_int8(0x54);
4891   emit_int8((unsigned char)(0xC0 | encode));
4892 }
4893 
4894 void Assembler::vandpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4895   assert(VM_Version::supports_avx(), "");
4896   InstructionMark im(this);
4897   InstructionAttr attributes(vector_len, /* vex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
4898   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
4899   attributes.set_rex_vex_w_reverted();
4900   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4901   emit_int8(0x54);
4902   emit_operand(dst, src);
4903 }
4904 
4905 void Assembler::vandps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4906   assert(VM_Version::supports_avx(), "");
4907   InstructionMark im(this);
4908   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
4909   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
4910   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
4911   emit_int8(0x54);
4912   emit_operand(dst, src);
4913 }
4914 
4915 void Assembler::unpckhpd(XMMRegister dst, XMMRegister src) {
4916   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4917   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4918   attributes.set_rex_vex_w_reverted();
4919   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4920   emit_int8(0x15);
4921   emit_int8((unsigned char)(0xC0 | encode));
4922 }
4923 
4924 void Assembler::unpcklpd(XMMRegister dst, XMMRegister src) {
4925   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4926   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4927   attributes.set_rex_vex_w_reverted();
4928   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4929   emit_int8(0x14);
4930   emit_int8((unsigned char)(0xC0 | encode));
4931 }
4932 
4933 void Assembler::xorpd(XMMRegister dst, XMMRegister src) {
4934   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4935   InstructionAttr attributes(AVX_128bit, /* rex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
4936   attributes.set_rex_vex_w_reverted();
4937   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4938   emit_int8(0x57);
4939   emit_int8((unsigned char)(0xC0 | encode));
4940 }
4941 
4942 void Assembler::xorps(XMMRegister dst, XMMRegister src) {
4943   NOT_LP64(assert(VM_Version::supports_sse(), ""));
4944   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
4945   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
4946   emit_int8(0x57);
4947   emit_int8((unsigned char)(0xC0 | encode));
4948 }
4949 
4950 void Assembler::xorpd(XMMRegister dst, Address src) {
4951   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4952   InstructionMark im(this);
4953   InstructionAttr attributes(AVX_128bit, /* rex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
4954   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
4955   attributes.set_rex_vex_w_reverted();
4956   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4957   emit_int8(0x57);
4958   emit_operand(dst, src);
4959 }
4960 
4961 void Assembler::xorps(XMMRegister dst, Address src) {
4962   NOT_LP64(assert(VM_Version::supports_sse(), ""));
4963   InstructionMark im(this);
4964   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
4965   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
4966   simd_prefix(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
4967   emit_int8(0x57);
4968   emit_operand(dst, src);
4969 }
4970 
4971 void Assembler::vxorpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4972   assert(VM_Version::supports_avx(), "");
4973   InstructionAttr attributes(vector_len, /* vex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
4974   attributes.set_rex_vex_w_reverted();
4975   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4976   emit_int8(0x57);
4977   emit_int8((unsigned char)(0xC0 | encode));
4978 }
4979 
4980 void Assembler::vxorps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4981   assert(VM_Version::supports_avx(), "");
4982   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
4983   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
4984   emit_int8(0x57);
4985   emit_int8((unsigned char)(0xC0 | encode));
4986 }
4987 
4988 void Assembler::vxorpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4989   assert(VM_Version::supports_avx(), "");
4990   InstructionMark im(this);
4991   InstructionAttr attributes(vector_len, /* vex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
4992   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
4993   attributes.set_rex_vex_w_reverted();
4994   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4995   emit_int8(0x57);
4996   emit_operand(dst, src);
4997 }
4998 
4999 void Assembler::vxorps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5000   assert(VM_Version::supports_avx(), "");
5001   InstructionMark im(this);
5002   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
5003   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
5004   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
5005   emit_int8(0x57);
5006   emit_operand(dst, src);
5007 }
5008 
5009 // Integer vector arithmetic
5010 void Assembler::vphaddw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5011   assert(VM_Version::supports_avx() && (vector_len == 0) ||
5012          VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
5013   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
5014   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5015   emit_int8(0x01);
5016   emit_int8((unsigned char)(0xC0 | encode));
5017 }
5018 
5019 void Assembler::vphaddd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5020   assert(VM_Version::supports_avx() && (vector_len == 0) ||
5021          VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
5022   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
5023   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5024   emit_int8(0x02);
5025   emit_int8((unsigned char)(0xC0 | encode));
5026 }
5027 
5028 void Assembler::paddb(XMMRegister dst, XMMRegister src) {
5029   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5030   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5031   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5032   emit_int8((unsigned char)0xFC);
5033   emit_int8((unsigned char)(0xC0 | encode));
5034 }
5035 
5036 void Assembler::paddw(XMMRegister dst, XMMRegister src) {
5037   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5038   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5039   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5040   emit_int8((unsigned char)0xFD);
5041   emit_int8((unsigned char)(0xC0 | encode));
5042 }
5043 
5044 void Assembler::paddd(XMMRegister dst, XMMRegister src) {
5045   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5046   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5047   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5048   emit_int8((unsigned char)0xFE);
5049   emit_int8((unsigned char)(0xC0 | encode));
5050 }
5051 
5052 void Assembler::paddd(XMMRegister dst, Address src) {
5053   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5054   InstructionMark im(this);
5055   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5056   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5057   emit_int8((unsigned char)0xFE);
5058   emit_operand(dst, src);
5059 }
5060 
5061 void Assembler::paddq(XMMRegister dst, XMMRegister src) {
5062   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5063   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5064   attributes.set_rex_vex_w_reverted();
5065   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5066   emit_int8((unsigned char)0xD4);
5067   emit_int8((unsigned char)(0xC0 | encode));
5068 }
5069 
5070 void Assembler::phaddw(XMMRegister dst, XMMRegister src) {
5071   NOT_LP64(assert(VM_Version::supports_sse3(), ""));
5072   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
5073   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5074   emit_int8(0x01);
5075   emit_int8((unsigned char)(0xC0 | encode));
5076 }
5077 
5078 void Assembler::phaddd(XMMRegister dst, XMMRegister src) {
5079   NOT_LP64(assert(VM_Version::supports_sse3(), ""));
5080   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
5081   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5082   emit_int8(0x02);
5083   emit_int8((unsigned char)(0xC0 | encode));
5084 }
5085 
5086 void Assembler::vpaddb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5087   assert(UseAVX > 0, "requires some form of AVX");
5088   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5089   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5090   emit_int8((unsigned char)0xFC);
5091   emit_int8((unsigned char)(0xC0 | encode));
5092 }
5093 
5094 void Assembler::vpaddw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5095   assert(UseAVX > 0, "requires some form of AVX");
5096   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5097   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5098   emit_int8((unsigned char)0xFD);
5099   emit_int8((unsigned char)(0xC0 | encode));
5100 }
5101 
5102 void Assembler::vpaddd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5103   assert(UseAVX > 0, "requires some form of AVX");
5104   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5105   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5106   emit_int8((unsigned char)0xFE);
5107   emit_int8((unsigned char)(0xC0 | encode));
5108 }
5109 
5110 void Assembler::vpaddq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5111   assert(UseAVX > 0, "requires some form of AVX");
5112   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5113   attributes.set_rex_vex_w_reverted();
5114   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5115   emit_int8((unsigned char)0xD4);
5116   emit_int8((unsigned char)(0xC0 | encode));
5117 }
5118 
5119 void Assembler::vpaddb(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5120   assert(UseAVX > 0, "requires some form of AVX");
5121   InstructionMark im(this);
5122   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5123   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
5124   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5125   emit_int8((unsigned char)0xFC);
5126   emit_operand(dst, src);
5127 }
5128 
5129 void Assembler::vpaddw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5130   assert(UseAVX > 0, "requires some form of AVX");
5131   InstructionMark im(this);
5132   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5133   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
5134   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5135   emit_int8((unsigned char)0xFD);
5136   emit_operand(dst, src);
5137 }
5138 
5139 void Assembler::vpaddd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5140   assert(UseAVX > 0, "requires some form of AVX");
5141   InstructionMark im(this);
5142   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5143   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
5144   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5145   emit_int8((unsigned char)0xFE);
5146   emit_operand(dst, src);
5147 }
5148 
5149 void Assembler::vpaddq(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5150   assert(UseAVX > 0, "requires some form of AVX");
5151   InstructionMark im(this);
5152   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5153   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
5154   attributes.set_rex_vex_w_reverted();
5155   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5156   emit_int8((unsigned char)0xD4);
5157   emit_operand(dst, src);
5158 }
5159 
5160 void Assembler::psubb(XMMRegister dst, XMMRegister src) {
5161   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5162   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5163   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5164   emit_int8((unsigned char)0xF8);
5165   emit_int8((unsigned char)(0xC0 | encode));
5166 }
5167 
5168 void Assembler::psubw(XMMRegister dst, XMMRegister src) {
5169   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5170   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5171   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5172   emit_int8((unsigned char)0xF9);
5173   emit_int8((unsigned char)(0xC0 | encode));
5174 }
5175 
5176 void Assembler::psubd(XMMRegister dst, XMMRegister src) {
5177   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5178   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5179   emit_int8((unsigned char)0xFA);
5180   emit_int8((unsigned char)(0xC0 | encode));
5181 }
5182 
5183 void Assembler::psubq(XMMRegister dst, XMMRegister src) {
5184   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5185   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5186   attributes.set_rex_vex_w_reverted();
5187   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5188   emit_int8((unsigned char)0xFB);
5189   emit_int8((unsigned char)(0xC0 | encode));
5190 }
5191 
5192 void Assembler::vpsubb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5193   assert(UseAVX > 0, "requires some form of AVX");
5194   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5195   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5196   emit_int8((unsigned char)0xF8);
5197   emit_int8((unsigned char)(0xC0 | encode));
5198 }
5199 
5200 void Assembler::vpsubw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5201   assert(UseAVX > 0, "requires some form of AVX");
5202   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5203   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5204   emit_int8((unsigned char)0xF9);
5205   emit_int8((unsigned char)(0xC0 | encode));
5206 }
5207 
5208 void Assembler::vpsubd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5209   assert(UseAVX > 0, "requires some form of AVX");
5210   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5211   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5212   emit_int8((unsigned char)0xFA);
5213   emit_int8((unsigned char)(0xC0 | encode));
5214 }
5215 
5216 void Assembler::vpsubq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5217   assert(UseAVX > 0, "requires some form of AVX");
5218   InstructionAttr attributes(vector_len, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5219   attributes.set_rex_vex_w_reverted();
5220   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5221   emit_int8((unsigned char)0xFB);
5222   emit_int8((unsigned char)(0xC0 | encode));
5223 }
5224 
5225 void Assembler::vpsubb(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5226   assert(UseAVX > 0, "requires some form of AVX");
5227   InstructionMark im(this);
5228   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5229   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
5230   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5231   emit_int8((unsigned char)0xF8);
5232   emit_operand(dst, src);
5233 }
5234 
5235 void Assembler::vpsubw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5236   assert(UseAVX > 0, "requires some form of AVX");
5237   InstructionMark im(this);
5238   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5239   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
5240   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5241   emit_int8((unsigned char)0xF9);
5242   emit_operand(dst, src);
5243 }
5244 
5245 void Assembler::vpsubd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5246   assert(UseAVX > 0, "requires some form of AVX");
5247   InstructionMark im(this);
5248   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5249   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
5250   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5251   emit_int8((unsigned char)0xFA);
5252   emit_operand(dst, src);
5253 }
5254 
5255 void Assembler::vpsubq(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5256   assert(UseAVX > 0, "requires some form of AVX");
5257   InstructionMark im(this);
5258   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5259   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
5260   attributes.set_rex_vex_w_reverted();
5261   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5262   emit_int8((unsigned char)0xFB);
5263   emit_operand(dst, src);
5264 }
5265 
5266 void Assembler::pmullw(XMMRegister dst, XMMRegister src) {
5267   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5268   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5269   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5270   emit_int8((unsigned char)0xD5);
5271   emit_int8((unsigned char)(0xC0 | encode));
5272 }
5273 
5274 void Assembler::pmulld(XMMRegister dst, XMMRegister src) {
5275   assert(VM_Version::supports_sse4_1(), "");
5276   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5277   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5278   emit_int8(0x40);
5279   emit_int8((unsigned char)(0xC0 | encode));
5280 }
5281 
5282 void Assembler::vpmullw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5283   assert(UseAVX > 0, "requires some form of AVX");
5284   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5285   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5286   emit_int8((unsigned char)0xD5);
5287   emit_int8((unsigned char)(0xC0 | encode));
5288 }
5289 
5290 void Assembler::vpmulld(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5291   assert(UseAVX > 0, "requires some form of AVX");
5292   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5293   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5294   emit_int8(0x40);
5295   emit_int8((unsigned char)(0xC0 | encode));
5296 }
5297 
5298 void Assembler::vpmullq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5299   assert(UseAVX > 2, "requires some form of EVEX");
5300   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
5301   attributes.set_is_evex_instruction();
5302   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5303   emit_int8(0x40);
5304   emit_int8((unsigned char)(0xC0 | encode));
5305 }
5306 
5307 void Assembler::vpmullw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5308   assert(UseAVX > 0, "requires some form of AVX");
5309   InstructionMark im(this);
5310   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5311   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
5312   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5313   emit_int8((unsigned char)0xD5);
5314   emit_operand(dst, src);
5315 }
5316 
5317 void Assembler::vpmulld(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5318   assert(UseAVX > 0, "requires some form of AVX");
5319   InstructionMark im(this);
5320   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5321   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
5322   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5323   emit_int8(0x40);
5324   emit_operand(dst, src);
5325 }
5326 
5327 void Assembler::vpmullq(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5328   assert(UseAVX > 2, "requires some form of EVEX");
5329   InstructionMark im(this);
5330   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
5331   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
5332   attributes.set_is_evex_instruction();
5333   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5334   emit_int8(0x40);
5335   emit_operand(dst, src);
5336 }
5337 
5338 // Shift packed integers left by specified number of bits.
5339 void Assembler::psllw(XMMRegister dst, int shift) {
5340   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5341   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5342   // XMM6 is for /6 encoding: 66 0F 71 /6 ib
5343   int encode = simd_prefix_and_encode(xmm6, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5344   emit_int8(0x71);
5345   emit_int8((unsigned char)(0xC0 | encode));
5346   emit_int8(shift & 0xFF);
5347 }
5348 
5349 void Assembler::pslld(XMMRegister dst, int shift) {
5350   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5351   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5352   // XMM6 is for /6 encoding: 66 0F 72 /6 ib
5353   int encode = simd_prefix_and_encode(xmm6, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5354   emit_int8(0x72);
5355   emit_int8((unsigned char)(0xC0 | encode));
5356   emit_int8(shift & 0xFF);
5357 }
5358 
5359 void Assembler::psllq(XMMRegister dst, int shift) {
5360   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5361   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5362   // XMM6 is for /6 encoding: 66 0F 73 /6 ib
5363   int encode = simd_prefix_and_encode(xmm6, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5364   emit_int8(0x73);
5365   emit_int8((unsigned char)(0xC0 | encode));
5366   emit_int8(shift & 0xFF);
5367 }
5368 
5369 void Assembler::psllw(XMMRegister dst, XMMRegister shift) {
5370   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5371   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5372   int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5373   emit_int8((unsigned char)0xF1);
5374   emit_int8((unsigned char)(0xC0 | encode));
5375 }
5376 
5377 void Assembler::pslld(XMMRegister dst, XMMRegister shift) {
5378   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5379   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5380   int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5381   emit_int8((unsigned char)0xF2);
5382   emit_int8((unsigned char)(0xC0 | encode));
5383 }
5384 
5385 void Assembler::psllq(XMMRegister dst, XMMRegister shift) {
5386   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5387   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5388   attributes.set_rex_vex_w_reverted();
5389   int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5390   emit_int8((unsigned char)0xF3);
5391   emit_int8((unsigned char)(0xC0 | encode));
5392 }
5393 
5394 void Assembler::vpsllw(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
5395   assert(UseAVX > 0, "requires some form of AVX");
5396   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5397   // XMM6 is for /6 encoding: 66 0F 71 /6 ib
5398   int encode = vex_prefix_and_encode(xmm6->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5399   emit_int8(0x71);
5400   emit_int8((unsigned char)(0xC0 | encode));
5401   emit_int8(shift & 0xFF);
5402 }
5403 
5404 void Assembler::vpslld(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
5405   assert(UseAVX > 0, "requires some form of AVX");
5406   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5407   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5408   // XMM6 is for /6 encoding: 66 0F 72 /6 ib
5409   int encode = vex_prefix_and_encode(xmm6->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5410   emit_int8(0x72);
5411   emit_int8((unsigned char)(0xC0 | encode));
5412   emit_int8(shift & 0xFF);
5413 }
5414 
5415 void Assembler::vpsllq(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
5416   assert(UseAVX > 0, "requires some form of AVX");
5417   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5418   attributes.set_rex_vex_w_reverted();
5419   // XMM6 is for /6 encoding: 66 0F 73 /6 ib
5420   int encode = vex_prefix_and_encode(xmm6->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5421   emit_int8(0x73);
5422   emit_int8((unsigned char)(0xC0 | encode));
5423   emit_int8(shift & 0xFF);
5424 }
5425 
5426 void Assembler::vpsllw(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
5427   assert(UseAVX > 0, "requires some form of AVX");
5428   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5429   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5430   emit_int8((unsigned char)0xF1);
5431   emit_int8((unsigned char)(0xC0 | encode));
5432 }
5433 
5434 void Assembler::vpslld(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
5435   assert(UseAVX > 0, "requires some form of AVX");
5436   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5437   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5438   emit_int8((unsigned char)0xF2);
5439   emit_int8((unsigned char)(0xC0 | encode));
5440 }
5441 
5442 void Assembler::vpsllq(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
5443   assert(UseAVX > 0, "requires some form of AVX");
5444   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5445   attributes.set_rex_vex_w_reverted();
5446   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5447   emit_int8((unsigned char)0xF3);
5448   emit_int8((unsigned char)(0xC0 | encode));
5449 }
5450 
5451 // Shift packed integers logically right by specified number of bits.
5452 void Assembler::psrlw(XMMRegister dst, int shift) {
5453   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5454   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5455   // XMM2 is for /2 encoding: 66 0F 71 /2 ib
5456   int encode = simd_prefix_and_encode(xmm2, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5457   emit_int8(0x71);
5458   emit_int8((unsigned char)(0xC0 | encode));
5459   emit_int8(shift & 0xFF);
5460 }
5461 
5462 void Assembler::psrld(XMMRegister dst, int shift) {
5463   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5464   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5465   // XMM2 is for /2 encoding: 66 0F 72 /2 ib
5466   int encode = simd_prefix_and_encode(xmm2, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5467   emit_int8(0x72);
5468   emit_int8((unsigned char)(0xC0 | encode));
5469   emit_int8(shift & 0xFF);
5470 }
5471 
5472 void Assembler::psrlq(XMMRegister dst, int shift) {
5473   // Do not confuse it with psrldq SSE2 instruction which
5474   // shifts 128 bit value in xmm register by number of bytes.
5475   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5476   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5477   attributes.set_rex_vex_w_reverted();
5478   // XMM2 is for /2 encoding: 66 0F 73 /2 ib
5479   int encode = simd_prefix_and_encode(xmm2, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5480   emit_int8(0x73);
5481   emit_int8((unsigned char)(0xC0 | encode));
5482   emit_int8(shift & 0xFF);
5483 }
5484 
5485 void Assembler::psrlw(XMMRegister dst, XMMRegister shift) {
5486   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5487   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5488   int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5489   emit_int8((unsigned char)0xD1);
5490   emit_int8((unsigned char)(0xC0 | encode));
5491 }
5492 
5493 void Assembler::psrld(XMMRegister dst, XMMRegister shift) {
5494   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5495   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5496   int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5497   emit_int8((unsigned char)0xD2);
5498   emit_int8((unsigned char)(0xC0 | encode));
5499 }
5500 
5501 void Assembler::psrlq(XMMRegister dst, XMMRegister shift) {
5502   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5503   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5504   attributes.set_rex_vex_w_reverted();
5505   int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5506   emit_int8((unsigned char)0xD3);
5507   emit_int8((unsigned char)(0xC0 | encode));
5508 }
5509 
5510 void Assembler::vpsrlw(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
5511   assert(UseAVX > 0, "requires some form of AVX");
5512   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5513   // XMM2 is for /2 encoding: 66 0F 71 /2 ib
5514   int encode = vex_prefix_and_encode(xmm2->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5515   emit_int8(0x71);
5516   emit_int8((unsigned char)(0xC0 | encode));
5517   emit_int8(shift & 0xFF);
5518 }
5519 
5520 void Assembler::vpsrld(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
5521   assert(UseAVX > 0, "requires some form of AVX");
5522   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5523   // XMM2 is for /2 encoding: 66 0F 72 /2 ib
5524   int encode = vex_prefix_and_encode(xmm2->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5525   emit_int8(0x72);
5526   emit_int8((unsigned char)(0xC0 | encode));
5527   emit_int8(shift & 0xFF);
5528 }
5529 
5530 void Assembler::vpsrlq(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
5531   assert(UseAVX > 0, "requires some form of AVX");
5532   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5533   attributes.set_rex_vex_w_reverted();
5534   // XMM2 is for /2 encoding: 66 0F 73 /2 ib
5535   int encode = vex_prefix_and_encode(xmm2->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5536   emit_int8(0x73);
5537   emit_int8((unsigned char)(0xC0 | encode));
5538   emit_int8(shift & 0xFF);
5539 }
5540 
5541 void Assembler::vpsrlw(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
5542   assert(UseAVX > 0, "requires some form of AVX");
5543   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5544   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5545   emit_int8((unsigned char)0xD1);
5546   emit_int8((unsigned char)(0xC0 | encode));
5547 }
5548 
5549 void Assembler::vpsrld(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
5550   assert(UseAVX > 0, "requires some form of AVX");
5551   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5552   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5553   emit_int8((unsigned char)0xD2);
5554   emit_int8((unsigned char)(0xC0 | encode));
5555 }
5556 
5557 void Assembler::vpsrlq(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
5558   assert(UseAVX > 0, "requires some form of AVX");
5559   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5560   attributes.set_rex_vex_w_reverted();
5561   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5562   emit_int8((unsigned char)0xD3);
5563   emit_int8((unsigned char)(0xC0 | encode));
5564 }
5565 
5566 // Shift packed integers arithmetically right by specified number of bits.
5567 void Assembler::psraw(XMMRegister dst, int shift) {
5568   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5569   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5570   // XMM4 is for /4 encoding: 66 0F 71 /4 ib
5571   int encode = simd_prefix_and_encode(xmm4, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5572   emit_int8(0x71);
5573   emit_int8((unsigned char)(0xC0 | encode));
5574   emit_int8(shift & 0xFF);
5575 }
5576 
5577 void Assembler::psrad(XMMRegister dst, int shift) {
5578   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5579   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5580   // XMM4 is for /4 encoding: 66 0F 72 /4 ib
5581   int encode = simd_prefix_and_encode(xmm4, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5582   emit_int8(0x72);
5583   emit_int8((unsigned char)(0xC0 | encode));
5584   emit_int8(shift & 0xFF);
5585 }
5586 
5587 void Assembler::psraw(XMMRegister dst, XMMRegister shift) {
5588   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5589   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5590   int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5591   emit_int8((unsigned char)0xE1);
5592   emit_int8((unsigned char)(0xC0 | encode));
5593 }
5594 
5595 void Assembler::psrad(XMMRegister dst, XMMRegister shift) {
5596   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5597   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5598   int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5599   emit_int8((unsigned char)0xE2);
5600   emit_int8((unsigned char)(0xC0 | encode));
5601 }
5602 
5603 void Assembler::vpsraw(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
5604   assert(UseAVX > 0, "requires some form of AVX");
5605   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5606   // XMM4 is for /4 encoding: 66 0F 71 /4 ib
5607   int encode = vex_prefix_and_encode(xmm4->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5608   emit_int8(0x71);
5609   emit_int8((unsigned char)(0xC0 | encode));
5610   emit_int8(shift & 0xFF);
5611 }
5612 
5613 void Assembler::vpsrad(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
5614   assert(UseAVX > 0, "requires some form of AVX");
5615   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5616   // XMM4 is for /4 encoding: 66 0F 71 /4 ib
5617   int encode = vex_prefix_and_encode(xmm4->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5618   emit_int8(0x72);
5619   emit_int8((unsigned char)(0xC0 | encode));
5620   emit_int8(shift & 0xFF);
5621 }
5622 
5623 void Assembler::vpsraw(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
5624   assert(UseAVX > 0, "requires some form of AVX");
5625   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5626   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5627   emit_int8((unsigned char)0xE1);
5628   emit_int8((unsigned char)(0xC0 | encode));
5629 }
5630 
5631 void Assembler::vpsrad(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
5632   assert(UseAVX > 0, "requires some form of AVX");
5633   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5634   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5635   emit_int8((unsigned char)0xE2);
5636   emit_int8((unsigned char)(0xC0 | encode));
5637 }
5638 
5639 
5640 // logical operations packed integers
5641 void Assembler::pand(XMMRegister dst, XMMRegister src) {
5642   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5643   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5644   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5645   emit_int8((unsigned char)0xDB);
5646   emit_int8((unsigned char)(0xC0 | encode));
5647 }
5648 
5649 void Assembler::vpand(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5650   assert(UseAVX > 0, "requires some form of AVX");
5651   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5652   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5653   emit_int8((unsigned char)0xDB);
5654   emit_int8((unsigned char)(0xC0 | encode));
5655 }
5656 
5657 void Assembler::vpand(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5658   assert(UseAVX > 0, "requires some form of AVX");
5659   InstructionMark im(this);
5660   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5661   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
5662   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5663   emit_int8((unsigned char)0xDB);
5664   emit_operand(dst, src);
5665 }
5666 
5667 void Assembler::pandn(XMMRegister dst, XMMRegister src) {
5668   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5669   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5670   attributes.set_rex_vex_w_reverted();
5671   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5672   emit_int8((unsigned char)0xDF);
5673   emit_int8((unsigned char)(0xC0 | encode));
5674 }
5675 
5676 void Assembler::por(XMMRegister dst, XMMRegister src) {
5677   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5678   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5679   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5680   emit_int8((unsigned char)0xEB);
5681   emit_int8((unsigned char)(0xC0 | encode));
5682 }
5683 
5684 void Assembler::vpor(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5685   assert(UseAVX > 0, "requires some form of AVX");
5686   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5687   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5688   emit_int8((unsigned char)0xEB);
5689   emit_int8((unsigned char)(0xC0 | encode));
5690 }
5691 
5692 void Assembler::vpor(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5693   assert(UseAVX > 0, "requires some form of AVX");
5694   InstructionMark im(this);
5695   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5696   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
5697   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5698   emit_int8((unsigned char)0xEB);
5699   emit_operand(dst, src);
5700 }
5701 
5702 void Assembler::pxor(XMMRegister dst, XMMRegister src) {
5703   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5704   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5705   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5706   emit_int8((unsigned char)0xEF);
5707   emit_int8((unsigned char)(0xC0 | encode));
5708 }
5709 
5710 void Assembler::vpxor(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5711   assert(UseAVX > 0, "requires some form of AVX");
5712   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5713   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5714   emit_int8((unsigned char)0xEF);
5715   emit_int8((unsigned char)(0xC0 | encode));
5716 }
5717 
5718 void Assembler::vpxor(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5719   assert(UseAVX > 0, "requires some form of AVX");
5720   InstructionMark im(this);
5721   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5722   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
5723   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5724   emit_int8((unsigned char)0xEF);
5725   emit_operand(dst, src);
5726 }
5727 
5728 
5729 // vinserti forms
5730 
5731 void Assembler::vinserti128(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) {
5732   assert(VM_Version::supports_avx2(), "");
5733   assert(imm8 <= 0x01, "imm8: %u", imm8);
5734   int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_256bit;
5735   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
5736   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
5737   emit_int8(0x38);
5738   emit_int8((unsigned char)(0xC0 | encode));
5739   // 0x00 - insert into lower 128 bits
5740   // 0x01 - insert into upper 128 bits
5741   emit_int8(imm8 & 0x01);
5742 }
5743 
5744 void Assembler::vinserti128(XMMRegister dst, XMMRegister nds, Address src, uint8_t imm8) {
5745   assert(VM_Version::supports_avx2(), "");
5746   assert(dst != xnoreg, "sanity");
5747   assert(imm8 <= 0x01, "imm8: %u", imm8);
5748   int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_256bit;
5749   InstructionMark im(this);
5750   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
5751   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
5752   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
5753   emit_int8(0x38);
5754   emit_operand(dst, src);
5755   // 0x00 - insert into lower 128 bits
5756   // 0x01 - insert into upper 128 bits
5757   emit_int8(imm8 & 0x01);
5758 }
5759 
5760 void Assembler::vinserti32x4(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) {
5761   assert(VM_Version::supports_evex(), "");
5762   assert(imm8 <= 0x03, "imm8: %u", imm8);
5763   InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
5764   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
5765   emit_int8(0x38);
5766   emit_int8((unsigned char)(0xC0 | encode));
5767   // 0x00 - insert into q0 128 bits (0..127)
5768   // 0x01 - insert into q1 128 bits (128..255)
5769   // 0x02 - insert into q2 128 bits (256..383)
5770   // 0x03 - insert into q3 128 bits (384..511)
5771   emit_int8(imm8 & 0x03);
5772 }
5773 
5774 void Assembler::vinserti32x4(XMMRegister dst, XMMRegister nds, Address src, uint8_t imm8) {
5775   assert(VM_Version::supports_avx(), "");
5776   assert(dst != xnoreg, "sanity");
5777   assert(imm8 <= 0x03, "imm8: %u", imm8);
5778   int vector_len = VM_Version::supports_evex() ? AVX_512bit : AVX_256bit;
5779   InstructionMark im(this);
5780   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
5781   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
5782   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
5783   emit_int8(0x18);
5784   emit_operand(dst, src);
5785   // 0x00 - insert into q0 128 bits (0..127)
5786   // 0x01 - insert into q1 128 bits (128..255)
5787   // 0x02 - insert into q2 128 bits (256..383)
5788   // 0x03 - insert into q3 128 bits (384..511)
5789   emit_int8(imm8 & 0x03);
5790 }
5791 
5792 void Assembler::vinserti64x4(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) {
5793   assert(VM_Version::supports_evex(), "");
5794   assert(imm8 <= 0x01, "imm8: %u", imm8);
5795   InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
5796   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
5797   emit_int8(0x38);
5798   emit_int8((unsigned char)(0xC0 | encode));
5799   // 0x00 - insert into lower 256 bits
5800   // 0x01 - insert into upper 256 bits
5801   emit_int8(imm8 & 0x01);
5802 }
5803 
5804 
5805 // vinsertf forms
5806 
5807 void Assembler::vinsertf128(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) {
5808   assert(VM_Version::supports_avx(), "");
5809   assert(imm8 <= 0x01, "imm8: %u", imm8);
5810   int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_256bit;
5811   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
5812   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
5813   emit_int8(0x18);
5814   emit_int8((unsigned char)(0xC0 | encode));
5815   // 0x00 - insert into lower 128 bits
5816   // 0x01 - insert into upper 128 bits
5817   emit_int8(imm8 & 0x01);
5818 }
5819 
5820 void Assembler::vinsertf128(XMMRegister dst, XMMRegister nds, Address src, uint8_t imm8) {
5821   assert(VM_Version::supports_avx(), "");
5822   assert(dst != xnoreg, "sanity");
5823   assert(imm8 <= 0x01, "imm8: %u", imm8);
5824   int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_256bit;
5825   InstructionMark im(this);
5826   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
5827   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
5828   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
5829   emit_int8(0x18);
5830   emit_operand(dst, src);
5831   // 0x00 - insert into lower 128 bits
5832   // 0x01 - insert into upper 128 bits
5833   emit_int8(imm8 & 0x01);
5834 }
5835 
5836 void Assembler::vinsertf32x4(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) {
5837   assert(VM_Version::supports_evex(), "");
5838   assert(imm8 <= 0x03, "imm8: %u", imm8);
5839   InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
5840   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
5841   emit_int8(0x18);
5842   emit_int8((unsigned char)(0xC0 | encode));
5843   // 0x00 - insert into q0 128 bits (0..127)
5844   // 0x01 - insert into q1 128 bits (128..255)
5845   // 0x02 - insert into q2 128 bits (256..383)
5846   // 0x03 - insert into q3 128 bits (384..511)
5847   emit_int8(imm8 & 0x03);
5848 }
5849 
5850 void Assembler::vinsertf32x4(XMMRegister dst, XMMRegister nds, Address src, uint8_t imm8) {
5851   assert(VM_Version::supports_avx(), "");
5852   assert(dst != xnoreg, "sanity");
5853   assert(imm8 <= 0x03, "imm8: %u", imm8);
5854   int vector_len = VM_Version::supports_evex() ? AVX_512bit : AVX_256bit;
5855   InstructionMark im(this);
5856   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
5857   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
5858   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
5859   emit_int8(0x18);
5860   emit_operand(dst, src);
5861   // 0x00 - insert into q0 128 bits (0..127)
5862   // 0x01 - insert into q1 128 bits (128..255)
5863   // 0x02 - insert into q2 128 bits (256..383)
5864   // 0x03 - insert into q3 128 bits (384..511)
5865   emit_int8(imm8 & 0x03);
5866 }
5867 
5868 void Assembler::vinsertf64x4(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) {
5869   assert(VM_Version::supports_evex(), "");
5870   assert(imm8 <= 0x01, "imm8: %u", imm8);
5871   InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
5872   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
5873   emit_int8(0x1A);
5874   emit_int8((unsigned char)(0xC0 | encode));
5875   // 0x00 - insert into lower 256 bits
5876   // 0x01 - insert into upper 256 bits
5877   emit_int8(imm8 & 0x01);
5878 }
5879 
5880 void Assembler::vinsertf64x4(XMMRegister dst, XMMRegister nds, Address src, uint8_t imm8) {
5881   assert(VM_Version::supports_evex(), "");
5882   assert(dst != xnoreg, "sanity");
5883   assert(imm8 <= 0x01, "imm8: %u", imm8);
5884   InstructionMark im(this);
5885   InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
5886   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_64bit);
5887   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
5888   emit_int8(0x1A);
5889   emit_operand(dst, src);
5890   // 0x00 - insert into lower 256 bits
5891   // 0x01 - insert into upper 256 bits
5892   emit_int8(imm8 & 0x01);
5893 }
5894 
5895 
5896 // vextracti forms
5897 
5898 void Assembler::vextracti128(XMMRegister dst, XMMRegister src, uint8_t imm8) {
5899   assert(VM_Version::supports_avx(), "");
5900   assert(imm8 <= 0x01, "imm8: %u", imm8);
5901   int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_256bit;
5902   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
5903   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
5904   emit_int8(0x39);
5905   emit_int8((unsigned char)(0xC0 | encode));
5906   // 0x00 - extract from lower 128 bits
5907   // 0x01 - extract from upper 128 bits
5908   emit_int8(imm8 & 0x01);
5909 }
5910 
5911 void Assembler::vextracti128(Address dst, XMMRegister src, uint8_t imm8) {
5912   assert(VM_Version::supports_avx2(), "");
5913   assert(src != xnoreg, "sanity");
5914   assert(imm8 <= 0x01, "imm8: %u", imm8);
5915   int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_256bit;
5916   InstructionMark im(this);
5917   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
5918   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
5919   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
5920   emit_int8(0x39);
5921   emit_operand(src, dst);
5922   // 0x00 - extract from lower 128 bits
5923   // 0x01 - extract from upper 128 bits
5924   emit_int8(imm8 & 0x01);
5925 }
5926 
5927 void Assembler::vextracti32x4(XMMRegister dst, XMMRegister src, uint8_t imm8) {
5928   assert(VM_Version::supports_avx(), "");
5929   assert(imm8 <= 0x03, "imm8: %u", imm8);
5930   int vector_len = VM_Version::supports_evex() ? AVX_512bit : AVX_256bit;
5931   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
5932   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
5933   emit_int8(0x39);
5934   emit_int8((unsigned char)(0xC0 | encode));
5935   // 0x00 - extract from bits 127:0
5936   // 0x01 - extract from bits 255:128
5937   // 0x02 - extract from bits 383:256
5938   // 0x03 - extract from bits 511:384
5939   emit_int8(imm8 & 0x03);
5940 }
5941 
5942 void Assembler::vextracti32x4(Address dst, XMMRegister src, uint8_t imm8) {
5943   assert(VM_Version::supports_evex(), "");
5944   assert(src != xnoreg, "sanity");
5945   assert(imm8 <= 0x03, "imm8: %u", imm8);
5946   InstructionMark im(this);
5947   InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
5948   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
5949   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
5950   emit_int8(0x39);
5951   emit_operand(src, dst);
5952   // 0x00 - extract from bits 127:0
5953   // 0x01 - extract from bits 255:128
5954   // 0x02 - extract from bits 383:256
5955   // 0x03 - extract from bits 511:384
5956   emit_int8(imm8 & 0x03);
5957 }
5958 
5959 void Assembler::vextracti64x2(XMMRegister dst, XMMRegister src, uint8_t imm8) {
5960   assert(VM_Version::supports_avx512dq(), "");
5961   assert(imm8 <= 0x03, "imm8: %u", imm8);
5962   InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
5963   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
5964   emit_int8(0x39);
5965   emit_int8((unsigned char)(0xC0 | encode));
5966   // 0x00 - extract from bits 127:0
5967   // 0x01 - extract from bits 255:128
5968   // 0x02 - extract from bits 383:256
5969   // 0x03 - extract from bits 511:384
5970   emit_int8(imm8 & 0x03);
5971 }
5972 
5973 void Assembler::vextracti64x4(XMMRegister dst, XMMRegister src, uint8_t imm8) {
5974   assert(VM_Version::supports_evex(), "");
5975   assert(imm8 <= 0x01, "imm8: %u", imm8);
5976   InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
5977   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
5978   emit_int8(0x3B);
5979   emit_int8((unsigned char)(0xC0 | encode));
5980   // 0x00 - extract from lower 256 bits
5981   // 0x01 - extract from upper 256 bits
5982   emit_int8(imm8 & 0x01);
5983 }
5984 
5985 
5986 // vextractf forms
5987 
5988 void Assembler::vextractf128(XMMRegister dst, XMMRegister src, uint8_t imm8) {
5989   assert(VM_Version::supports_avx(), "");
5990   assert(imm8 <= 0x01, "imm8: %u", imm8);
5991   int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_256bit;
5992   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
5993   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
5994   emit_int8(0x19);
5995   emit_int8((unsigned char)(0xC0 | encode));
5996   // 0x00 - extract from lower 128 bits
5997   // 0x01 - extract from upper 128 bits
5998   emit_int8(imm8 & 0x01);
5999 }
6000 
6001 void Assembler::vextractf128(Address dst, XMMRegister src, uint8_t imm8) {
6002   assert(VM_Version::supports_avx(), "");
6003   assert(src != xnoreg, "sanity");
6004   assert(imm8 <= 0x01, "imm8: %u", imm8);
6005   int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_256bit;
6006   InstructionMark im(this);
6007   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6008   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
6009   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6010   emit_int8(0x19);
6011   emit_operand(src, dst);
6012   // 0x00 - extract from lower 128 bits
6013   // 0x01 - extract from upper 128 bits
6014   emit_int8(imm8 & 0x01);
6015 }
6016 
6017 void Assembler::vextractf32x4(XMMRegister dst, XMMRegister src, uint8_t imm8) {
6018   assert(VM_Version::supports_avx(), "");
6019   assert(imm8 <= 0x03, "imm8: %u", imm8);
6020   int vector_len = VM_Version::supports_evex() ? AVX_512bit : AVX_256bit;
6021   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6022   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6023   emit_int8(0x19);
6024   emit_int8((unsigned char)(0xC0 | encode));
6025   // 0x00 - extract from bits 127:0
6026   // 0x01 - extract from bits 255:128
6027   // 0x02 - extract from bits 383:256
6028   // 0x03 - extract from bits 511:384
6029   emit_int8(imm8 & 0x03);
6030 }
6031 
6032 void Assembler::vextractf32x4(Address dst, XMMRegister src, uint8_t imm8) {
6033   assert(VM_Version::supports_evex(), "");
6034   assert(src != xnoreg, "sanity");
6035   assert(imm8 <= 0x03, "imm8: %u", imm8);
6036   InstructionMark im(this);
6037   InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6038   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
6039   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6040   emit_int8(0x19);
6041   emit_operand(src, dst);
6042   // 0x00 - extract from bits 127:0
6043   // 0x01 - extract from bits 255:128
6044   // 0x02 - extract from bits 383:256
6045   // 0x03 - extract from bits 511:384
6046   emit_int8(imm8 & 0x03);
6047 }
6048 
6049 void Assembler::vextractf64x2(XMMRegister dst, XMMRegister src, uint8_t imm8) {
6050   assert(VM_Version::supports_avx512dq(), "");
6051   assert(imm8 <= 0x03, "imm8: %u", imm8);
6052   InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6053   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6054   emit_int8(0x19);
6055   emit_int8((unsigned char)(0xC0 | encode));
6056   // 0x00 - extract from bits 127:0
6057   // 0x01 - extract from bits 255:128
6058   // 0x02 - extract from bits 383:256
6059   // 0x03 - extract from bits 511:384
6060   emit_int8(imm8 & 0x03);
6061 }
6062 
6063 void Assembler::vextractf64x4(XMMRegister dst, XMMRegister src, uint8_t imm8) {
6064   assert(VM_Version::supports_evex(), "");
6065   assert(imm8 <= 0x01, "imm8: %u", imm8);
6066   InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6067   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6068   emit_int8(0x1B);
6069   emit_int8((unsigned char)(0xC0 | encode));
6070   // 0x00 - extract from lower 256 bits
6071   // 0x01 - extract from upper 256 bits
6072   emit_int8(imm8 & 0x01);
6073 }
6074 
6075 void Assembler::vextractf64x4(Address dst, XMMRegister src, uint8_t imm8) {
6076   assert(VM_Version::supports_evex(), "");
6077   assert(src != xnoreg, "sanity");
6078   assert(imm8 <= 0x01, "imm8: %u", imm8);
6079   InstructionMark im(this);
6080   InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6081   attributes.set_address_attributes(/* tuple_type */ EVEX_T4,/* input_size_in_bits */  EVEX_64bit);
6082   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6083   emit_int8(0x1B);
6084   emit_operand(src, dst);
6085   // 0x00 - extract from lower 256 bits
6086   // 0x01 - extract from upper 256 bits
6087   emit_int8(imm8 & 0x01);
6088 }
6089 
6090 
6091 // legacy word/dword replicate
6092 void Assembler::vpbroadcastw(XMMRegister dst, XMMRegister src) {
6093   assert(VM_Version::supports_avx2(), "");
6094   InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
6095   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6096   emit_int8(0x79);
6097   emit_int8((unsigned char)(0xC0 | encode));
6098 }
6099 
6100 void Assembler::vpbroadcastd(XMMRegister dst, XMMRegister src) {
6101   assert(VM_Version::supports_avx2(), "");
6102   InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6103   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6104   emit_int8(0x58);
6105   emit_int8((unsigned char)(0xC0 | encode));
6106 }
6107 
6108 
6109 // xmm/mem sourced byte/word/dword/qword replicate
6110 
6111 // duplicate 1-byte integer data from src into programmed locations in dest : requires AVX512BW and AVX512VL
6112 void Assembler::evpbroadcastb(XMMRegister dst, XMMRegister src, int vector_len) {
6113   assert(VM_Version::supports_evex(), "");
6114   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
6115   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6116   emit_int8(0x78);
6117   emit_int8((unsigned char)(0xC0 | encode));
6118 }
6119 
6120 void Assembler::evpbroadcastb(XMMRegister dst, Address src, int vector_len) {
6121   assert(VM_Version::supports_evex(), "");
6122   assert(dst != xnoreg, "sanity");
6123   InstructionMark im(this);
6124   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
6125   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_8bit);
6126   // swap src<->dst for encoding
6127   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6128   emit_int8(0x78);
6129   emit_operand(dst, src);
6130 }
6131 
6132 // duplicate 2-byte integer data from src into programmed locations in dest : requires AVX512BW and AVX512VL
6133 void Assembler::evpbroadcastw(XMMRegister dst, XMMRegister src, int vector_len) {
6134   assert(VM_Version::supports_evex(), "");
6135   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
6136   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6137   emit_int8(0x79);
6138   emit_int8((unsigned char)(0xC0 | encode));
6139 }
6140 
6141 void Assembler::evpbroadcastw(XMMRegister dst, Address src, int vector_len) {
6142   assert(VM_Version::supports_evex(), "");
6143   assert(dst != xnoreg, "sanity");
6144   InstructionMark im(this);
6145   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
6146   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_16bit);
6147   // swap src<->dst for encoding
6148   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6149   emit_int8(0x79);
6150   emit_operand(dst, src);
6151 }
6152 
6153 // duplicate 4-byte integer data from src into programmed locations in dest : requires AVX512VL
6154 void Assembler::evpbroadcastd(XMMRegister dst, XMMRegister src, int vector_len) {
6155   assert(VM_Version::supports_evex(), "");
6156   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6157   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6158   emit_int8(0x58);
6159   emit_int8((unsigned char)(0xC0 | encode));
6160 }
6161 
6162 void Assembler::evpbroadcastd(XMMRegister dst, Address src, int vector_len) {
6163   assert(VM_Version::supports_evex(), "");
6164   assert(dst != xnoreg, "sanity");
6165   InstructionMark im(this);
6166   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6167   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
6168   // swap src<->dst for encoding
6169   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6170   emit_int8(0x58);
6171   emit_operand(dst, src);
6172 }
6173 
6174 // duplicate 8-byte integer data from src into programmed locations in dest : requires AVX512VL
6175 void Assembler::evpbroadcastq(XMMRegister dst, XMMRegister src, int vector_len) {
6176   assert(VM_Version::supports_evex(), "");
6177   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6178   attributes.set_rex_vex_w_reverted();
6179   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6180   emit_int8(0x59);
6181   emit_int8((unsigned char)(0xC0 | encode));
6182 }
6183 
6184 void Assembler::evpbroadcastq(XMMRegister dst, Address src, int vector_len) {
6185   assert(VM_Version::supports_evex(), "");
6186   assert(dst != xnoreg, "sanity");
6187   InstructionMark im(this);
6188   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6189   attributes.set_rex_vex_w_reverted();
6190   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
6191   // swap src<->dst for encoding
6192   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6193   emit_int8(0x59);
6194   emit_operand(dst, src);
6195 }
6196 
6197 
6198 // scalar single/double precision replicate
6199 
6200 // duplicate single precision data from src into programmed locations in dest : requires AVX512VL
6201 void Assembler::evpbroadcastss(XMMRegister dst, XMMRegister src, int vector_len) {
6202   assert(VM_Version::supports_evex(), "");
6203   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6204   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6205   emit_int8(0x18);
6206   emit_int8((unsigned char)(0xC0 | encode));
6207 }
6208 
6209 void Assembler::evpbroadcastss(XMMRegister dst, Address src, int vector_len) {
6210   assert(VM_Version::supports_evex(), "");
6211   assert(dst != xnoreg, "sanity");
6212   InstructionMark im(this);
6213   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6214   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
6215   // swap src<->dst for encoding
6216   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6217   emit_int8(0x18);
6218   emit_operand(dst, src);
6219 }
6220 
6221 // duplicate double precision data from src into programmed locations in dest : requires AVX512VL
6222 void Assembler::evpbroadcastsd(XMMRegister dst, XMMRegister src, int vector_len) {
6223   assert(VM_Version::supports_evex(), "");
6224   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6225   attributes.set_rex_vex_w_reverted();
6226   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6227   emit_int8(0x19);
6228   emit_int8((unsigned char)(0xC0 | encode));
6229 }
6230 
6231 void Assembler::evpbroadcastsd(XMMRegister dst, Address src, int vector_len) {
6232   assert(VM_Version::supports_evex(), "");
6233   assert(dst != xnoreg, "sanity");
6234   InstructionMark im(this);
6235   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6236   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
6237   attributes.set_rex_vex_w_reverted();
6238   // swap src<->dst for encoding
6239   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6240   emit_int8(0x19);
6241   emit_operand(dst, src);
6242 }
6243 
6244 
6245 // gpr source broadcast forms
6246 
6247 // duplicate 1-byte integer data from src into programmed locations in dest : requires AVX512BW and AVX512VL
6248 void Assembler::evpbroadcastb(XMMRegister dst, Register src, int vector_len) {
6249   assert(VM_Version::supports_evex(), "");
6250   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
6251   attributes.set_is_evex_instruction();
6252   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6253   emit_int8(0x7A);
6254   emit_int8((unsigned char)(0xC0 | encode));
6255 }
6256 
6257 // duplicate 2-byte integer data from src into programmed locations in dest : requires AVX512BW and AVX512VL
6258 void Assembler::evpbroadcastw(XMMRegister dst, Register src, int vector_len) {
6259   assert(VM_Version::supports_evex(), "");
6260   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
6261   attributes.set_is_evex_instruction();
6262   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6263   emit_int8(0x7B);
6264   emit_int8((unsigned char)(0xC0 | encode));
6265 }
6266 
6267 // duplicate 4-byte integer data from src into programmed locations in dest : requires AVX512VL
6268 void Assembler::evpbroadcastd(XMMRegister dst, Register src, int vector_len) {
6269   assert(VM_Version::supports_evex(), "");
6270   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6271   attributes.set_is_evex_instruction();
6272   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6273   emit_int8(0x7C);
6274   emit_int8((unsigned char)(0xC0 | encode));
6275 }
6276 
6277 // duplicate 8-byte integer data from src into programmed locations in dest : requires AVX512VL
6278 void Assembler::evpbroadcastq(XMMRegister dst, Register src, int vector_len) {
6279   assert(VM_Version::supports_evex(), "");
6280   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6281   attributes.set_is_evex_instruction();
6282   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6283   emit_int8(0x7C);
6284   emit_int8((unsigned char)(0xC0 | encode));
6285 }
6286 
6287 
6288 // Carry-Less Multiplication Quadword
6289 void Assembler::pclmulqdq(XMMRegister dst, XMMRegister src, int mask) {
6290   assert(VM_Version::supports_clmul(), "");
6291   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
6292   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6293   emit_int8(0x44);
6294   emit_int8((unsigned char)(0xC0 | encode));
6295   emit_int8((unsigned char)mask);
6296 }
6297 
6298 // Carry-Less Multiplication Quadword
6299 void Assembler::vpclmulqdq(XMMRegister dst, XMMRegister nds, XMMRegister src, int mask) {
6300   assert(VM_Version::supports_avx() && VM_Version::supports_clmul(), "");
6301   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
6302   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6303   emit_int8(0x44);
6304   emit_int8((unsigned char)(0xC0 | encode));
6305   emit_int8((unsigned char)mask);
6306 }
6307 
6308 void Assembler::vzeroupper() {
6309   assert(VM_Version::supports_avx(), "");
6310   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
6311   (void)vex_prefix_and_encode(0, 0, 0, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
6312   emit_int8(0x77);
6313 }
6314 
6315 
6316 #ifndef _LP64
6317 // 32bit only pieces of the assembler
6318 
6319 void Assembler::cmp_literal32(Register src1, int32_t imm32, RelocationHolder const& rspec) {
6320   // NO PREFIX AS NEVER 64BIT
6321   InstructionMark im(this);
6322   emit_int8((unsigned char)0x81);
6323   emit_int8((unsigned char)(0xF8 | src1->encoding()));
6324   emit_data(imm32, rspec, 0);
6325 }
6326 
6327 void Assembler::cmp_literal32(Address src1, int32_t imm32, RelocationHolder const& rspec) {
6328   // NO PREFIX AS NEVER 64BIT (not even 32bit versions of 64bit regs
6329   InstructionMark im(this);
6330   emit_int8((unsigned char)0x81);
6331   emit_operand(rdi, src1);
6332   emit_data(imm32, rspec, 0);
6333 }
6334 
6335 // The 64-bit (32bit platform) cmpxchg compares the value at adr with the contents of rdx:rax,
6336 // and stores rcx:rbx into adr if so; otherwise, the value at adr is loaded
6337 // into rdx:rax.  The ZF is set if the compared values were equal, and cleared otherwise.
6338 void Assembler::cmpxchg8(Address adr) {
6339   InstructionMark im(this);
6340   emit_int8(0x0F);
6341   emit_int8((unsigned char)0xC7);
6342   emit_operand(rcx, adr);
6343 }
6344 
6345 void Assembler::decl(Register dst) {
6346   // Don't use it directly. Use MacroAssembler::decrementl() instead.
6347  emit_int8(0x48 | dst->encoding());
6348 }
6349 
6350 #endif // _LP64
6351 
6352 // 64bit typically doesn't use the x87 but needs to for the trig funcs
6353 
6354 void Assembler::fabs() {
6355   emit_int8((unsigned char)0xD9);
6356   emit_int8((unsigned char)0xE1);
6357 }
6358 
6359 void Assembler::fadd(int i) {
6360   emit_farith(0xD8, 0xC0, i);
6361 }
6362 
6363 void Assembler::fadd_d(Address src) {
6364   InstructionMark im(this);
6365   emit_int8((unsigned char)0xDC);
6366   emit_operand32(rax, src);
6367 }
6368 
6369 void Assembler::fadd_s(Address src) {
6370   InstructionMark im(this);
6371   emit_int8((unsigned char)0xD8);
6372   emit_operand32(rax, src);
6373 }
6374 
6375 void Assembler::fadda(int i) {
6376   emit_farith(0xDC, 0xC0, i);
6377 }
6378 
6379 void Assembler::faddp(int i) {
6380   emit_farith(0xDE, 0xC0, i);
6381 }
6382 
6383 void Assembler::fchs() {
6384   emit_int8((unsigned char)0xD9);
6385   emit_int8((unsigned char)0xE0);
6386 }
6387 
6388 void Assembler::fcom(int i) {
6389   emit_farith(0xD8, 0xD0, i);
6390 }
6391 
6392 void Assembler::fcomp(int i) {
6393   emit_farith(0xD8, 0xD8, i);
6394 }
6395 
6396 void Assembler::fcomp_d(Address src) {
6397   InstructionMark im(this);
6398   emit_int8((unsigned char)0xDC);
6399   emit_operand32(rbx, src);
6400 }
6401 
6402 void Assembler::fcomp_s(Address src) {
6403   InstructionMark im(this);
6404   emit_int8((unsigned char)0xD8);
6405   emit_operand32(rbx, src);
6406 }
6407 
6408 void Assembler::fcompp() {
6409   emit_int8((unsigned char)0xDE);
6410   emit_int8((unsigned char)0xD9);
6411 }
6412 
6413 void Assembler::fcos() {
6414   emit_int8((unsigned char)0xD9);
6415   emit_int8((unsigned char)0xFF);
6416 }
6417 
6418 void Assembler::fdecstp() {
6419   emit_int8((unsigned char)0xD9);
6420   emit_int8((unsigned char)0xF6);
6421 }
6422 
6423 void Assembler::fdiv(int i) {
6424   emit_farith(0xD8, 0xF0, i);
6425 }
6426 
6427 void Assembler::fdiv_d(Address src) {
6428   InstructionMark im(this);
6429   emit_int8((unsigned char)0xDC);
6430   emit_operand32(rsi, src);
6431 }
6432 
6433 void Assembler::fdiv_s(Address src) {
6434   InstructionMark im(this);
6435   emit_int8((unsigned char)0xD8);
6436   emit_operand32(rsi, src);
6437 }
6438 
6439 void Assembler::fdiva(int i) {
6440   emit_farith(0xDC, 0xF8, i);
6441 }
6442 
6443 // Note: The Intel manual (Pentium Processor User's Manual, Vol.3, 1994)
6444 //       is erroneous for some of the floating-point instructions below.
6445 
6446 void Assembler::fdivp(int i) {
6447   emit_farith(0xDE, 0xF8, i);                    // ST(0) <- ST(0) / ST(1) and pop (Intel manual wrong)
6448 }
6449 
6450 void Assembler::fdivr(int i) {
6451   emit_farith(0xD8, 0xF8, i);
6452 }
6453 
6454 void Assembler::fdivr_d(Address src) {
6455   InstructionMark im(this);
6456   emit_int8((unsigned char)0xDC);
6457   emit_operand32(rdi, src);
6458 }
6459 
6460 void Assembler::fdivr_s(Address src) {
6461   InstructionMark im(this);
6462   emit_int8((unsigned char)0xD8);
6463   emit_operand32(rdi, src);
6464 }
6465 
6466 void Assembler::fdivra(int i) {
6467   emit_farith(0xDC, 0xF0, i);
6468 }
6469 
6470 void Assembler::fdivrp(int i) {
6471   emit_farith(0xDE, 0xF0, i);                    // ST(0) <- ST(1) / ST(0) and pop (Intel manual wrong)
6472 }
6473 
6474 void Assembler::ffree(int i) {
6475   emit_farith(0xDD, 0xC0, i);
6476 }
6477 
6478 void Assembler::fild_d(Address adr) {
6479   InstructionMark im(this);
6480   emit_int8((unsigned char)0xDF);
6481   emit_operand32(rbp, adr);
6482 }
6483 
6484 void Assembler::fild_s(Address adr) {
6485   InstructionMark im(this);
6486   emit_int8((unsigned char)0xDB);
6487   emit_operand32(rax, adr);
6488 }
6489 
6490 void Assembler::fincstp() {
6491   emit_int8((unsigned char)0xD9);
6492   emit_int8((unsigned char)0xF7);
6493 }
6494 
6495 void Assembler::finit() {
6496   emit_int8((unsigned char)0x9B);
6497   emit_int8((unsigned char)0xDB);
6498   emit_int8((unsigned char)0xE3);
6499 }
6500 
6501 void Assembler::fist_s(Address adr) {
6502   InstructionMark im(this);
6503   emit_int8((unsigned char)0xDB);
6504   emit_operand32(rdx, adr);
6505 }
6506 
6507 void Assembler::fistp_d(Address adr) {
6508   InstructionMark im(this);
6509   emit_int8((unsigned char)0xDF);
6510   emit_operand32(rdi, adr);
6511 }
6512 
6513 void Assembler::fistp_s(Address adr) {
6514   InstructionMark im(this);
6515   emit_int8((unsigned char)0xDB);
6516   emit_operand32(rbx, adr);
6517 }
6518 
6519 void Assembler::fld1() {
6520   emit_int8((unsigned char)0xD9);
6521   emit_int8((unsigned char)0xE8);
6522 }
6523 
6524 void Assembler::fld_d(Address adr) {
6525   InstructionMark im(this);
6526   emit_int8((unsigned char)0xDD);
6527   emit_operand32(rax, adr);
6528 }
6529 
6530 void Assembler::fld_s(Address adr) {
6531   InstructionMark im(this);
6532   emit_int8((unsigned char)0xD9);
6533   emit_operand32(rax, adr);
6534 }
6535 
6536 
6537 void Assembler::fld_s(int index) {
6538   emit_farith(0xD9, 0xC0, index);
6539 }
6540 
6541 void Assembler::fld_x(Address adr) {
6542   InstructionMark im(this);
6543   emit_int8((unsigned char)0xDB);
6544   emit_operand32(rbp, adr);
6545 }
6546 
6547 void Assembler::fldcw(Address src) {
6548   InstructionMark im(this);
6549   emit_int8((unsigned char)0xD9);
6550   emit_operand32(rbp, src);
6551 }
6552 
6553 void Assembler::fldenv(Address src) {
6554   InstructionMark im(this);
6555   emit_int8((unsigned char)0xD9);
6556   emit_operand32(rsp, src);
6557 }
6558 
6559 void Assembler::fldlg2() {
6560   emit_int8((unsigned char)0xD9);
6561   emit_int8((unsigned char)0xEC);
6562 }
6563 
6564 void Assembler::fldln2() {
6565   emit_int8((unsigned char)0xD9);
6566   emit_int8((unsigned char)0xED);
6567 }
6568 
6569 void Assembler::fldz() {
6570   emit_int8((unsigned char)0xD9);
6571   emit_int8((unsigned char)0xEE);
6572 }
6573 
6574 void Assembler::flog() {
6575   fldln2();
6576   fxch();
6577   fyl2x();
6578 }
6579 
6580 void Assembler::flog10() {
6581   fldlg2();
6582   fxch();
6583   fyl2x();
6584 }
6585 
6586 void Assembler::fmul(int i) {
6587   emit_farith(0xD8, 0xC8, i);
6588 }
6589 
6590 void Assembler::fmul_d(Address src) {
6591   InstructionMark im(this);
6592   emit_int8((unsigned char)0xDC);
6593   emit_operand32(rcx, src);
6594 }
6595 
6596 void Assembler::fmul_s(Address src) {
6597   InstructionMark im(this);
6598   emit_int8((unsigned char)0xD8);
6599   emit_operand32(rcx, src);
6600 }
6601 
6602 void Assembler::fmula(int i) {
6603   emit_farith(0xDC, 0xC8, i);
6604 }
6605 
6606 void Assembler::fmulp(int i) {
6607   emit_farith(0xDE, 0xC8, i);
6608 }
6609 
6610 void Assembler::fnsave(Address dst) {
6611   InstructionMark im(this);
6612   emit_int8((unsigned char)0xDD);
6613   emit_operand32(rsi, dst);
6614 }
6615 
6616 void Assembler::fnstcw(Address src) {
6617   InstructionMark im(this);
6618   emit_int8((unsigned char)0x9B);
6619   emit_int8((unsigned char)0xD9);
6620   emit_operand32(rdi, src);
6621 }
6622 
6623 void Assembler::fnstsw_ax() {
6624   emit_int8((unsigned char)0xDF);
6625   emit_int8((unsigned char)0xE0);
6626 }
6627 
6628 void Assembler::fprem() {
6629   emit_int8((unsigned char)0xD9);
6630   emit_int8((unsigned char)0xF8);
6631 }
6632 
6633 void Assembler::fprem1() {
6634   emit_int8((unsigned char)0xD9);
6635   emit_int8((unsigned char)0xF5);
6636 }
6637 
6638 void Assembler::frstor(Address src) {
6639   InstructionMark im(this);
6640   emit_int8((unsigned char)0xDD);
6641   emit_operand32(rsp, src);
6642 }
6643 
6644 void Assembler::fsin() {
6645   emit_int8((unsigned char)0xD9);
6646   emit_int8((unsigned char)0xFE);
6647 }
6648 
6649 void Assembler::fsqrt() {
6650   emit_int8((unsigned char)0xD9);
6651   emit_int8((unsigned char)0xFA);
6652 }
6653 
6654 void Assembler::fst_d(Address adr) {
6655   InstructionMark im(this);
6656   emit_int8((unsigned char)0xDD);
6657   emit_operand32(rdx, adr);
6658 }
6659 
6660 void Assembler::fst_s(Address adr) {
6661   InstructionMark im(this);
6662   emit_int8((unsigned char)0xD9);
6663   emit_operand32(rdx, adr);
6664 }
6665 
6666 void Assembler::fstp_d(Address adr) {
6667   InstructionMark im(this);
6668   emit_int8((unsigned char)0xDD);
6669   emit_operand32(rbx, adr);
6670 }
6671 
6672 void Assembler::fstp_d(int index) {
6673   emit_farith(0xDD, 0xD8, index);
6674 }
6675 
6676 void Assembler::fstp_s(Address adr) {
6677   InstructionMark im(this);
6678   emit_int8((unsigned char)0xD9);
6679   emit_operand32(rbx, adr);
6680 }
6681 
6682 void Assembler::fstp_x(Address adr) {
6683   InstructionMark im(this);
6684   emit_int8((unsigned char)0xDB);
6685   emit_operand32(rdi, adr);
6686 }
6687 
6688 void Assembler::fsub(int i) {
6689   emit_farith(0xD8, 0xE0, i);
6690 }
6691 
6692 void Assembler::fsub_d(Address src) {
6693   InstructionMark im(this);
6694   emit_int8((unsigned char)0xDC);
6695   emit_operand32(rsp, src);
6696 }
6697 
6698 void Assembler::fsub_s(Address src) {
6699   InstructionMark im(this);
6700   emit_int8((unsigned char)0xD8);
6701   emit_operand32(rsp, src);
6702 }
6703 
6704 void Assembler::fsuba(int i) {
6705   emit_farith(0xDC, 0xE8, i);
6706 }
6707 
6708 void Assembler::fsubp(int i) {
6709   emit_farith(0xDE, 0xE8, i);                    // ST(0) <- ST(0) - ST(1) and pop (Intel manual wrong)
6710 }
6711 
6712 void Assembler::fsubr(int i) {
6713   emit_farith(0xD8, 0xE8, i);
6714 }
6715 
6716 void Assembler::fsubr_d(Address src) {
6717   InstructionMark im(this);
6718   emit_int8((unsigned char)0xDC);
6719   emit_operand32(rbp, src);
6720 }
6721 
6722 void Assembler::fsubr_s(Address src) {
6723   InstructionMark im(this);
6724   emit_int8((unsigned char)0xD8);
6725   emit_operand32(rbp, src);
6726 }
6727 
6728 void Assembler::fsubra(int i) {
6729   emit_farith(0xDC, 0xE0, i);
6730 }
6731 
6732 void Assembler::fsubrp(int i) {
6733   emit_farith(0xDE, 0xE0, i);                    // ST(0) <- ST(1) - ST(0) and pop (Intel manual wrong)
6734 }
6735 
6736 void Assembler::ftan() {
6737   emit_int8((unsigned char)0xD9);
6738   emit_int8((unsigned char)0xF2);
6739   emit_int8((unsigned char)0xDD);
6740   emit_int8((unsigned char)0xD8);
6741 }
6742 
6743 void Assembler::ftst() {
6744   emit_int8((unsigned char)0xD9);
6745   emit_int8((unsigned char)0xE4);
6746 }
6747 
6748 void Assembler::fucomi(int i) {
6749   // make sure the instruction is supported (introduced for P6, together with cmov)
6750   guarantee(VM_Version::supports_cmov(), "illegal instruction");
6751   emit_farith(0xDB, 0xE8, i);
6752 }
6753 
6754 void Assembler::fucomip(int i) {
6755   // make sure the instruction is supported (introduced for P6, together with cmov)
6756   guarantee(VM_Version::supports_cmov(), "illegal instruction");
6757   emit_farith(0xDF, 0xE8, i);
6758 }
6759 
6760 void Assembler::fwait() {
6761   emit_int8((unsigned char)0x9B);
6762 }
6763 
6764 void Assembler::fxch(int i) {
6765   emit_farith(0xD9, 0xC8, i);
6766 }
6767 
6768 void Assembler::fyl2x() {
6769   emit_int8((unsigned char)0xD9);
6770   emit_int8((unsigned char)0xF1);
6771 }
6772 
6773 void Assembler::frndint() {
6774   emit_int8((unsigned char)0xD9);
6775   emit_int8((unsigned char)0xFC);
6776 }
6777 
6778 void Assembler::f2xm1() {
6779   emit_int8((unsigned char)0xD9);
6780   emit_int8((unsigned char)0xF0);
6781 }
6782 
6783 void Assembler::fldl2e() {
6784   emit_int8((unsigned char)0xD9);
6785   emit_int8((unsigned char)0xEA);
6786 }
6787 
6788 // SSE SIMD prefix byte values corresponding to VexSimdPrefix encoding.
6789 static int simd_pre[4] = { 0, 0x66, 0xF3, 0xF2 };
6790 // SSE opcode second byte values (first is 0x0F) corresponding to VexOpcode encoding.
6791 static int simd_opc[4] = { 0,    0, 0x38, 0x3A };
6792 
6793 // Generate SSE legacy REX prefix and SIMD opcode based on VEX encoding.
6794 void Assembler::rex_prefix(Address adr, XMMRegister xreg, VexSimdPrefix pre, VexOpcode opc, bool rex_w) {
6795   if (pre > 0) {
6796     emit_int8(simd_pre[pre]);
6797   }
6798   if (rex_w) {
6799     prefixq(adr, xreg);
6800   } else {
6801     prefix(adr, xreg);
6802   }
6803   if (opc > 0) {
6804     emit_int8(0x0F);
6805     int opc2 = simd_opc[opc];
6806     if (opc2 > 0) {
6807       emit_int8(opc2);
6808     }
6809   }
6810 }
6811 
6812 int Assembler::rex_prefix_and_encode(int dst_enc, int src_enc, VexSimdPrefix pre, VexOpcode opc, bool rex_w) {
6813   if (pre > 0) {
6814     emit_int8(simd_pre[pre]);
6815   }
6816   int encode = (rex_w) ? prefixq_and_encode(dst_enc, src_enc) : prefix_and_encode(dst_enc, src_enc);
6817   if (opc > 0) {
6818     emit_int8(0x0F);
6819     int opc2 = simd_opc[opc];
6820     if (opc2 > 0) {
6821       emit_int8(opc2);
6822     }
6823   }
6824   return encode;
6825 }
6826 
6827 
6828 void Assembler::vex_prefix(bool vex_r, bool vex_b, bool vex_x, int nds_enc, VexSimdPrefix pre, VexOpcode opc) {
6829   int vector_len = _attributes->get_vector_len();
6830   bool vex_w = _attributes->is_rex_vex_w();
6831   if (vex_b || vex_x || vex_w || (opc == VEX_OPCODE_0F_38) || (opc == VEX_OPCODE_0F_3A)) {
6832     prefix(VEX_3bytes);
6833 
6834     int byte1 = (vex_r ? VEX_R : 0) | (vex_x ? VEX_X : 0) | (vex_b ? VEX_B : 0);
6835     byte1 = (~byte1) & 0xE0;
6836     byte1 |= opc;
6837     emit_int8(byte1);
6838 
6839     int byte2 = ((~nds_enc) & 0xf) << 3;
6840     byte2 |= (vex_w ? VEX_W : 0) | ((vector_len > 0) ? 4 : 0) | pre;
6841     emit_int8(byte2);
6842   } else {
6843     prefix(VEX_2bytes);
6844 
6845     int byte1 = vex_r ? VEX_R : 0;
6846     byte1 = (~byte1) & 0x80;
6847     byte1 |= ((~nds_enc) & 0xf) << 3;
6848     byte1 |= ((vector_len > 0 ) ? 4 : 0) | pre;
6849     emit_int8(byte1);
6850   }
6851 }
6852 
6853 // This is a 4 byte encoding
6854 void Assembler::evex_prefix(bool vex_r, bool vex_b, bool vex_x, bool evex_r, bool evex_v, int nds_enc, VexSimdPrefix pre, VexOpcode opc){
6855   // EVEX 0x62 prefix
6856   prefix(EVEX_4bytes);
6857   bool vex_w = _attributes->is_rex_vex_w();
6858   int evex_encoding = (vex_w ? VEX_W : 0);
6859   // EVEX.b is not currently used for broadcast of single element or data rounding modes
6860   _attributes->set_evex_encoding(evex_encoding);
6861 
6862   // P0: byte 2, initialized to RXBR`00mm
6863   // instead of not'd
6864   int byte2 = (vex_r ? VEX_R : 0) | (vex_x ? VEX_X : 0) | (vex_b ? VEX_B : 0) | (evex_r ? EVEX_Rb : 0);
6865   byte2 = (~byte2) & 0xF0;
6866   // confine opc opcode extensions in mm bits to lower two bits
6867   // of form {0F, 0F_38, 0F_3A}
6868   byte2 |= opc;
6869   emit_int8(byte2);
6870 
6871   // P1: byte 3 as Wvvvv1pp
6872   int byte3 = ((~nds_enc) & 0xf) << 3;
6873   // p[10] is always 1
6874   byte3 |= EVEX_F;
6875   byte3 |= (vex_w & 1) << 7;
6876   // confine pre opcode extensions in pp bits to lower two bits
6877   // of form {66, F3, F2}
6878   byte3 |= pre;
6879   emit_int8(byte3);
6880 
6881   // P2: byte 4 as zL'Lbv'aaa
6882   int byte4 = (_attributes->is_no_reg_mask()) ? 0 : 1; // kregs are implemented in the low 3 bits as aaa (hard code k1, it will be initialized for now)
6883   // EVEX.v` for extending EVEX.vvvv or VIDX
6884   byte4 |= (evex_v ? 0: EVEX_V);
6885   // third EXEC.b for broadcast actions
6886   byte4 |= (_attributes->is_extended_context() ? EVEX_Rb : 0);
6887   // fourth EVEX.L'L for vector length : 0 is 128, 1 is 256, 2 is 512, currently we do not support 1024
6888   byte4 |= ((_attributes->get_vector_len())& 0x3) << 5;
6889   // last is EVEX.z for zero/merge actions
6890   byte4 |= (_attributes->is_clear_context() ? EVEX_Z : 0);
6891   emit_int8(byte4);
6892 }
6893 
6894 void Assembler::vex_prefix(Address adr, int nds_enc, int xreg_enc, VexSimdPrefix pre, VexOpcode opc, InstructionAttr *attributes) {
6895   bool vex_r = ((xreg_enc & 8) == 8) ? 1 : 0;
6896   bool vex_b = adr.base_needs_rex();
6897   bool vex_x = adr.index_needs_rex();
6898   set_attributes(attributes);
6899   attributes->set_current_assembler(this);
6900 
6901   // if vector length is turned off, revert to AVX for vectors smaller than 512-bit
6902   if (UseAVX > 2 && _legacy_mode_vl && attributes->uses_vl()) {
6903     switch (attributes->get_vector_len()) {
6904     case AVX_128bit:
6905     case AVX_256bit:
6906       attributes->set_is_legacy_mode();
6907       break;
6908     }
6909   }
6910 
6911   // For pure EVEX check and see if this instruction
6912   // is allowed in legacy mode and has resources which will
6913   // fit in it.  Pure EVEX instructions will use set_is_evex_instruction in their definition,
6914   // else that field is set when we encode to EVEX
6915   if (UseAVX > 2 && !attributes->is_legacy_mode() &&
6916       !_is_managed && !attributes->is_evex_instruction()) {
6917     if (!_legacy_mode_vl && attributes->get_vector_len() != AVX_512bit) {
6918       bool check_register_bank = NOT_IA32(true) IA32_ONLY(false);
6919       if (check_register_bank) {
6920         // check nds_enc and xreg_enc for upper bank usage
6921         if (nds_enc < 16 && xreg_enc < 16) {
6922           attributes->set_is_legacy_mode();
6923         }
6924       } else {
6925         attributes->set_is_legacy_mode();
6926       }
6927     }
6928   }
6929 
6930   _is_managed = false;
6931   if (UseAVX > 2 && !attributes->is_legacy_mode())
6932   {
6933     bool evex_r = (xreg_enc >= 16);
6934     bool evex_v = (nds_enc >= 16);
6935     attributes->set_is_evex_instruction();
6936     evex_prefix(vex_r, vex_b, vex_x, evex_r, evex_v, nds_enc, pre, opc);
6937   } else {
6938     if (UseAVX > 2 && attributes->is_rex_vex_w_reverted()) {
6939       attributes->set_rex_vex_w(false);
6940     }
6941     vex_prefix(vex_r, vex_b, vex_x, nds_enc, pre, opc);
6942   }
6943 }
6944 
6945 int Assembler::vex_prefix_and_encode(int dst_enc, int nds_enc, int src_enc, VexSimdPrefix pre, VexOpcode opc, InstructionAttr *attributes) {
6946   bool vex_r = ((dst_enc & 8) == 8) ? 1 : 0;
6947   bool vex_b = ((src_enc & 8) == 8) ? 1 : 0;
6948   bool vex_x = false;
6949   set_attributes(attributes);
6950   attributes->set_current_assembler(this);
6951   bool check_register_bank = NOT_IA32(true) IA32_ONLY(false);
6952 
6953   // if vector length is turned off, revert to AVX for vectors smaller than 512-bit
6954   if (UseAVX > 2 && _legacy_mode_vl && attributes->uses_vl()) {
6955     switch (attributes->get_vector_len()) {
6956     case AVX_128bit:
6957     case AVX_256bit:
6958       if (check_register_bank) {
6959         if (dst_enc >= 16 || nds_enc >= 16 || src_enc >= 16) {
6960           // up propagate arithmetic instructions to meet RA requirements
6961           attributes->set_vector_len(AVX_512bit);
6962         } else {
6963           attributes->set_is_legacy_mode();
6964         }
6965       } else {
6966         attributes->set_is_legacy_mode();
6967       }
6968       break;
6969     }
6970   }
6971 
6972   // For pure EVEX check and see if this instruction
6973   // is allowed in legacy mode and has resources which will
6974   // fit in it.  Pure EVEX instructions will use set_is_evex_instruction in their definition,
6975   // else that field is set when we encode to EVEX
6976   if (UseAVX > 2 && !attributes->is_legacy_mode() &&
6977       !_is_managed && !attributes->is_evex_instruction()) {
6978     if (!_legacy_mode_vl && attributes->get_vector_len() != AVX_512bit) {
6979       if (check_register_bank) {
6980         // check dst_enc, nds_enc and src_enc for upper bank usage
6981         if (dst_enc < 16 && nds_enc < 16 && src_enc < 16) {
6982           attributes->set_is_legacy_mode();
6983         }
6984       } else {
6985         attributes->set_is_legacy_mode();
6986       }
6987     }
6988   }
6989 
6990   _is_managed = false;
6991   if (UseAVX > 2 && !attributes->is_legacy_mode())
6992   {
6993     bool evex_r = (dst_enc >= 16);
6994     bool evex_v = (nds_enc >= 16);
6995     // can use vex_x as bank extender on rm encoding
6996     vex_x = (src_enc >= 16);
6997     attributes->set_is_evex_instruction();
6998     evex_prefix(vex_r, vex_b, vex_x, evex_r, evex_v, nds_enc, pre, opc);
6999   } else {
7000     if (UseAVX > 2 && attributes->is_rex_vex_w_reverted()) {
7001       attributes->set_rex_vex_w(false);
7002     }
7003     vex_prefix(vex_r, vex_b, vex_x, nds_enc, pre, opc);
7004   }
7005 
7006   // return modrm byte components for operands
7007   return (((dst_enc & 7) << 3) | (src_enc & 7));
7008 }
7009 
7010 
7011 void Assembler::simd_prefix(XMMRegister xreg, XMMRegister nds, Address adr, VexSimdPrefix pre,
7012                             VexOpcode opc, InstructionAttr *attributes) {
7013   if (UseAVX > 0) {
7014     int xreg_enc = xreg->encoding();
7015     int nds_enc = nds->is_valid() ? nds->encoding() : 0;
7016     vex_prefix(adr, nds_enc, xreg_enc, pre, opc, attributes);
7017   } else {
7018     assert((nds == xreg) || (nds == xnoreg), "wrong sse encoding");
7019     rex_prefix(adr, xreg, pre, opc, attributes->is_rex_vex_w());
7020   }
7021 }
7022 
7023 int Assembler::simd_prefix_and_encode(XMMRegister dst, XMMRegister nds, XMMRegister src, VexSimdPrefix pre,
7024                                       VexOpcode opc, InstructionAttr *attributes) {
7025   int dst_enc = dst->encoding();
7026   int src_enc = src->encoding();
7027   if (UseAVX > 0) {
7028     int nds_enc = nds->is_valid() ? nds->encoding() : 0;
7029     return vex_prefix_and_encode(dst_enc, nds_enc, src_enc, pre, opc, attributes);
7030   } else {
7031     assert((nds == dst) || (nds == src) || (nds == xnoreg), "wrong sse encoding");
7032     return rex_prefix_and_encode(dst_enc, src_enc, pre, opc, attributes->is_rex_vex_w());
7033   }
7034 }
7035 
7036 void Assembler::cmppd(XMMRegister dst, XMMRegister nds, XMMRegister src, int cop, int vector_len) {
7037   assert(VM_Version::supports_avx(), "");
7038   assert(!VM_Version::supports_evex(), "");
7039   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
7040   int encode = simd_prefix_and_encode(dst, nds, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7041   emit_int8((unsigned char)0xC2);
7042   emit_int8((unsigned char)(0xC0 | encode));
7043   emit_int8((unsigned char)(0xF & cop));
7044 }
7045 
7046 void Assembler::vpblendd(XMMRegister dst, XMMRegister nds, XMMRegister src1, XMMRegister src2, int vector_len) {
7047   assert(VM_Version::supports_avx(), "");
7048   assert(!VM_Version::supports_evex(), "");
7049   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
7050   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src1->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
7051   emit_int8((unsigned char)0x4B);
7052   emit_int8((unsigned char)(0xC0 | encode));
7053   int src2_enc = src2->encoding();
7054   emit_int8((unsigned char)(0xF0 & src2_enc<<4));
7055 }
7056 
7057 
7058 #ifndef _LP64
7059 
7060 void Assembler::incl(Register dst) {
7061   // Don't use it directly. Use MacroAssembler::incrementl() instead.
7062   emit_int8(0x40 | dst->encoding());
7063 }
7064 
7065 void Assembler::lea(Register dst, Address src) {
7066   leal(dst, src);
7067 }
7068 
7069 void Assembler::mov_literal32(Address dst, int32_t imm32, RelocationHolder const& rspec) {
7070   InstructionMark im(this);
7071   emit_int8((unsigned char)0xC7);
7072   emit_operand(rax, dst);
7073   emit_data((int)imm32, rspec, 0);
7074 }
7075 
7076 void Assembler::mov_literal32(Register dst, int32_t imm32, RelocationHolder const& rspec) {
7077   InstructionMark im(this);
7078   int encode = prefix_and_encode(dst->encoding());
7079   emit_int8((unsigned char)(0xB8 | encode));
7080   emit_data((int)imm32, rspec, 0);
7081 }
7082 
7083 void Assembler::popa() { // 32bit
7084   emit_int8(0x61);
7085 }
7086 
7087 void Assembler::push_literal32(int32_t imm32, RelocationHolder const& rspec) {
7088   InstructionMark im(this);
7089   emit_int8(0x68);
7090   emit_data(imm32, rspec, 0);
7091 }
7092 
7093 void Assembler::pusha() { // 32bit
7094   emit_int8(0x60);
7095 }
7096 
7097 void Assembler::set_byte_if_not_zero(Register dst) {
7098   emit_int8(0x0F);
7099   emit_int8((unsigned char)0x95);
7100   emit_int8((unsigned char)(0xE0 | dst->encoding()));
7101 }
7102 
7103 void Assembler::shldl(Register dst, Register src) {
7104   emit_int8(0x0F);
7105   emit_int8((unsigned char)0xA5);
7106   emit_int8((unsigned char)(0xC0 | src->encoding() << 3 | dst->encoding()));
7107 }
7108 
7109 // 0F A4 / r ib
7110 void Assembler::shldl(Register dst, Register src, int8_t imm8) {
7111   emit_int8(0x0F);
7112   emit_int8((unsigned char)0xA4);
7113   emit_int8((unsigned char)(0xC0 | src->encoding() << 3 | dst->encoding()));
7114   emit_int8(imm8);
7115 }
7116 
7117 void Assembler::shrdl(Register dst, Register src) {
7118   emit_int8(0x0F);
7119   emit_int8((unsigned char)0xAD);
7120   emit_int8((unsigned char)(0xC0 | src->encoding() << 3 | dst->encoding()));
7121 }
7122 
7123 #else // LP64
7124 
7125 void Assembler::set_byte_if_not_zero(Register dst) {
7126   int enc = prefix_and_encode(dst->encoding(), true);
7127   emit_int8(0x0F);
7128   emit_int8((unsigned char)0x95);
7129   emit_int8((unsigned char)(0xE0 | enc));
7130 }
7131 
7132 // 64bit only pieces of the assembler
7133 // This should only be used by 64bit instructions that can use rip-relative
7134 // it cannot be used by instructions that want an immediate value.
7135 
7136 bool Assembler::reachable(AddressLiteral adr) {
7137   int64_t disp;
7138   // None will force a 64bit literal to the code stream. Likely a placeholder
7139   // for something that will be patched later and we need to certain it will
7140   // always be reachable.
7141   if (adr.reloc() == relocInfo::none) {
7142     return false;
7143   }
7144   if (adr.reloc() == relocInfo::internal_word_type) {
7145     // This should be rip relative and easily reachable.
7146     return true;
7147   }
7148   if (adr.reloc() == relocInfo::virtual_call_type ||
7149       adr.reloc() == relocInfo::opt_virtual_call_type ||
7150       adr.reloc() == relocInfo::static_call_type ||
7151       adr.reloc() == relocInfo::static_stub_type ) {
7152     // This should be rip relative within the code cache and easily
7153     // reachable until we get huge code caches. (At which point
7154     // ic code is going to have issues).
7155     return true;
7156   }
7157   if (adr.reloc() != relocInfo::external_word_type &&
7158       adr.reloc() != relocInfo::poll_return_type &&  // these are really external_word but need special
7159       adr.reloc() != relocInfo::poll_type &&         // relocs to identify them
7160       adr.reloc() != relocInfo::runtime_call_type ) {
7161     return false;
7162   }
7163 
7164   // Stress the correction code
7165   if (ForceUnreachable) {
7166     // Must be runtimecall reloc, see if it is in the codecache
7167     // Flipping stuff in the codecache to be unreachable causes issues
7168     // with things like inline caches where the additional instructions
7169     // are not handled.
7170     if (CodeCache::find_blob(adr._target) == NULL) {
7171       return false;
7172     }
7173   }
7174   // For external_word_type/runtime_call_type if it is reachable from where we
7175   // are now (possibly a temp buffer) and where we might end up
7176   // anywhere in the codeCache then we are always reachable.
7177   // This would have to change if we ever save/restore shared code
7178   // to be more pessimistic.
7179   disp = (int64_t)adr._target - ((int64_t)CodeCache::low_bound() + sizeof(int));
7180   if (!is_simm32(disp)) return false;
7181   disp = (int64_t)adr._target - ((int64_t)CodeCache::high_bound() + sizeof(int));
7182   if (!is_simm32(disp)) return false;
7183 
7184   disp = (int64_t)adr._target - ((int64_t)pc() + sizeof(int));
7185 
7186   // Because rip relative is a disp + address_of_next_instruction and we
7187   // don't know the value of address_of_next_instruction we apply a fudge factor
7188   // to make sure we will be ok no matter the size of the instruction we get placed into.
7189   // We don't have to fudge the checks above here because they are already worst case.
7190 
7191   // 12 == override/rex byte, opcode byte, rm byte, sib byte, a 4-byte disp , 4-byte literal
7192   // + 4 because better safe than sorry.
7193   const int fudge = 12 + 4;
7194   if (disp < 0) {
7195     disp -= fudge;
7196   } else {
7197     disp += fudge;
7198   }
7199   return is_simm32(disp);
7200 }
7201 
7202 // Check if the polling page is not reachable from the code cache using rip-relative
7203 // addressing.
7204 bool Assembler::is_polling_page_far() {
7205   intptr_t addr = (intptr_t)os::get_polling_page();
7206   return ForceUnreachable ||
7207          !is_simm32(addr - (intptr_t)CodeCache::low_bound()) ||
7208          !is_simm32(addr - (intptr_t)CodeCache::high_bound());
7209 }
7210 
7211 void Assembler::emit_data64(jlong data,
7212                             relocInfo::relocType rtype,
7213                             int format) {
7214   if (rtype == relocInfo::none) {
7215     emit_int64(data);
7216   } else {
7217     emit_data64(data, Relocation::spec_simple(rtype), format);
7218   }
7219 }
7220 
7221 void Assembler::emit_data64(jlong data,
7222                             RelocationHolder const& rspec,
7223                             int format) {
7224   assert(imm_operand == 0, "default format must be immediate in this file");
7225   assert(imm_operand == format, "must be immediate");
7226   assert(inst_mark() != NULL, "must be inside InstructionMark");
7227   // Do not use AbstractAssembler::relocate, which is not intended for
7228   // embedded words.  Instead, relocate to the enclosing instruction.
7229   code_section()->relocate(inst_mark(), rspec, format);
7230 #ifdef ASSERT
7231   check_relocation(rspec, format);
7232 #endif
7233   emit_int64(data);
7234 }
7235 
7236 int Assembler::prefix_and_encode(int reg_enc, bool byteinst) {
7237   if (reg_enc >= 8) {
7238     prefix(REX_B);
7239     reg_enc -= 8;
7240   } else if (byteinst && reg_enc >= 4) {
7241     prefix(REX);
7242   }
7243   return reg_enc;
7244 }
7245 
7246 int Assembler::prefixq_and_encode(int reg_enc) {
7247   if (reg_enc < 8) {
7248     prefix(REX_W);
7249   } else {
7250     prefix(REX_WB);
7251     reg_enc -= 8;
7252   }
7253   return reg_enc;
7254 }
7255 
7256 int Assembler::prefix_and_encode(int dst_enc, bool dst_is_byte, int src_enc, bool src_is_byte) {
7257   if (dst_enc < 8) {
7258     if (src_enc >= 8) {
7259       prefix(REX_B);
7260       src_enc -= 8;
7261     } else if ((src_is_byte && src_enc >= 4) || (dst_is_byte && dst_enc >= 4)) {
7262       prefix(REX);
7263     }
7264   } else {
7265     if (src_enc < 8) {
7266       prefix(REX_R);
7267     } else {
7268       prefix(REX_RB);
7269       src_enc -= 8;
7270     }
7271     dst_enc -= 8;
7272   }
7273   return dst_enc << 3 | src_enc;
7274 }
7275 
7276 int Assembler::prefixq_and_encode(int dst_enc, int src_enc) {
7277   if (dst_enc < 8) {
7278     if (src_enc < 8) {
7279       prefix(REX_W);
7280     } else {
7281       prefix(REX_WB);
7282       src_enc -= 8;
7283     }
7284   } else {
7285     if (src_enc < 8) {
7286       prefix(REX_WR);
7287     } else {
7288       prefix(REX_WRB);
7289       src_enc -= 8;
7290     }
7291     dst_enc -= 8;
7292   }
7293   return dst_enc << 3 | src_enc;
7294 }
7295 
7296 void Assembler::prefix(Register reg) {
7297   if (reg->encoding() >= 8) {
7298     prefix(REX_B);
7299   }
7300 }
7301 
7302 void Assembler::prefix(Register dst, Register src, Prefix p) {
7303   if (src->encoding() >= 8) {
7304     p = (Prefix)(p | REX_B);
7305   }
7306   if (dst->encoding() >= 8) {
7307     p = (Prefix)( p | REX_R);
7308   }
7309   if (p != Prefix_EMPTY) {
7310     // do not generate an empty prefix
7311     prefix(p);
7312   }
7313 }
7314 
7315 void Assembler::prefix(Register dst, Address adr, Prefix p) {
7316   if (adr.base_needs_rex()) {
7317     if (adr.index_needs_rex()) {
7318       assert(false, "prefix(Register dst, Address adr, Prefix p) does not support handling of an X");
7319     } else {
7320       prefix(REX_B);
7321     }
7322   } else {
7323     if (adr.index_needs_rex()) {
7324       assert(false, "prefix(Register dst, Address adr, Prefix p) does not support handling of an X");
7325     }
7326   }
7327   if (dst->encoding() >= 8) {
7328     p = (Prefix)(p | REX_R);
7329   }
7330   if (p != Prefix_EMPTY) {
7331     // do not generate an empty prefix
7332     prefix(p);
7333   }
7334 }
7335 
7336 void Assembler::prefix(Address adr) {
7337   if (adr.base_needs_rex()) {
7338     if (adr.index_needs_rex()) {
7339       prefix(REX_XB);
7340     } else {
7341       prefix(REX_B);
7342     }
7343   } else {
7344     if (adr.index_needs_rex()) {
7345       prefix(REX_X);
7346     }
7347   }
7348 }
7349 
7350 void Assembler::prefixq(Address adr) {
7351   if (adr.base_needs_rex()) {
7352     if (adr.index_needs_rex()) {
7353       prefix(REX_WXB);
7354     } else {
7355       prefix(REX_WB);
7356     }
7357   } else {
7358     if (adr.index_needs_rex()) {
7359       prefix(REX_WX);
7360     } else {
7361       prefix(REX_W);
7362     }
7363   }
7364 }
7365 
7366 
7367 void Assembler::prefix(Address adr, Register reg, bool byteinst) {
7368   if (reg->encoding() < 8) {
7369     if (adr.base_needs_rex()) {
7370       if (adr.index_needs_rex()) {
7371         prefix(REX_XB);
7372       } else {
7373         prefix(REX_B);
7374       }
7375     } else {
7376       if (adr.index_needs_rex()) {
7377         prefix(REX_X);
7378       } else if (byteinst && reg->encoding() >= 4 ) {
7379         prefix(REX);
7380       }
7381     }
7382   } else {
7383     if (adr.base_needs_rex()) {
7384       if (adr.index_needs_rex()) {
7385         prefix(REX_RXB);
7386       } else {
7387         prefix(REX_RB);
7388       }
7389     } else {
7390       if (adr.index_needs_rex()) {
7391         prefix(REX_RX);
7392       } else {
7393         prefix(REX_R);
7394       }
7395     }
7396   }
7397 }
7398 
7399 void Assembler::prefixq(Address adr, Register src) {
7400   if (src->encoding() < 8) {
7401     if (adr.base_needs_rex()) {
7402       if (adr.index_needs_rex()) {
7403         prefix(REX_WXB);
7404       } else {
7405         prefix(REX_WB);
7406       }
7407     } else {
7408       if (adr.index_needs_rex()) {
7409         prefix(REX_WX);
7410       } else {
7411         prefix(REX_W);
7412       }
7413     }
7414   } else {
7415     if (adr.base_needs_rex()) {
7416       if (adr.index_needs_rex()) {
7417         prefix(REX_WRXB);
7418       } else {
7419         prefix(REX_WRB);
7420       }
7421     } else {
7422       if (adr.index_needs_rex()) {
7423         prefix(REX_WRX);
7424       } else {
7425         prefix(REX_WR);
7426       }
7427     }
7428   }
7429 }
7430 
7431 void Assembler::prefix(Address adr, XMMRegister reg) {
7432   if (reg->encoding() < 8) {
7433     if (adr.base_needs_rex()) {
7434       if (adr.index_needs_rex()) {
7435         prefix(REX_XB);
7436       } else {
7437         prefix(REX_B);
7438       }
7439     } else {
7440       if (adr.index_needs_rex()) {
7441         prefix(REX_X);
7442       }
7443     }
7444   } else {
7445     if (adr.base_needs_rex()) {
7446       if (adr.index_needs_rex()) {
7447         prefix(REX_RXB);
7448       } else {
7449         prefix(REX_RB);
7450       }
7451     } else {
7452       if (adr.index_needs_rex()) {
7453         prefix(REX_RX);
7454       } else {
7455         prefix(REX_R);
7456       }
7457     }
7458   }
7459 }
7460 
7461 void Assembler::prefixq(Address adr, XMMRegister src) {
7462   if (src->encoding() < 8) {
7463     if (adr.base_needs_rex()) {
7464       if (adr.index_needs_rex()) {
7465         prefix(REX_WXB);
7466       } else {
7467         prefix(REX_WB);
7468       }
7469     } else {
7470       if (adr.index_needs_rex()) {
7471         prefix(REX_WX);
7472       } else {
7473         prefix(REX_W);
7474       }
7475     }
7476   } else {
7477     if (adr.base_needs_rex()) {
7478       if (adr.index_needs_rex()) {
7479         prefix(REX_WRXB);
7480       } else {
7481         prefix(REX_WRB);
7482       }
7483     } else {
7484       if (adr.index_needs_rex()) {
7485         prefix(REX_WRX);
7486       } else {
7487         prefix(REX_WR);
7488       }
7489     }
7490   }
7491 }
7492 
7493 void Assembler::adcq(Register dst, int32_t imm32) {
7494   (void) prefixq_and_encode(dst->encoding());
7495   emit_arith(0x81, 0xD0, dst, imm32);
7496 }
7497 
7498 void Assembler::adcq(Register dst, Address src) {
7499   InstructionMark im(this);
7500   prefixq(src, dst);
7501   emit_int8(0x13);
7502   emit_operand(dst, src);
7503 }
7504 
7505 void Assembler::adcq(Register dst, Register src) {
7506   (void) prefixq_and_encode(dst->encoding(), src->encoding());
7507   emit_arith(0x13, 0xC0, dst, src);
7508 }
7509 
7510 void Assembler::addq(Address dst, int32_t imm32) {
7511   InstructionMark im(this);
7512   prefixq(dst);
7513   emit_arith_operand(0x81, rax, dst,imm32);
7514 }
7515 
7516 void Assembler::addq(Address dst, Register src) {
7517   InstructionMark im(this);
7518   prefixq(dst, src);
7519   emit_int8(0x01);
7520   emit_operand(src, dst);
7521 }
7522 
7523 void Assembler::addq(Register dst, int32_t imm32) {
7524   (void) prefixq_and_encode(dst->encoding());
7525   emit_arith(0x81, 0xC0, dst, imm32);
7526 }
7527 
7528 void Assembler::addq(Register dst, Address src) {
7529   InstructionMark im(this);
7530   prefixq(src, dst);
7531   emit_int8(0x03);
7532   emit_operand(dst, src);
7533 }
7534 
7535 void Assembler::addq(Register dst, Register src) {
7536   (void) prefixq_and_encode(dst->encoding(), src->encoding());
7537   emit_arith(0x03, 0xC0, dst, src);
7538 }
7539 
7540 void Assembler::adcxq(Register dst, Register src) {
7541   //assert(VM_Version::supports_adx(), "adx instructions not supported");
7542   emit_int8((unsigned char)0x66);
7543   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
7544   emit_int8(0x0F);
7545   emit_int8(0x38);
7546   emit_int8((unsigned char)0xF6);
7547   emit_int8((unsigned char)(0xC0 | encode));
7548 }
7549 
7550 void Assembler::adoxq(Register dst, Register src) {
7551   //assert(VM_Version::supports_adx(), "adx instructions not supported");
7552   emit_int8((unsigned char)0xF3);
7553   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
7554   emit_int8(0x0F);
7555   emit_int8(0x38);
7556   emit_int8((unsigned char)0xF6);
7557   emit_int8((unsigned char)(0xC0 | encode));
7558 }
7559 
7560 void Assembler::andq(Address dst, int32_t imm32) {
7561   InstructionMark im(this);
7562   prefixq(dst);
7563   emit_int8((unsigned char)0x81);
7564   emit_operand(rsp, dst, 4);
7565   emit_int32(imm32);
7566 }
7567 
7568 void Assembler::andq(Register dst, int32_t imm32) {
7569   (void) prefixq_and_encode(dst->encoding());
7570   emit_arith(0x81, 0xE0, dst, imm32);
7571 }
7572 
7573 void Assembler::andq(Register dst, Address src) {
7574   InstructionMark im(this);
7575   prefixq(src, dst);
7576   emit_int8(0x23);
7577   emit_operand(dst, src);
7578 }
7579 
7580 void Assembler::andq(Register dst, Register src) {
7581   (void) prefixq_and_encode(dst->encoding(), src->encoding());
7582   emit_arith(0x23, 0xC0, dst, src);
7583 }
7584 
7585 void Assembler::andnq(Register dst, Register src1, Register src2) {
7586   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
7587   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
7588   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
7589   emit_int8((unsigned char)0xF2);
7590   emit_int8((unsigned char)(0xC0 | encode));
7591 }
7592 
7593 void Assembler::andnq(Register dst, Register src1, Address src2) {
7594   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
7595   InstructionMark im(this);
7596   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
7597   vex_prefix(src2, src1->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
7598   emit_int8((unsigned char)0xF2);
7599   emit_operand(dst, src2);
7600 }
7601 
7602 void Assembler::bsfq(Register dst, Register src) {
7603   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
7604   emit_int8(0x0F);
7605   emit_int8((unsigned char)0xBC);
7606   emit_int8((unsigned char)(0xC0 | encode));
7607 }
7608 
7609 void Assembler::bsrq(Register dst, Register src) {
7610   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
7611   emit_int8(0x0F);
7612   emit_int8((unsigned char)0xBD);
7613   emit_int8((unsigned char)(0xC0 | encode));
7614 }
7615 
7616 void Assembler::bswapq(Register reg) {
7617   int encode = prefixq_and_encode(reg->encoding());
7618   emit_int8(0x0F);
7619   emit_int8((unsigned char)(0xC8 | encode));
7620 }
7621 
7622 void Assembler::blsiq(Register dst, Register src) {
7623   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
7624   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
7625   int encode = vex_prefix_and_encode(rbx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
7626   emit_int8((unsigned char)0xF3);
7627   emit_int8((unsigned char)(0xC0 | encode));
7628 }
7629 
7630 void Assembler::blsiq(Register dst, Address src) {
7631   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
7632   InstructionMark im(this);
7633   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
7634   vex_prefix(src, dst->encoding(), rbx->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
7635   emit_int8((unsigned char)0xF3);
7636   emit_operand(rbx, src);
7637 }
7638 
7639 void Assembler::blsmskq(Register dst, Register src) {
7640   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
7641   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
7642   int encode = vex_prefix_and_encode(rdx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
7643   emit_int8((unsigned char)0xF3);
7644   emit_int8((unsigned char)(0xC0 | encode));
7645 }
7646 
7647 void Assembler::blsmskq(Register dst, Address src) {
7648   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
7649   InstructionMark im(this);
7650   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
7651   vex_prefix(src, dst->encoding(), rdx->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
7652   emit_int8((unsigned char)0xF3);
7653   emit_operand(rdx, src);
7654 }
7655 
7656 void Assembler::blsrq(Register dst, Register src) {
7657   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
7658   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
7659   int encode = vex_prefix_and_encode(rcx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
7660   emit_int8((unsigned char)0xF3);
7661   emit_int8((unsigned char)(0xC0 | encode));
7662 }
7663 
7664 void Assembler::blsrq(Register dst, Address src) {
7665   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
7666   InstructionMark im(this);
7667   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
7668   vex_prefix(src, dst->encoding(), rcx->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
7669   emit_int8((unsigned char)0xF3);
7670   emit_operand(rcx, src);
7671 }
7672 
7673 void Assembler::cdqq() {
7674   prefix(REX_W);
7675   emit_int8((unsigned char)0x99);
7676 }
7677 
7678 void Assembler::clflush(Address adr) {
7679   prefix(adr);
7680   emit_int8(0x0F);
7681   emit_int8((unsigned char)0xAE);
7682   emit_operand(rdi, adr);
7683 }
7684 
7685 void Assembler::cmovq(Condition cc, Register dst, Register src) {
7686   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
7687   emit_int8(0x0F);
7688   emit_int8(0x40 | cc);
7689   emit_int8((unsigned char)(0xC0 | encode));
7690 }
7691 
7692 void Assembler::cmovq(Condition cc, Register dst, Address src) {
7693   InstructionMark im(this);
7694   prefixq(src, dst);
7695   emit_int8(0x0F);
7696   emit_int8(0x40 | cc);
7697   emit_operand(dst, src);
7698 }
7699 
7700 void Assembler::cmpq(Address dst, int32_t imm32) {
7701   InstructionMark im(this);
7702   prefixq(dst);
7703   emit_int8((unsigned char)0x81);
7704   emit_operand(rdi, dst, 4);
7705   emit_int32(imm32);
7706 }
7707 
7708 void Assembler::cmpq(Register dst, int32_t imm32) {
7709   (void) prefixq_and_encode(dst->encoding());
7710   emit_arith(0x81, 0xF8, dst, imm32);
7711 }
7712 
7713 void Assembler::cmpq(Address dst, Register src) {
7714   InstructionMark im(this);
7715   prefixq(dst, src);
7716   emit_int8(0x3B);
7717   emit_operand(src, dst);
7718 }
7719 
7720 void Assembler::cmpq(Register dst, Register src) {
7721   (void) prefixq_and_encode(dst->encoding(), src->encoding());
7722   emit_arith(0x3B, 0xC0, dst, src);
7723 }
7724 
7725 void Assembler::cmpq(Register dst, Address  src) {
7726   InstructionMark im(this);
7727   prefixq(src, dst);
7728   emit_int8(0x3B);
7729   emit_operand(dst, src);
7730 }
7731 
7732 void Assembler::cmpxchgq(Register reg, Address adr) {
7733   InstructionMark im(this);
7734   prefixq(adr, reg);
7735   emit_int8(0x0F);
7736   emit_int8((unsigned char)0xB1);
7737   emit_operand(reg, adr);
7738 }
7739 
7740 void Assembler::cvtsi2sdq(XMMRegister dst, Register src) {
7741   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
7742   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
7743   int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
7744   emit_int8(0x2A);
7745   emit_int8((unsigned char)(0xC0 | encode));
7746 }
7747 
7748 void Assembler::cvtsi2sdq(XMMRegister dst, Address src) {
7749   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
7750   InstructionMark im(this);
7751   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
7752   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
7753   simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
7754   emit_int8(0x2A);
7755   emit_operand(dst, src);
7756 }
7757 
7758 void Assembler::cvtsi2ssq(XMMRegister dst, Address src) {
7759   NOT_LP64(assert(VM_Version::supports_sse(), ""));
7760   InstructionMark im(this);
7761   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
7762   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
7763   simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
7764   emit_int8(0x2A);
7765   emit_operand(dst, src);
7766 }
7767 
7768 void Assembler::cvttsd2siq(Register dst, XMMRegister src) {
7769   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
7770   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
7771   int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
7772   emit_int8(0x2C);
7773   emit_int8((unsigned char)(0xC0 | encode));
7774 }
7775 
7776 void Assembler::cvttss2siq(Register dst, XMMRegister src) {
7777   NOT_LP64(assert(VM_Version::supports_sse(), ""));
7778   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
7779   int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
7780   emit_int8(0x2C);
7781   emit_int8((unsigned char)(0xC0 | encode));
7782 }
7783 
7784 void Assembler::decl(Register dst) {
7785   // Don't use it directly. Use MacroAssembler::decrementl() instead.
7786   // Use two-byte form (one-byte form is a REX prefix in 64-bit mode)
7787   int encode = prefix_and_encode(dst->encoding());
7788   emit_int8((unsigned char)0xFF);
7789   emit_int8((unsigned char)(0xC8 | encode));
7790 }
7791 
7792 void Assembler::decq(Register dst) {
7793   // Don't use it directly. Use MacroAssembler::decrementq() instead.
7794   // Use two-byte form (one-byte from is a REX prefix in 64-bit mode)
7795   int encode = prefixq_and_encode(dst->encoding());
7796   emit_int8((unsigned char)0xFF);
7797   emit_int8(0xC8 | encode);
7798 }
7799 
7800 void Assembler::decq(Address dst) {
7801   // Don't use it directly. Use MacroAssembler::decrementq() instead.
7802   InstructionMark im(this);
7803   prefixq(dst);
7804   emit_int8((unsigned char)0xFF);
7805   emit_operand(rcx, dst);
7806 }
7807 
7808 void Assembler::fxrstor(Address src) {
7809   prefixq(src);
7810   emit_int8(0x0F);
7811   emit_int8((unsigned char)0xAE);
7812   emit_operand(as_Register(1), src);
7813 }
7814 
7815 void Assembler::xrstor(Address src) {
7816   prefixq(src);
7817   emit_int8(0x0F);
7818   emit_int8((unsigned char)0xAE);
7819   emit_operand(as_Register(5), src);
7820 }
7821 
7822 void Assembler::fxsave(Address dst) {
7823   prefixq(dst);
7824   emit_int8(0x0F);
7825   emit_int8((unsigned char)0xAE);
7826   emit_operand(as_Register(0), dst);
7827 }
7828 
7829 void Assembler::xsave(Address dst) {
7830   prefixq(dst);
7831   emit_int8(0x0F);
7832   emit_int8((unsigned char)0xAE);
7833   emit_operand(as_Register(4), dst);
7834 }
7835 
7836 void Assembler::idivq(Register src) {
7837   int encode = prefixq_and_encode(src->encoding());
7838   emit_int8((unsigned char)0xF7);
7839   emit_int8((unsigned char)(0xF8 | encode));
7840 }
7841 
7842 void Assembler::imulq(Register dst, Register src) {
7843   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
7844   emit_int8(0x0F);
7845   emit_int8((unsigned char)0xAF);
7846   emit_int8((unsigned char)(0xC0 | encode));
7847 }
7848 
7849 void Assembler::imulq(Register dst, Register src, int value) {
7850   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
7851   if (is8bit(value)) {
7852     emit_int8(0x6B);
7853     emit_int8((unsigned char)(0xC0 | encode));
7854     emit_int8(value & 0xFF);
7855   } else {
7856     emit_int8(0x69);
7857     emit_int8((unsigned char)(0xC0 | encode));
7858     emit_int32(value);
7859   }
7860 }
7861 
7862 void Assembler::imulq(Register dst, Address src) {
7863   InstructionMark im(this);
7864   prefixq(src, dst);
7865   emit_int8(0x0F);
7866   emit_int8((unsigned char) 0xAF);
7867   emit_operand(dst, src);
7868 }
7869 
7870 void Assembler::incl(Register dst) {
7871   // Don't use it directly. Use MacroAssembler::incrementl() instead.
7872   // Use two-byte form (one-byte from is a REX prefix in 64-bit mode)
7873   int encode = prefix_and_encode(dst->encoding());
7874   emit_int8((unsigned char)0xFF);
7875   emit_int8((unsigned char)(0xC0 | encode));
7876 }
7877 
7878 void Assembler::incq(Register dst) {
7879   // Don't use it directly. Use MacroAssembler::incrementq() instead.
7880   // Use two-byte form (one-byte from is a REX prefix in 64-bit mode)
7881   int encode = prefixq_and_encode(dst->encoding());
7882   emit_int8((unsigned char)0xFF);
7883   emit_int8((unsigned char)(0xC0 | encode));
7884 }
7885 
7886 void Assembler::incq(Address dst) {
7887   // Don't use it directly. Use MacroAssembler::incrementq() instead.
7888   InstructionMark im(this);
7889   prefixq(dst);
7890   emit_int8((unsigned char)0xFF);
7891   emit_operand(rax, dst);
7892 }
7893 
7894 void Assembler::lea(Register dst, Address src) {
7895   leaq(dst, src);
7896 }
7897 
7898 void Assembler::leaq(Register dst, Address src) {
7899   InstructionMark im(this);
7900   prefixq(src, dst);
7901   emit_int8((unsigned char)0x8D);
7902   emit_operand(dst, src);
7903 }
7904 
7905 void Assembler::mov64(Register dst, int64_t imm64) {
7906   InstructionMark im(this);
7907   int encode = prefixq_and_encode(dst->encoding());
7908   emit_int8((unsigned char)(0xB8 | encode));
7909   emit_int64(imm64);
7910 }
7911 
7912 void Assembler::mov_literal64(Register dst, intptr_t imm64, RelocationHolder const& rspec) {
7913   InstructionMark im(this);
7914   int encode = prefixq_and_encode(dst->encoding());
7915   emit_int8(0xB8 | encode);
7916   emit_data64(imm64, rspec);
7917 }
7918 
7919 void Assembler::mov_narrow_oop(Register dst, int32_t imm32, RelocationHolder const& rspec) {
7920   InstructionMark im(this);
7921   int encode = prefix_and_encode(dst->encoding());
7922   emit_int8((unsigned char)(0xB8 | encode));
7923   emit_data((int)imm32, rspec, narrow_oop_operand);
7924 }
7925 
7926 void Assembler::mov_narrow_oop(Address dst, int32_t imm32,  RelocationHolder const& rspec) {
7927   InstructionMark im(this);
7928   prefix(dst);
7929   emit_int8((unsigned char)0xC7);
7930   emit_operand(rax, dst, 4);
7931   emit_data((int)imm32, rspec, narrow_oop_operand);
7932 }
7933 
7934 void Assembler::cmp_narrow_oop(Register src1, int32_t imm32, RelocationHolder const& rspec) {
7935   InstructionMark im(this);
7936   int encode = prefix_and_encode(src1->encoding());
7937   emit_int8((unsigned char)0x81);
7938   emit_int8((unsigned char)(0xF8 | encode));
7939   emit_data((int)imm32, rspec, narrow_oop_operand);
7940 }
7941 
7942 void Assembler::cmp_narrow_oop(Address src1, int32_t imm32, RelocationHolder const& rspec) {
7943   InstructionMark im(this);
7944   prefix(src1);
7945   emit_int8((unsigned char)0x81);
7946   emit_operand(rax, src1, 4);
7947   emit_data((int)imm32, rspec, narrow_oop_operand);
7948 }
7949 
7950 void Assembler::lzcntq(Register dst, Register src) {
7951   assert(VM_Version::supports_lzcnt(), "encoding is treated as BSR");
7952   emit_int8((unsigned char)0xF3);
7953   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
7954   emit_int8(0x0F);
7955   emit_int8((unsigned char)0xBD);
7956   emit_int8((unsigned char)(0xC0 | encode));
7957 }
7958 
7959 void Assembler::movdq(XMMRegister dst, Register src) {
7960   // table D-1 says MMX/SSE2
7961   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
7962   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
7963   int encode = simd_prefix_and_encode(dst, xnoreg, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7964   emit_int8(0x6E);
7965   emit_int8((unsigned char)(0xC0 | encode));
7966 }
7967 
7968 void Assembler::movdq(Register dst, XMMRegister src) {
7969   // table D-1 says MMX/SSE2
7970   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
7971   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
7972   // swap src/dst to get correct prefix
7973   int encode = simd_prefix_and_encode(src, xnoreg, as_XMMRegister(dst->encoding()), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7974   emit_int8(0x7E);
7975   emit_int8((unsigned char)(0xC0 | encode));
7976 }
7977 
7978 void Assembler::movq(Register dst, Register src) {
7979   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
7980   emit_int8((unsigned char)0x8B);
7981   emit_int8((unsigned char)(0xC0 | encode));
7982 }
7983 
7984 void Assembler::movq(Register dst, Address src) {
7985   InstructionMark im(this);
7986   prefixq(src, dst);
7987   emit_int8((unsigned char)0x8B);
7988   emit_operand(dst, src);
7989 }
7990 
7991 void Assembler::movq(Address dst, Register src) {
7992   InstructionMark im(this);
7993   prefixq(dst, src);
7994   emit_int8((unsigned char)0x89);
7995   emit_operand(src, dst);
7996 }
7997 
7998 void Assembler::movsbq(Register dst, Address src) {
7999   InstructionMark im(this);
8000   prefixq(src, dst);
8001   emit_int8(0x0F);
8002   emit_int8((unsigned char)0xBE);
8003   emit_operand(dst, src);
8004 }
8005 
8006 void Assembler::movsbq(Register dst, Register src) {
8007   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
8008   emit_int8(0x0F);
8009   emit_int8((unsigned char)0xBE);
8010   emit_int8((unsigned char)(0xC0 | encode));
8011 }
8012 
8013 void Assembler::movslq(Register dst, int32_t imm32) {
8014   // dbx shows movslq(rcx, 3) as movq     $0x0000000049000000,(%rbx)
8015   // and movslq(r8, 3); as movl     $0x0000000048000000,(%rbx)
8016   // as a result we shouldn't use until tested at runtime...
8017   ShouldNotReachHere();
8018   InstructionMark im(this);
8019   int encode = prefixq_and_encode(dst->encoding());
8020   emit_int8((unsigned char)(0xC7 | encode));
8021   emit_int32(imm32);
8022 }
8023 
8024 void Assembler::movslq(Address dst, int32_t imm32) {
8025   assert(is_simm32(imm32), "lost bits");
8026   InstructionMark im(this);
8027   prefixq(dst);
8028   emit_int8((unsigned char)0xC7);
8029   emit_operand(rax, dst, 4);
8030   emit_int32(imm32);
8031 }
8032 
8033 void Assembler::movslq(Register dst, Address src) {
8034   InstructionMark im(this);
8035   prefixq(src, dst);
8036   emit_int8(0x63);
8037   emit_operand(dst, src);
8038 }
8039 
8040 void Assembler::movslq(Register dst, Register src) {
8041   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
8042   emit_int8(0x63);
8043   emit_int8((unsigned char)(0xC0 | encode));
8044 }
8045 
8046 void Assembler::movswq(Register dst, Address src) {
8047   InstructionMark im(this);
8048   prefixq(src, dst);
8049   emit_int8(0x0F);
8050   emit_int8((unsigned char)0xBF);
8051   emit_operand(dst, src);
8052 }
8053 
8054 void Assembler::movswq(Register dst, Register src) {
8055   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
8056   emit_int8((unsigned char)0x0F);
8057   emit_int8((unsigned char)0xBF);
8058   emit_int8((unsigned char)(0xC0 | encode));
8059 }
8060 
8061 void Assembler::movzbq(Register dst, Address src) {
8062   InstructionMark im(this);
8063   prefixq(src, dst);
8064   emit_int8((unsigned char)0x0F);
8065   emit_int8((unsigned char)0xB6);
8066   emit_operand(dst, src);
8067 }
8068 
8069 void Assembler::movzbq(Register dst, Register src) {
8070   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
8071   emit_int8(0x0F);
8072   emit_int8((unsigned char)0xB6);
8073   emit_int8(0xC0 | encode);
8074 }
8075 
8076 void Assembler::movzwq(Register dst, Address src) {
8077   InstructionMark im(this);
8078   prefixq(src, dst);
8079   emit_int8((unsigned char)0x0F);
8080   emit_int8((unsigned char)0xB7);
8081   emit_operand(dst, src);
8082 }
8083 
8084 void Assembler::movzwq(Register dst, Register src) {
8085   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
8086   emit_int8((unsigned char)0x0F);
8087   emit_int8((unsigned char)0xB7);
8088   emit_int8((unsigned char)(0xC0 | encode));
8089 }
8090 
8091 void Assembler::mulq(Address src) {
8092   InstructionMark im(this);
8093   prefixq(src);
8094   emit_int8((unsigned char)0xF7);
8095   emit_operand(rsp, src);
8096 }
8097 
8098 void Assembler::mulq(Register src) {
8099   int encode = prefixq_and_encode(src->encoding());
8100   emit_int8((unsigned char)0xF7);
8101   emit_int8((unsigned char)(0xE0 | encode));
8102 }
8103 
8104 void Assembler::mulxq(Register dst1, Register dst2, Register src) {
8105   assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
8106   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
8107   int encode = vex_prefix_and_encode(dst1->encoding(), dst2->encoding(), src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_38, &attributes);
8108   emit_int8((unsigned char)0xF6);
8109   emit_int8((unsigned char)(0xC0 | encode));
8110 }
8111 
8112 void Assembler::negq(Register dst) {
8113   int encode = prefixq_and_encode(dst->encoding());
8114   emit_int8((unsigned char)0xF7);
8115   emit_int8((unsigned char)(0xD8 | encode));
8116 }
8117 
8118 void Assembler::notq(Register dst) {
8119   int encode = prefixq_and_encode(dst->encoding());
8120   emit_int8((unsigned char)0xF7);
8121   emit_int8((unsigned char)(0xD0 | encode));
8122 }
8123 
8124 void Assembler::orq(Address dst, int32_t imm32) {
8125   InstructionMark im(this);
8126   prefixq(dst);
8127   emit_int8((unsigned char)0x81);
8128   emit_operand(rcx, dst, 4);
8129   emit_int32(imm32);
8130 }
8131 
8132 void Assembler::orq(Register dst, int32_t imm32) {
8133   (void) prefixq_and_encode(dst->encoding());
8134   emit_arith(0x81, 0xC8, dst, imm32);
8135 }
8136 
8137 void Assembler::orq(Register dst, Address src) {
8138   InstructionMark im(this);
8139   prefixq(src, dst);
8140   emit_int8(0x0B);
8141   emit_operand(dst, src);
8142 }
8143 
8144 void Assembler::orq(Register dst, Register src) {
8145   (void) prefixq_and_encode(dst->encoding(), src->encoding());
8146   emit_arith(0x0B, 0xC0, dst, src);
8147 }
8148 
8149 void Assembler::popa() { // 64bit
8150   movq(r15, Address(rsp, 0));
8151   movq(r14, Address(rsp, wordSize));
8152   movq(r13, Address(rsp, 2 * wordSize));
8153   movq(r12, Address(rsp, 3 * wordSize));
8154   movq(r11, Address(rsp, 4 * wordSize));
8155   movq(r10, Address(rsp, 5 * wordSize));
8156   movq(r9,  Address(rsp, 6 * wordSize));
8157   movq(r8,  Address(rsp, 7 * wordSize));
8158   movq(rdi, Address(rsp, 8 * wordSize));
8159   movq(rsi, Address(rsp, 9 * wordSize));
8160   movq(rbp, Address(rsp, 10 * wordSize));
8161   // skip rsp
8162   movq(rbx, Address(rsp, 12 * wordSize));
8163   movq(rdx, Address(rsp, 13 * wordSize));
8164   movq(rcx, Address(rsp, 14 * wordSize));
8165   movq(rax, Address(rsp, 15 * wordSize));
8166 
8167   addq(rsp, 16 * wordSize);
8168 }
8169 
8170 void Assembler::popcntq(Register dst, Address src) {
8171   assert(VM_Version::supports_popcnt(), "must support");
8172   InstructionMark im(this);
8173   emit_int8((unsigned char)0xF3);
8174   prefixq(src, dst);
8175   emit_int8((unsigned char)0x0F);
8176   emit_int8((unsigned char)0xB8);
8177   emit_operand(dst, src);
8178 }
8179 
8180 void Assembler::popcntq(Register dst, Register src) {
8181   assert(VM_Version::supports_popcnt(), "must support");
8182   emit_int8((unsigned char)0xF3);
8183   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
8184   emit_int8((unsigned char)0x0F);
8185   emit_int8((unsigned char)0xB8);
8186   emit_int8((unsigned char)(0xC0 | encode));
8187 }
8188 
8189 void Assembler::popq(Address dst) {
8190   InstructionMark im(this);
8191   prefixq(dst);
8192   emit_int8((unsigned char)0x8F);
8193   emit_operand(rax, dst);
8194 }
8195 
8196 void Assembler::pusha() { // 64bit
8197   // we have to store original rsp.  ABI says that 128 bytes
8198   // below rsp are local scratch.
8199   movq(Address(rsp, -5 * wordSize), rsp);
8200 
8201   subq(rsp, 16 * wordSize);
8202 
8203   movq(Address(rsp, 15 * wordSize), rax);
8204   movq(Address(rsp, 14 * wordSize), rcx);
8205   movq(Address(rsp, 13 * wordSize), rdx);
8206   movq(Address(rsp, 12 * wordSize), rbx);
8207   // skip rsp
8208   movq(Address(rsp, 10 * wordSize), rbp);
8209   movq(Address(rsp, 9 * wordSize), rsi);
8210   movq(Address(rsp, 8 * wordSize), rdi);
8211   movq(Address(rsp, 7 * wordSize), r8);
8212   movq(Address(rsp, 6 * wordSize), r9);
8213   movq(Address(rsp, 5 * wordSize), r10);
8214   movq(Address(rsp, 4 * wordSize), r11);
8215   movq(Address(rsp, 3 * wordSize), r12);
8216   movq(Address(rsp, 2 * wordSize), r13);
8217   movq(Address(rsp, wordSize), r14);
8218   movq(Address(rsp, 0), r15);
8219 }
8220 
8221 void Assembler::pushq(Address src) {
8222   InstructionMark im(this);
8223   prefixq(src);
8224   emit_int8((unsigned char)0xFF);
8225   emit_operand(rsi, src);
8226 }
8227 
8228 void Assembler::rclq(Register dst, int imm8) {
8229   assert(isShiftCount(imm8 >> 1), "illegal shift count");
8230   int encode = prefixq_and_encode(dst->encoding());
8231   if (imm8 == 1) {
8232     emit_int8((unsigned char)0xD1);
8233     emit_int8((unsigned char)(0xD0 | encode));
8234   } else {
8235     emit_int8((unsigned char)0xC1);
8236     emit_int8((unsigned char)(0xD0 | encode));
8237     emit_int8(imm8);
8238   }
8239 }
8240 
8241 void Assembler::rcrq(Register dst, int imm8) {
8242   assert(isShiftCount(imm8 >> 1), "illegal shift count");
8243   int encode = prefixq_and_encode(dst->encoding());
8244   if (imm8 == 1) {
8245     emit_int8((unsigned char)0xD1);
8246     emit_int8((unsigned char)(0xD8 | encode));
8247   } else {
8248     emit_int8((unsigned char)0xC1);
8249     emit_int8((unsigned char)(0xD8 | encode));
8250     emit_int8(imm8);
8251   }
8252 }
8253 
8254 void Assembler::rorq(Register dst, int imm8) {
8255   assert(isShiftCount(imm8 >> 1), "illegal shift count");
8256   int encode = prefixq_and_encode(dst->encoding());
8257   if (imm8 == 1) {
8258     emit_int8((unsigned char)0xD1);
8259     emit_int8((unsigned char)(0xC8 | encode));
8260   } else {
8261     emit_int8((unsigned char)0xC1);
8262     emit_int8((unsigned char)(0xc8 | encode));
8263     emit_int8(imm8);
8264   }
8265 }
8266 
8267 void Assembler::rorxq(Register dst, Register src, int imm8) {
8268   assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
8269   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
8270   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_3A, &attributes);
8271   emit_int8((unsigned char)0xF0);
8272   emit_int8((unsigned char)(0xC0 | encode));
8273   emit_int8(imm8);
8274 }
8275 
8276 void Assembler::sarq(Register dst, int imm8) {
8277   assert(isShiftCount(imm8 >> 1), "illegal shift count");
8278   int encode = prefixq_and_encode(dst->encoding());
8279   if (imm8 == 1) {
8280     emit_int8((unsigned char)0xD1);
8281     emit_int8((unsigned char)(0xF8 | encode));
8282   } else {
8283     emit_int8((unsigned char)0xC1);
8284     emit_int8((unsigned char)(0xF8 | encode));
8285     emit_int8(imm8);
8286   }
8287 }
8288 
8289 void Assembler::sarq(Register dst) {
8290   int encode = prefixq_and_encode(dst->encoding());
8291   emit_int8((unsigned char)0xD3);
8292   emit_int8((unsigned char)(0xF8 | encode));
8293 }
8294 
8295 void Assembler::sbbq(Address dst, int32_t imm32) {
8296   InstructionMark im(this);
8297   prefixq(dst);
8298   emit_arith_operand(0x81, rbx, dst, imm32);
8299 }
8300 
8301 void Assembler::sbbq(Register dst, int32_t imm32) {
8302   (void) prefixq_and_encode(dst->encoding());
8303   emit_arith(0x81, 0xD8, dst, imm32);
8304 }
8305 
8306 void Assembler::sbbq(Register dst, Address src) {
8307   InstructionMark im(this);
8308   prefixq(src, dst);
8309   emit_int8(0x1B);
8310   emit_operand(dst, src);
8311 }
8312 
8313 void Assembler::sbbq(Register dst, Register src) {
8314   (void) prefixq_and_encode(dst->encoding(), src->encoding());
8315   emit_arith(0x1B, 0xC0, dst, src);
8316 }
8317 
8318 void Assembler::shlq(Register dst, int imm8) {
8319   assert(isShiftCount(imm8 >> 1), "illegal shift count");
8320   int encode = prefixq_and_encode(dst->encoding());
8321   if (imm8 == 1) {
8322     emit_int8((unsigned char)0xD1);
8323     emit_int8((unsigned char)(0xE0 | encode));
8324   } else {
8325     emit_int8((unsigned char)0xC1);
8326     emit_int8((unsigned char)(0xE0 | encode));
8327     emit_int8(imm8);
8328   }
8329 }
8330 
8331 void Assembler::shlq(Register dst) {
8332   int encode = prefixq_and_encode(dst->encoding());
8333   emit_int8((unsigned char)0xD3);
8334   emit_int8((unsigned char)(0xE0 | encode));
8335 }
8336 
8337 void Assembler::shrq(Register dst, int imm8) {
8338   assert(isShiftCount(imm8 >> 1), "illegal shift count");
8339   int encode = prefixq_and_encode(dst->encoding());
8340   emit_int8((unsigned char)0xC1);
8341   emit_int8((unsigned char)(0xE8 | encode));
8342   emit_int8(imm8);
8343 }
8344 
8345 void Assembler::shrq(Register dst) {
8346   int encode = prefixq_and_encode(dst->encoding());
8347   emit_int8((unsigned char)0xD3);
8348   emit_int8(0xE8 | encode);
8349 }
8350 
8351 void Assembler::subq(Address dst, int32_t imm32) {
8352   InstructionMark im(this);
8353   prefixq(dst);
8354   emit_arith_operand(0x81, rbp, dst, imm32);
8355 }
8356 
8357 void Assembler::subq(Address dst, Register src) {
8358   InstructionMark im(this);
8359   prefixq(dst, src);
8360   emit_int8(0x29);
8361   emit_operand(src, dst);
8362 }
8363 
8364 void Assembler::subq(Register dst, int32_t imm32) {
8365   (void) prefixq_and_encode(dst->encoding());
8366   emit_arith(0x81, 0xE8, dst, imm32);
8367 }
8368 
8369 // Force generation of a 4 byte immediate value even if it fits into 8bit
8370 void Assembler::subq_imm32(Register dst, int32_t imm32) {
8371   (void) prefixq_and_encode(dst->encoding());
8372   emit_arith_imm32(0x81, 0xE8, dst, imm32);
8373 }
8374 
8375 void Assembler::subq(Register dst, Address src) {
8376   InstructionMark im(this);
8377   prefixq(src, dst);
8378   emit_int8(0x2B);
8379   emit_operand(dst, src);
8380 }
8381 
8382 void Assembler::subq(Register dst, Register src) {
8383   (void) prefixq_and_encode(dst->encoding(), src->encoding());
8384   emit_arith(0x2B, 0xC0, dst, src);
8385 }
8386 
8387 void Assembler::testq(Register dst, int32_t imm32) {
8388   // not using emit_arith because test
8389   // doesn't support sign-extension of
8390   // 8bit operands
8391   int encode = dst->encoding();
8392   if (encode == 0) {
8393     prefix(REX_W);
8394     emit_int8((unsigned char)0xA9);
8395   } else {
8396     encode = prefixq_and_encode(encode);
8397     emit_int8((unsigned char)0xF7);
8398     emit_int8((unsigned char)(0xC0 | encode));
8399   }
8400   emit_int32(imm32);
8401 }
8402 
8403 void Assembler::testq(Register dst, Register src) {
8404   (void) prefixq_and_encode(dst->encoding(), src->encoding());
8405   emit_arith(0x85, 0xC0, dst, src);
8406 }
8407 
8408 void Assembler::xaddq(Address dst, Register src) {
8409   InstructionMark im(this);
8410   prefixq(dst, src);
8411   emit_int8(0x0F);
8412   emit_int8((unsigned char)0xC1);
8413   emit_operand(src, dst);
8414 }
8415 
8416 void Assembler::xchgq(Register dst, Address src) {
8417   InstructionMark im(this);
8418   prefixq(src, dst);
8419   emit_int8((unsigned char)0x87);
8420   emit_operand(dst, src);
8421 }
8422 
8423 void Assembler::xchgq(Register dst, Register src) {
8424   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
8425   emit_int8((unsigned char)0x87);
8426   emit_int8((unsigned char)(0xc0 | encode));
8427 }
8428 
8429 void Assembler::xorq(Register dst, Register src) {
8430   (void) prefixq_and_encode(dst->encoding(), src->encoding());
8431   emit_arith(0x33, 0xC0, dst, src);
8432 }
8433 
8434 void Assembler::xorq(Register dst, Address src) {
8435   InstructionMark im(this);
8436   prefixq(src, dst);
8437   emit_int8(0x33);
8438   emit_operand(dst, src);
8439 }
8440 
8441 #endif // !LP64