1 /*
   2  * Copyright (c) 1997, 2018, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #include "precompiled.hpp"
  26 #include "asm/assembler.hpp"
  27 #include "asm/assembler.inline.hpp"
  28 #include "gc/shared/cardTableBarrierSet.hpp"
  29 #include "gc/shared/collectedHeap.inline.hpp"
  30 #include "interpreter/interpreter.hpp"
  31 #include "memory/resourceArea.hpp"
  32 #include "prims/methodHandles.hpp"
  33 #include "runtime/biasedLocking.hpp"
  34 #include "runtime/objectMonitor.hpp"
  35 #include "runtime/os.hpp"
  36 #include "runtime/sharedRuntime.hpp"
  37 #include "runtime/stubRoutines.hpp"
  38 #include "utilities/macros.hpp"
  39 
  40 #ifdef PRODUCT
  41 #define BLOCK_COMMENT(str) /* nothing */
  42 #define STOP(error) stop(error)
  43 #else
  44 #define BLOCK_COMMENT(str) block_comment(str)
  45 #define STOP(error) block_comment(error); stop(error)
  46 #endif
  47 
  48 #define BIND(label) bind(label); BLOCK_COMMENT(#label ":")
  49 // Implementation of AddressLiteral
  50 
  51 // A 2-D table for managing compressed displacement(disp8) on EVEX enabled platforms.
  52 unsigned char tuple_table[Assembler::EVEX_ETUP + 1][Assembler::AVX_512bit + 1] = {
  53   // -----------------Table 4.5 -------------------- //
  54   16, 32, 64,  // EVEX_FV(0)
  55   4,  4,  4,   // EVEX_FV(1) - with Evex.b
  56   16, 32, 64,  // EVEX_FV(2) - with Evex.w
  57   8,  8,  8,   // EVEX_FV(3) - with Evex.w and Evex.b
  58   8,  16, 32,  // EVEX_HV(0)
  59   4,  4,  4,   // EVEX_HV(1) - with Evex.b
  60   // -----------------Table 4.6 -------------------- //
  61   16, 32, 64,  // EVEX_FVM(0)
  62   1,  1,  1,   // EVEX_T1S(0)
  63   2,  2,  2,   // EVEX_T1S(1)
  64   4,  4,  4,   // EVEX_T1S(2)
  65   8,  8,  8,   // EVEX_T1S(3)
  66   4,  4,  4,   // EVEX_T1F(0)
  67   8,  8,  8,   // EVEX_T1F(1)
  68   8,  8,  8,   // EVEX_T2(0)
  69   0,  16, 16,  // EVEX_T2(1)
  70   0,  16, 16,  // EVEX_T4(0)
  71   0,  0,  32,  // EVEX_T4(1)
  72   0,  0,  32,  // EVEX_T8(0)
  73   8,  16, 32,  // EVEX_HVM(0)
  74   4,  8,  16,  // EVEX_QVM(0)
  75   2,  4,  8,   // EVEX_OVM(0)
  76   16, 16, 16,  // EVEX_M128(0)
  77   8,  32, 64,  // EVEX_DUP(0)
  78   0,  0,  0    // EVEX_NTUP
  79 };
  80 
  81 AddressLiteral::AddressLiteral(address target, relocInfo::relocType rtype) {
  82   _is_lval = false;
  83   _target = target;
  84   switch (rtype) {
  85   case relocInfo::oop_type:
  86   case relocInfo::metadata_type:
  87     // Oops are a special case. Normally they would be their own section
  88     // but in cases like icBuffer they are literals in the code stream that
  89     // we don't have a section for. We use none so that we get a literal address
  90     // which is always patchable.
  91     break;
  92   case relocInfo::external_word_type:
  93     _rspec = external_word_Relocation::spec(target);
  94     break;
  95   case relocInfo::internal_word_type:
  96     _rspec = internal_word_Relocation::spec(target);
  97     break;
  98   case relocInfo::opt_virtual_call_type:
  99     _rspec = opt_virtual_call_Relocation::spec();
 100     break;
 101   case relocInfo::static_call_type:
 102     _rspec = static_call_Relocation::spec();
 103     break;
 104   case relocInfo::runtime_call_type:
 105     _rspec = runtime_call_Relocation::spec();
 106     break;
 107   case relocInfo::poll_type:
 108   case relocInfo::poll_return_type:
 109     _rspec = Relocation::spec_simple(rtype);
 110     break;
 111   case relocInfo::none:
 112     break;
 113   default:
 114     ShouldNotReachHere();
 115     break;
 116   }
 117 }
 118 
 119 // Implementation of Address
 120 
 121 #ifdef _LP64
 122 
 123 Address Address::make_array(ArrayAddress adr) {
 124   // Not implementable on 64bit machines
 125   // Should have been handled higher up the call chain.
 126   ShouldNotReachHere();
 127   return Address();
 128 }
 129 
 130 // exceedingly dangerous constructor
 131 Address::Address(int disp, address loc, relocInfo::relocType rtype) {
 132   _base  = noreg;
 133   _index = noreg;
 134   _scale = no_scale;
 135   _disp  = disp;
 136   _xmmindex = xnoreg;
 137   _isxmmindex = false;
 138   switch (rtype) {
 139     case relocInfo::external_word_type:
 140       _rspec = external_word_Relocation::spec(loc);
 141       break;
 142     case relocInfo::internal_word_type:
 143       _rspec = internal_word_Relocation::spec(loc);
 144       break;
 145     case relocInfo::runtime_call_type:
 146       // HMM
 147       _rspec = runtime_call_Relocation::spec();
 148       break;
 149     case relocInfo::poll_type:
 150     case relocInfo::poll_return_type:
 151       _rspec = Relocation::spec_simple(rtype);
 152       break;
 153     case relocInfo::none:
 154       break;
 155     default:
 156       ShouldNotReachHere();
 157   }
 158 }
 159 #else // LP64
 160 
 161 Address Address::make_array(ArrayAddress adr) {
 162   AddressLiteral base = adr.base();
 163   Address index = adr.index();
 164   assert(index._disp == 0, "must not have disp"); // maybe it can?
 165   Address array(index._base, index._index, index._scale, (intptr_t) base.target());
 166   array._rspec = base._rspec;
 167   return array;
 168 }
 169 
 170 // exceedingly dangerous constructor
 171 Address::Address(address loc, RelocationHolder spec) {
 172   _base  = noreg;
 173   _index = noreg;
 174   _scale = no_scale;
 175   _disp  = (intptr_t) loc;
 176   _rspec = spec;
 177   _xmmindex = xnoreg;
 178   _isxmmindex = false;
 179 }
 180 
 181 #endif // _LP64
 182 
 183 
 184 
 185 // Convert the raw encoding form into the form expected by the constructor for
 186 // Address.  An index of 4 (rsp) corresponds to having no index, so convert
 187 // that to noreg for the Address constructor.
 188 Address Address::make_raw(int base, int index, int scale, int disp, relocInfo::relocType disp_reloc) {
 189   RelocationHolder rspec;
 190   if (disp_reloc != relocInfo::none) {
 191     rspec = Relocation::spec_simple(disp_reloc);
 192   }
 193   bool valid_index = index != rsp->encoding();
 194   if (valid_index) {
 195     Address madr(as_Register(base), as_Register(index), (Address::ScaleFactor)scale, in_ByteSize(disp));
 196     madr._rspec = rspec;
 197     return madr;
 198   } else {
 199     Address madr(as_Register(base), noreg, Address::no_scale, in_ByteSize(disp));
 200     madr._rspec = rspec;
 201     return madr;
 202   }
 203 }
 204 
 205 // Implementation of Assembler
 206 
 207 int AbstractAssembler::code_fill_byte() {
 208   return (u_char)'\xF4'; // hlt
 209 }
 210 
 211 // make this go away someday
 212 void Assembler::emit_data(jint data, relocInfo::relocType rtype, int format) {
 213   if (rtype == relocInfo::none)
 214     emit_int32(data);
 215   else
 216     emit_data(data, Relocation::spec_simple(rtype), format);
 217 }
 218 
 219 void Assembler::emit_data(jint data, RelocationHolder const& rspec, int format) {
 220   assert(imm_operand == 0, "default format must be immediate in this file");
 221   assert(inst_mark() != NULL, "must be inside InstructionMark");
 222   if (rspec.type() !=  relocInfo::none) {
 223     #ifdef ASSERT
 224       check_relocation(rspec, format);
 225     #endif
 226     // Do not use AbstractAssembler::relocate, which is not intended for
 227     // embedded words.  Instead, relocate to the enclosing instruction.
 228 
 229     // hack. call32 is too wide for mask so use disp32
 230     if (format == call32_operand)
 231       code_section()->relocate(inst_mark(), rspec, disp32_operand);
 232     else
 233       code_section()->relocate(inst_mark(), rspec, format);
 234   }
 235   emit_int32(data);
 236 }
 237 
 238 static int encode(Register r) {
 239   int enc = r->encoding();
 240   if (enc >= 8) {
 241     enc -= 8;
 242   }
 243   return enc;
 244 }
 245 
 246 void Assembler::emit_arith_b(int op1, int op2, Register dst, int imm8) {
 247   assert(dst->has_byte_register(), "must have byte register");
 248   assert(isByte(op1) && isByte(op2), "wrong opcode");
 249   assert(isByte(imm8), "not a byte");
 250   assert((op1 & 0x01) == 0, "should be 8bit operation");
 251   emit_int8(op1);
 252   emit_int8(op2 | encode(dst));
 253   emit_int8(imm8);
 254 }
 255 
 256 
 257 void Assembler::emit_arith(int op1, int op2, Register dst, int32_t imm32) {
 258   assert(isByte(op1) && isByte(op2), "wrong opcode");
 259   assert((op1 & 0x01) == 1, "should be 32bit operation");
 260   assert((op1 & 0x02) == 0, "sign-extension bit should not be set");
 261   if (is8bit(imm32)) {
 262     emit_int8(op1 | 0x02); // set sign bit
 263     emit_int8(op2 | encode(dst));
 264     emit_int8(imm32 & 0xFF);
 265   } else {
 266     emit_int8(op1);
 267     emit_int8(op2 | encode(dst));
 268     emit_int32(imm32);
 269   }
 270 }
 271 
 272 // Force generation of a 4 byte immediate value even if it fits into 8bit
 273 void Assembler::emit_arith_imm32(int op1, int op2, Register dst, int32_t imm32) {
 274   assert(isByte(op1) && isByte(op2), "wrong opcode");
 275   assert((op1 & 0x01) == 1, "should be 32bit operation");
 276   assert((op1 & 0x02) == 0, "sign-extension bit should not be set");
 277   emit_int8(op1);
 278   emit_int8(op2 | encode(dst));
 279   emit_int32(imm32);
 280 }
 281 
 282 // immediate-to-memory forms
 283 void Assembler::emit_arith_operand(int op1, Register rm, Address adr, int32_t imm32) {
 284   assert((op1 & 0x01) == 1, "should be 32bit operation");
 285   assert((op1 & 0x02) == 0, "sign-extension bit should not be set");
 286   if (is8bit(imm32)) {
 287     emit_int8(op1 | 0x02); // set sign bit
 288     emit_operand(rm, adr, 1);
 289     emit_int8(imm32 & 0xFF);
 290   } else {
 291     emit_int8(op1);
 292     emit_operand(rm, adr, 4);
 293     emit_int32(imm32);
 294   }
 295 }
 296 
 297 
 298 void Assembler::emit_arith(int op1, int op2, Register dst, Register src) {
 299   assert(isByte(op1) && isByte(op2), "wrong opcode");
 300   emit_int8(op1);
 301   emit_int8(op2 | encode(dst) << 3 | encode(src));
 302 }
 303 
 304 
 305 bool Assembler::query_compressed_disp_byte(int disp, bool is_evex_inst, int vector_len,
 306                                            int cur_tuple_type, int in_size_in_bits, int cur_encoding) {
 307   int mod_idx = 0;
 308   // We will test if the displacement fits the compressed format and if so
 309   // apply the compression to the displacment iff the result is8bit.
 310   if (VM_Version::supports_evex() && is_evex_inst) {
 311     switch (cur_tuple_type) {
 312     case EVEX_FV:
 313       if ((cur_encoding & VEX_W) == VEX_W) {
 314         mod_idx = ((cur_encoding & EVEX_Rb) == EVEX_Rb) ? 3 : 2;
 315       } else {
 316         mod_idx = ((cur_encoding & EVEX_Rb) == EVEX_Rb) ? 1 : 0;
 317       }
 318       break;
 319 
 320     case EVEX_HV:
 321       mod_idx = ((cur_encoding & EVEX_Rb) == EVEX_Rb) ? 1 : 0;
 322       break;
 323 
 324     case EVEX_FVM:
 325       break;
 326 
 327     case EVEX_T1S:
 328       switch (in_size_in_bits) {
 329       case EVEX_8bit:
 330         break;
 331 
 332       case EVEX_16bit:
 333         mod_idx = 1;
 334         break;
 335 
 336       case EVEX_32bit:
 337         mod_idx = 2;
 338         break;
 339 
 340       case EVEX_64bit:
 341         mod_idx = 3;
 342         break;
 343       }
 344       break;
 345 
 346     case EVEX_T1F:
 347     case EVEX_T2:
 348     case EVEX_T4:
 349       mod_idx = (in_size_in_bits == EVEX_64bit) ? 1 : 0;
 350       break;
 351 
 352     case EVEX_T8:
 353       break;
 354 
 355     case EVEX_HVM:
 356       break;
 357 
 358     case EVEX_QVM:
 359       break;
 360 
 361     case EVEX_OVM:
 362       break;
 363 
 364     case EVEX_M128:
 365       break;
 366 
 367     case EVEX_DUP:
 368       break;
 369 
 370     default:
 371       assert(0, "no valid evex tuple_table entry");
 372       break;
 373     }
 374 
 375     if (vector_len >= AVX_128bit && vector_len <= AVX_512bit) {
 376       int disp_factor = tuple_table[cur_tuple_type + mod_idx][vector_len];
 377       if ((disp % disp_factor) == 0) {
 378         int new_disp = disp / disp_factor;
 379         if ((-0x80 <= new_disp && new_disp < 0x80)) {
 380           disp = new_disp;
 381         }
 382       } else {
 383         return false;
 384       }
 385     }
 386   }
 387   return (-0x80 <= disp && disp < 0x80);
 388 }
 389 
 390 
 391 bool Assembler::emit_compressed_disp_byte(int &disp) {
 392   int mod_idx = 0;
 393   // We will test if the displacement fits the compressed format and if so
 394   // apply the compression to the displacment iff the result is8bit.
 395   if (VM_Version::supports_evex() && _attributes && _attributes->is_evex_instruction()) {
 396     int evex_encoding = _attributes->get_evex_encoding();
 397     int tuple_type = _attributes->get_tuple_type();
 398     switch (tuple_type) {
 399     case EVEX_FV:
 400       if ((evex_encoding & VEX_W) == VEX_W) {
 401         mod_idx = ((evex_encoding & EVEX_Rb) == EVEX_Rb) ? 3 : 2;
 402       } else {
 403         mod_idx = ((evex_encoding & EVEX_Rb) == EVEX_Rb) ? 1 : 0;
 404       }
 405       break;
 406 
 407     case EVEX_HV:
 408       mod_idx = ((evex_encoding & EVEX_Rb) == EVEX_Rb) ? 1 : 0;
 409       break;
 410 
 411     case EVEX_FVM:
 412       break;
 413 
 414     case EVEX_T1S:
 415       switch (_attributes->get_input_size()) {
 416       case EVEX_8bit:
 417         break;
 418 
 419       case EVEX_16bit:
 420         mod_idx = 1;
 421         break;
 422 
 423       case EVEX_32bit:
 424         mod_idx = 2;
 425         break;
 426 
 427       case EVEX_64bit:
 428         mod_idx = 3;
 429         break;
 430       }
 431       break;
 432 
 433     case EVEX_T1F:
 434     case EVEX_T2:
 435     case EVEX_T4:
 436       mod_idx = (_attributes->get_input_size() == EVEX_64bit) ? 1 : 0;
 437       break;
 438 
 439     case EVEX_T8:
 440       break;
 441 
 442     case EVEX_HVM:
 443       break;
 444 
 445     case EVEX_QVM:
 446       break;
 447 
 448     case EVEX_OVM:
 449       break;
 450 
 451     case EVEX_M128:
 452       break;
 453 
 454     case EVEX_DUP:
 455       break;
 456 
 457     default:
 458       assert(0, "no valid evex tuple_table entry");
 459       break;
 460     }
 461 
 462     int vector_len = _attributes->get_vector_len();
 463     if (vector_len >= AVX_128bit && vector_len <= AVX_512bit) {
 464       int disp_factor = tuple_table[tuple_type + mod_idx][vector_len];
 465       if ((disp % disp_factor) == 0) {
 466         int new_disp = disp / disp_factor;
 467         if (is8bit(new_disp)) {
 468           disp = new_disp;
 469         }
 470       } else {
 471         return false;
 472       }
 473     }
 474   }
 475   return is8bit(disp);
 476 }
 477 
 478 
 479 void Assembler::emit_operand(Register reg, Register base, Register index,
 480                              Address::ScaleFactor scale, int disp,
 481                              RelocationHolder const& rspec,
 482                              int rip_relative_correction) {
 483   relocInfo::relocType rtype = (relocInfo::relocType) rspec.type();
 484 
 485   // Encode the registers as needed in the fields they are used in
 486 
 487   int regenc = encode(reg) << 3;
 488   int indexenc = index->is_valid() ? encode(index) << 3 : 0;
 489   int baseenc = base->is_valid() ? encode(base) : 0;
 490 
 491   if (base->is_valid()) {
 492     if (index->is_valid()) {
 493       assert(scale != Address::no_scale, "inconsistent address");
 494       // [base + index*scale + disp]
 495       if (disp == 0 && rtype == relocInfo::none  &&
 496           base != rbp LP64_ONLY(&& base != r13)) {
 497         // [base + index*scale]
 498         // [00 reg 100][ss index base]
 499         assert(index != rsp, "illegal addressing mode");
 500         emit_int8(0x04 | regenc);
 501         emit_int8(scale << 6 | indexenc | baseenc);
 502       } else if (emit_compressed_disp_byte(disp) && rtype == relocInfo::none) {
 503         // [base + index*scale + imm8]
 504         // [01 reg 100][ss index base] imm8
 505         assert(index != rsp, "illegal addressing mode");
 506         emit_int8(0x44 | regenc);
 507         emit_int8(scale << 6 | indexenc | baseenc);
 508         emit_int8(disp & 0xFF);
 509       } else {
 510         // [base + index*scale + disp32]
 511         // [10 reg 100][ss index base] disp32
 512         assert(index != rsp, "illegal addressing mode");
 513         emit_int8(0x84 | regenc);
 514         emit_int8(scale << 6 | indexenc | baseenc);
 515         emit_data(disp, rspec, disp32_operand);
 516       }
 517     } else if (base == rsp LP64_ONLY(|| base == r12)) {
 518       // [rsp + disp]
 519       if (disp == 0 && rtype == relocInfo::none) {
 520         // [rsp]
 521         // [00 reg 100][00 100 100]
 522         emit_int8(0x04 | regenc);
 523         emit_int8(0x24);
 524       } else if (emit_compressed_disp_byte(disp) && rtype == relocInfo::none) {
 525         // [rsp + imm8]
 526         // [01 reg 100][00 100 100] disp8
 527         emit_int8(0x44 | regenc);
 528         emit_int8(0x24);
 529         emit_int8(disp & 0xFF);
 530       } else {
 531         // [rsp + imm32]
 532         // [10 reg 100][00 100 100] disp32
 533         emit_int8(0x84 | regenc);
 534         emit_int8(0x24);
 535         emit_data(disp, rspec, disp32_operand);
 536       }
 537     } else {
 538       // [base + disp]
 539       assert(base != rsp LP64_ONLY(&& base != r12), "illegal addressing mode");
 540       if (disp == 0 && rtype == relocInfo::none &&
 541           base != rbp LP64_ONLY(&& base != r13)) {
 542         // [base]
 543         // [00 reg base]
 544         emit_int8(0x00 | regenc | baseenc);
 545       } else if (emit_compressed_disp_byte(disp) && rtype == relocInfo::none) {
 546         // [base + disp8]
 547         // [01 reg base] disp8
 548         emit_int8(0x40 | regenc | baseenc);
 549         emit_int8(disp & 0xFF);
 550       } else {
 551         // [base + disp32]
 552         // [10 reg base] disp32
 553         emit_int8(0x80 | regenc | baseenc);
 554         emit_data(disp, rspec, disp32_operand);
 555       }
 556     }
 557   } else {
 558     if (index->is_valid()) {
 559       assert(scale != Address::no_scale, "inconsistent address");
 560       // [index*scale + disp]
 561       // [00 reg 100][ss index 101] disp32
 562       assert(index != rsp, "illegal addressing mode");
 563       emit_int8(0x04 | regenc);
 564       emit_int8(scale << 6 | indexenc | 0x05);
 565       emit_data(disp, rspec, disp32_operand);
 566     } else if (rtype != relocInfo::none ) {
 567       // [disp] (64bit) RIP-RELATIVE (32bit) abs
 568       // [00 000 101] disp32
 569 
 570       emit_int8(0x05 | regenc);
 571       // Note that the RIP-rel. correction applies to the generated
 572       // disp field, but _not_ to the target address in the rspec.
 573 
 574       // disp was created by converting the target address minus the pc
 575       // at the start of the instruction. That needs more correction here.
 576       // intptr_t disp = target - next_ip;
 577       assert(inst_mark() != NULL, "must be inside InstructionMark");
 578       address next_ip = pc() + sizeof(int32_t) + rip_relative_correction;
 579       int64_t adjusted = disp;
 580       // Do rip-rel adjustment for 64bit
 581       LP64_ONLY(adjusted -=  (next_ip - inst_mark()));
 582       assert(is_simm32(adjusted),
 583              "must be 32bit offset (RIP relative address)");
 584       emit_data((int32_t) adjusted, rspec, disp32_operand);
 585 
 586     } else {
 587       // 32bit never did this, did everything as the rip-rel/disp code above
 588       // [disp] ABSOLUTE
 589       // [00 reg 100][00 100 101] disp32
 590       emit_int8(0x04 | regenc);
 591       emit_int8(0x25);
 592       emit_data(disp, rspec, disp32_operand);
 593     }
 594   }
 595 }
 596 
 597 void Assembler::emit_operand(XMMRegister reg, Register base, Register index,
 598                              Address::ScaleFactor scale, int disp,
 599                              RelocationHolder const& rspec) {
 600   if (UseAVX > 2) {
 601     int xreg_enc = reg->encoding();
 602     if (xreg_enc > 15) {
 603       XMMRegister new_reg = as_XMMRegister(xreg_enc & 0xf);
 604       emit_operand((Register)new_reg, base, index, scale, disp, rspec);
 605       return;
 606     }
 607   }
 608   emit_operand((Register)reg, base, index, scale, disp, rspec);
 609 }
 610 
 611 void Assembler::emit_operand(XMMRegister reg, Register base, XMMRegister index,
 612                              Address::ScaleFactor scale, int disp,
 613                              RelocationHolder const& rspec) {
 614   if (UseAVX > 2) {
 615     int xreg_enc = reg->encoding();
 616     int xmmindex_enc = index->encoding();
 617     XMMRegister new_reg = as_XMMRegister(xreg_enc & 0xf);
 618     XMMRegister new_index = as_XMMRegister(xmmindex_enc & 0xf);
 619     emit_operand((Register)new_reg, base, (Register)new_index, scale, disp, rspec);
 620   } else {
 621     emit_operand((Register)reg, base, (Register)index, scale, disp, rspec);
 622   }
 623 }
 624 
 625 
 626 // Secret local extension to Assembler::WhichOperand:
 627 #define end_pc_operand (_WhichOperand_limit)
 628 
 629 address Assembler::locate_operand(address inst, WhichOperand which) {
 630   // Decode the given instruction, and return the address of
 631   // an embedded 32-bit operand word.
 632 
 633   // If "which" is disp32_operand, selects the displacement portion
 634   // of an effective address specifier.
 635   // If "which" is imm64_operand, selects the trailing immediate constant.
 636   // If "which" is call32_operand, selects the displacement of a call or jump.
 637   // Caller is responsible for ensuring that there is such an operand,
 638   // and that it is 32/64 bits wide.
 639 
 640   // If "which" is end_pc_operand, find the end of the instruction.
 641 
 642   address ip = inst;
 643   bool is_64bit = false;
 644 
 645   debug_only(bool has_disp32 = false);
 646   int tail_size = 0; // other random bytes (#32, #16, etc.) at end of insn
 647 
 648   again_after_prefix:
 649   switch (0xFF & *ip++) {
 650 
 651   // These convenience macros generate groups of "case" labels for the switch.
 652 #define REP4(x) (x)+0: case (x)+1: case (x)+2: case (x)+3
 653 #define REP8(x) (x)+0: case (x)+1: case (x)+2: case (x)+3: \
 654              case (x)+4: case (x)+5: case (x)+6: case (x)+7
 655 #define REP16(x) REP8((x)+0): \
 656               case REP8((x)+8)
 657 
 658   case CS_segment:
 659   case SS_segment:
 660   case DS_segment:
 661   case ES_segment:
 662   case FS_segment:
 663   case GS_segment:
 664     // Seems dubious
 665     LP64_ONLY(assert(false, "shouldn't have that prefix"));
 666     assert(ip == inst+1, "only one prefix allowed");
 667     goto again_after_prefix;
 668 
 669   case 0x67:
 670   case REX:
 671   case REX_B:
 672   case REX_X:
 673   case REX_XB:
 674   case REX_R:
 675   case REX_RB:
 676   case REX_RX:
 677   case REX_RXB:
 678     NOT_LP64(assert(false, "64bit prefixes"));
 679     goto again_after_prefix;
 680 
 681   case REX_W:
 682   case REX_WB:
 683   case REX_WX:
 684   case REX_WXB:
 685   case REX_WR:
 686   case REX_WRB:
 687   case REX_WRX:
 688   case REX_WRXB:
 689     NOT_LP64(assert(false, "64bit prefixes"));
 690     is_64bit = true;
 691     goto again_after_prefix;
 692 
 693   case 0xFF: // pushq a; decl a; incl a; call a; jmp a
 694   case 0x88: // movb a, r
 695   case 0x89: // movl a, r
 696   case 0x8A: // movb r, a
 697   case 0x8B: // movl r, a
 698   case 0x8F: // popl a
 699     debug_only(has_disp32 = true);
 700     break;
 701 
 702   case 0x68: // pushq #32
 703     if (which == end_pc_operand) {
 704       return ip + 4;
 705     }
 706     assert(which == imm_operand && !is_64bit, "pushl has no disp32 or 64bit immediate");
 707     return ip;                  // not produced by emit_operand
 708 
 709   case 0x66: // movw ... (size prefix)
 710     again_after_size_prefix2:
 711     switch (0xFF & *ip++) {
 712     case REX:
 713     case REX_B:
 714     case REX_X:
 715     case REX_XB:
 716     case REX_R:
 717     case REX_RB:
 718     case REX_RX:
 719     case REX_RXB:
 720     case REX_W:
 721     case REX_WB:
 722     case REX_WX:
 723     case REX_WXB:
 724     case REX_WR:
 725     case REX_WRB:
 726     case REX_WRX:
 727     case REX_WRXB:
 728       NOT_LP64(assert(false, "64bit prefix found"));
 729       goto again_after_size_prefix2;
 730     case 0x8B: // movw r, a
 731     case 0x89: // movw a, r
 732       debug_only(has_disp32 = true);
 733       break;
 734     case 0xC7: // movw a, #16
 735       debug_only(has_disp32 = true);
 736       tail_size = 2;  // the imm16
 737       break;
 738     case 0x0F: // several SSE/SSE2 variants
 739       ip--;    // reparse the 0x0F
 740       goto again_after_prefix;
 741     default:
 742       ShouldNotReachHere();
 743     }
 744     break;
 745 
 746   case REP8(0xB8): // movl/q r, #32/#64(oop?)
 747     if (which == end_pc_operand)  return ip + (is_64bit ? 8 : 4);
 748     // these asserts are somewhat nonsensical
 749 #ifndef _LP64
 750     assert(which == imm_operand || which == disp32_operand,
 751            "which %d is_64_bit %d ip " INTPTR_FORMAT, which, is_64bit, p2i(ip));
 752 #else
 753     assert((which == call32_operand || which == imm_operand) && is_64bit ||
 754            which == narrow_oop_operand && !is_64bit,
 755            "which %d is_64_bit %d ip " INTPTR_FORMAT, which, is_64bit, p2i(ip));
 756 #endif // _LP64
 757     return ip;
 758 
 759   case 0x69: // imul r, a, #32
 760   case 0xC7: // movl a, #32(oop?)
 761     tail_size = 4;
 762     debug_only(has_disp32 = true); // has both kinds of operands!
 763     break;
 764 
 765   case 0x0F: // movx..., etc.
 766     switch (0xFF & *ip++) {
 767     case 0x3A: // pcmpestri
 768       tail_size = 1;
 769     case 0x38: // ptest, pmovzxbw
 770       ip++; // skip opcode
 771       debug_only(has_disp32 = true); // has both kinds of operands!
 772       break;
 773 
 774     case 0x70: // pshufd r, r/a, #8
 775       debug_only(has_disp32 = true); // has both kinds of operands!
 776     case 0x73: // psrldq r, #8
 777       tail_size = 1;
 778       break;
 779 
 780     case 0x12: // movlps
 781     case 0x28: // movaps
 782     case 0x2E: // ucomiss
 783     case 0x2F: // comiss
 784     case 0x54: // andps
 785     case 0x55: // andnps
 786     case 0x56: // orps
 787     case 0x57: // xorps
 788     case 0x58: // addpd
 789     case 0x59: // mulpd
 790     case 0x6E: // movd
 791     case 0x7E: // movd
 792     case 0xAE: // ldmxcsr, stmxcsr, fxrstor, fxsave, clflush
 793     case 0xFE: // paddd
 794       debug_only(has_disp32 = true);
 795       break;
 796 
 797     case 0xAD: // shrd r, a, %cl
 798     case 0xAF: // imul r, a
 799     case 0xBE: // movsbl r, a (movsxb)
 800     case 0xBF: // movswl r, a (movsxw)
 801     case 0xB6: // movzbl r, a (movzxb)
 802     case 0xB7: // movzwl r, a (movzxw)
 803     case REP16(0x40): // cmovl cc, r, a
 804     case 0xB0: // cmpxchgb
 805     case 0xB1: // cmpxchg
 806     case 0xC1: // xaddl
 807     case 0xC7: // cmpxchg8
 808     case REP16(0x90): // setcc a
 809       debug_only(has_disp32 = true);
 810       // fall out of the switch to decode the address
 811       break;
 812 
 813     case 0xC4: // pinsrw r, a, #8
 814       debug_only(has_disp32 = true);
 815     case 0xC5: // pextrw r, r, #8
 816       tail_size = 1;  // the imm8
 817       break;
 818 
 819     case 0xAC: // shrd r, a, #8
 820       debug_only(has_disp32 = true);
 821       tail_size = 1;  // the imm8
 822       break;
 823 
 824     case REP16(0x80): // jcc rdisp32
 825       if (which == end_pc_operand)  return ip + 4;
 826       assert(which == call32_operand, "jcc has no disp32 or imm");
 827       return ip;
 828     default:
 829       ShouldNotReachHere();
 830     }
 831     break;
 832 
 833   case 0x81: // addl a, #32; addl r, #32
 834     // also: orl, adcl, sbbl, andl, subl, xorl, cmpl
 835     // on 32bit in the case of cmpl, the imm might be an oop
 836     tail_size = 4;
 837     debug_only(has_disp32 = true); // has both kinds of operands!
 838     break;
 839 
 840   case 0x83: // addl a, #8; addl r, #8
 841     // also: orl, adcl, sbbl, andl, subl, xorl, cmpl
 842     debug_only(has_disp32 = true); // has both kinds of operands!
 843     tail_size = 1;
 844     break;
 845 
 846   case 0x9B:
 847     switch (0xFF & *ip++) {
 848     case 0xD9: // fnstcw a
 849       debug_only(has_disp32 = true);
 850       break;
 851     default:
 852       ShouldNotReachHere();
 853     }
 854     break;
 855 
 856   case REP4(0x00): // addb a, r; addl a, r; addb r, a; addl r, a
 857   case REP4(0x10): // adc...
 858   case REP4(0x20): // and...
 859   case REP4(0x30): // xor...
 860   case REP4(0x08): // or...
 861   case REP4(0x18): // sbb...
 862   case REP4(0x28): // sub...
 863   case 0xF7: // mull a
 864   case 0x8D: // lea r, a
 865   case 0x87: // xchg r, a
 866   case REP4(0x38): // cmp...
 867   case 0x85: // test r, a
 868     debug_only(has_disp32 = true); // has both kinds of operands!
 869     break;
 870 
 871   case 0xC1: // sal a, #8; sar a, #8; shl a, #8; shr a, #8
 872   case 0xC6: // movb a, #8
 873   case 0x80: // cmpb a, #8
 874   case 0x6B: // imul r, a, #8
 875     debug_only(has_disp32 = true); // has both kinds of operands!
 876     tail_size = 1; // the imm8
 877     break;
 878 
 879   case 0xC4: // VEX_3bytes
 880   case 0xC5: // VEX_2bytes
 881     assert((UseAVX > 0), "shouldn't have VEX prefix");
 882     assert(ip == inst+1, "no prefixes allowed");
 883     // C4 and C5 are also used as opcodes for PINSRW and PEXTRW instructions
 884     // but they have prefix 0x0F and processed when 0x0F processed above.
 885     //
 886     // In 32-bit mode the VEX first byte C4 and C5 alias onto LDS and LES
 887     // instructions (these instructions are not supported in 64-bit mode).
 888     // To distinguish them bits [7:6] are set in the VEX second byte since
 889     // ModRM byte can not be of the form 11xxxxxx in 32-bit mode. To set
 890     // those VEX bits REX and vvvv bits are inverted.
 891     //
 892     // Fortunately C2 doesn't generate these instructions so we don't need
 893     // to check for them in product version.
 894 
 895     // Check second byte
 896     NOT_LP64(assert((0xC0 & *ip) == 0xC0, "shouldn't have LDS and LES instructions"));
 897 
 898     int vex_opcode;
 899     // First byte
 900     if ((0xFF & *inst) == VEX_3bytes) {
 901       vex_opcode = VEX_OPCODE_MASK & *ip;
 902       ip++; // third byte
 903       is_64bit = ((VEX_W & *ip) == VEX_W);
 904     } else {
 905       vex_opcode = VEX_OPCODE_0F;
 906     }
 907     ip++; // opcode
 908     // To find the end of instruction (which == end_pc_operand).
 909     switch (vex_opcode) {
 910       case VEX_OPCODE_0F:
 911         switch (0xFF & *ip) {
 912         case 0x70: // pshufd r, r/a, #8
 913         case 0x71: // ps[rl|ra|ll]w r, #8
 914         case 0x72: // ps[rl|ra|ll]d r, #8
 915         case 0x73: // ps[rl|ra|ll]q r, #8
 916         case 0xC2: // cmp[ps|pd|ss|sd] r, r, r/a, #8
 917         case 0xC4: // pinsrw r, r, r/a, #8
 918         case 0xC5: // pextrw r/a, r, #8
 919         case 0xC6: // shufp[s|d] r, r, r/a, #8
 920           tail_size = 1;  // the imm8
 921           break;
 922         }
 923         break;
 924       case VEX_OPCODE_0F_3A:
 925         tail_size = 1;
 926         break;
 927     }
 928     ip++; // skip opcode
 929     debug_only(has_disp32 = true); // has both kinds of operands!
 930     break;
 931 
 932   case 0x62: // EVEX_4bytes
 933     assert(VM_Version::supports_evex(), "shouldn't have EVEX prefix");
 934     assert(ip == inst+1, "no prefixes allowed");
 935     // no EVEX collisions, all instructions that have 0x62 opcodes
 936     // have EVEX versions and are subopcodes of 0x66
 937     ip++; // skip P0 and exmaine W in P1
 938     is_64bit = ((VEX_W & *ip) == VEX_W);
 939     ip++; // move to P2
 940     ip++; // skip P2, move to opcode
 941     // To find the end of instruction (which == end_pc_operand).
 942     switch (0xFF & *ip) {
 943     case 0x22: // pinsrd r, r/a, #8
 944     case 0x61: // pcmpestri r, r/a, #8
 945     case 0x70: // pshufd r, r/a, #8
 946     case 0x73: // psrldq r, #8
 947       tail_size = 1;  // the imm8
 948       break;
 949     default:
 950       break;
 951     }
 952     ip++; // skip opcode
 953     debug_only(has_disp32 = true); // has both kinds of operands!
 954     break;
 955 
 956   case 0xD1: // sal a, 1; sar a, 1; shl a, 1; shr a, 1
 957   case 0xD3: // sal a, %cl; sar a, %cl; shl a, %cl; shr a, %cl
 958   case 0xD9: // fld_s a; fst_s a; fstp_s a; fldcw a
 959   case 0xDD: // fld_d a; fst_d a; fstp_d a
 960   case 0xDB: // fild_s a; fistp_s a; fld_x a; fstp_x a
 961   case 0xDF: // fild_d a; fistp_d a
 962   case 0xD8: // fadd_s a; fsubr_s a; fmul_s a; fdivr_s a; fcomp_s a
 963   case 0xDC: // fadd_d a; fsubr_d a; fmul_d a; fdivr_d a; fcomp_d a
 964   case 0xDE: // faddp_d a; fsubrp_d a; fmulp_d a; fdivrp_d a; fcompp_d a
 965     debug_only(has_disp32 = true);
 966     break;
 967 
 968   case 0xE8: // call rdisp32
 969   case 0xE9: // jmp  rdisp32
 970     if (which == end_pc_operand)  return ip + 4;
 971     assert(which == call32_operand, "call has no disp32 or imm");
 972     return ip;
 973 
 974   case 0xF0:                    // Lock
 975     assert(os::is_MP(), "only on MP");
 976     goto again_after_prefix;
 977 
 978   case 0xF3:                    // For SSE
 979   case 0xF2:                    // For SSE2
 980     switch (0xFF & *ip++) {
 981     case REX:
 982     case REX_B:
 983     case REX_X:
 984     case REX_XB:
 985     case REX_R:
 986     case REX_RB:
 987     case REX_RX:
 988     case REX_RXB:
 989     case REX_W:
 990     case REX_WB:
 991     case REX_WX:
 992     case REX_WXB:
 993     case REX_WR:
 994     case REX_WRB:
 995     case REX_WRX:
 996     case REX_WRXB:
 997       NOT_LP64(assert(false, "found 64bit prefix"));
 998       ip++;
 999     default:
1000       ip++;
1001     }
1002     debug_only(has_disp32 = true); // has both kinds of operands!
1003     break;
1004 
1005   default:
1006     ShouldNotReachHere();
1007 
1008 #undef REP8
1009 #undef REP16
1010   }
1011 
1012   assert(which != call32_operand, "instruction is not a call, jmp, or jcc");
1013 #ifdef _LP64
1014   assert(which != imm_operand, "instruction is not a movq reg, imm64");
1015 #else
1016   // assert(which != imm_operand || has_imm32, "instruction has no imm32 field");
1017   assert(which != imm_operand || has_disp32, "instruction has no imm32 field");
1018 #endif // LP64
1019   assert(which != disp32_operand || has_disp32, "instruction has no disp32 field");
1020 
1021   // parse the output of emit_operand
1022   int op2 = 0xFF & *ip++;
1023   int base = op2 & 0x07;
1024   int op3 = -1;
1025   const int b100 = 4;
1026   const int b101 = 5;
1027   if (base == b100 && (op2 >> 6) != 3) {
1028     op3 = 0xFF & *ip++;
1029     base = op3 & 0x07;   // refetch the base
1030   }
1031   // now ip points at the disp (if any)
1032 
1033   switch (op2 >> 6) {
1034   case 0:
1035     // [00 reg  100][ss index base]
1036     // [00 reg  100][00   100  esp]
1037     // [00 reg base]
1038     // [00 reg  100][ss index  101][disp32]
1039     // [00 reg  101]               [disp32]
1040 
1041     if (base == b101) {
1042       if (which == disp32_operand)
1043         return ip;              // caller wants the disp32
1044       ip += 4;                  // skip the disp32
1045     }
1046     break;
1047 
1048   case 1:
1049     // [01 reg  100][ss index base][disp8]
1050     // [01 reg  100][00   100  esp][disp8]
1051     // [01 reg base]               [disp8]
1052     ip += 1;                    // skip the disp8
1053     break;
1054 
1055   case 2:
1056     // [10 reg  100][ss index base][disp32]
1057     // [10 reg  100][00   100  esp][disp32]
1058     // [10 reg base]               [disp32]
1059     if (which == disp32_operand)
1060       return ip;                // caller wants the disp32
1061     ip += 4;                    // skip the disp32
1062     break;
1063 
1064   case 3:
1065     // [11 reg base]  (not a memory addressing mode)
1066     break;
1067   }
1068 
1069   if (which == end_pc_operand) {
1070     return ip + tail_size;
1071   }
1072 
1073 #ifdef _LP64
1074   assert(which == narrow_oop_operand && !is_64bit, "instruction is not a movl adr, imm32");
1075 #else
1076   assert(which == imm_operand, "instruction has only an imm field");
1077 #endif // LP64
1078   return ip;
1079 }
1080 
1081 address Assembler::locate_next_instruction(address inst) {
1082   // Secretly share code with locate_operand:
1083   return locate_operand(inst, end_pc_operand);
1084 }
1085 
1086 
1087 #ifdef ASSERT
1088 void Assembler::check_relocation(RelocationHolder const& rspec, int format) {
1089   address inst = inst_mark();
1090   assert(inst != NULL && inst < pc(), "must point to beginning of instruction");
1091   address opnd;
1092 
1093   Relocation* r = rspec.reloc();
1094   if (r->type() == relocInfo::none) {
1095     return;
1096   } else if (r->is_call() || format == call32_operand) {
1097     // assert(format == imm32_operand, "cannot specify a nonzero format");
1098     opnd = locate_operand(inst, call32_operand);
1099   } else if (r->is_data()) {
1100     assert(format == imm_operand || format == disp32_operand
1101            LP64_ONLY(|| format == narrow_oop_operand), "format ok");
1102     opnd = locate_operand(inst, (WhichOperand)format);
1103   } else {
1104     assert(format == imm_operand, "cannot specify a format");
1105     return;
1106   }
1107   assert(opnd == pc(), "must put operand where relocs can find it");
1108 }
1109 #endif // ASSERT
1110 
1111 void Assembler::emit_operand32(Register reg, Address adr) {
1112   assert(reg->encoding() < 8, "no extended registers");
1113   assert(!adr.base_needs_rex() && !adr.index_needs_rex(), "no extended registers");
1114   emit_operand(reg, adr._base, adr._index, adr._scale, adr._disp,
1115                adr._rspec);
1116 }
1117 
1118 void Assembler::emit_operand(Register reg, Address adr,
1119                              int rip_relative_correction) {
1120   emit_operand(reg, adr._base, adr._index, adr._scale, adr._disp,
1121                adr._rspec,
1122                rip_relative_correction);
1123 }
1124 
1125 void Assembler::emit_operand(XMMRegister reg, Address adr) {
1126     if (adr.isxmmindex()) {
1127        emit_operand(reg, adr._base, adr._xmmindex, adr._scale, adr._disp, adr._rspec);
1128     } else {
1129        emit_operand(reg, adr._base, adr._index, adr._scale, adr._disp,
1130        adr._rspec);
1131     }
1132 }
1133 
1134 // MMX operations
1135 void Assembler::emit_operand(MMXRegister reg, Address adr) {
1136   assert(!adr.base_needs_rex() && !adr.index_needs_rex(), "no extended registers");
1137   emit_operand((Register)reg, adr._base, adr._index, adr._scale, adr._disp, adr._rspec);
1138 }
1139 
1140 // work around gcc (3.2.1-7a) bug
1141 void Assembler::emit_operand(Address adr, MMXRegister reg) {
1142   assert(!adr.base_needs_rex() && !adr.index_needs_rex(), "no extended registers");
1143   emit_operand((Register)reg, adr._base, adr._index, adr._scale, adr._disp, adr._rspec);
1144 }
1145 
1146 
1147 void Assembler::emit_farith(int b1, int b2, int i) {
1148   assert(isByte(b1) && isByte(b2), "wrong opcode");
1149   assert(0 <= i &&  i < 8, "illegal stack offset");
1150   emit_int8(b1);
1151   emit_int8(b2 + i);
1152 }
1153 
1154 
1155 // Now the Assembler instructions (identical for 32/64 bits)
1156 
1157 void Assembler::adcl(Address dst, int32_t imm32) {
1158   InstructionMark im(this);
1159   prefix(dst);
1160   emit_arith_operand(0x81, rdx, dst, imm32);
1161 }
1162 
1163 void Assembler::adcl(Address dst, Register src) {
1164   InstructionMark im(this);
1165   prefix(dst, src);
1166   emit_int8(0x11);
1167   emit_operand(src, dst);
1168 }
1169 
1170 void Assembler::adcl(Register dst, int32_t imm32) {
1171   prefix(dst);
1172   emit_arith(0x81, 0xD0, dst, imm32);
1173 }
1174 
1175 void Assembler::adcl(Register dst, Address src) {
1176   InstructionMark im(this);
1177   prefix(src, dst);
1178   emit_int8(0x13);
1179   emit_operand(dst, src);
1180 }
1181 
1182 void Assembler::adcl(Register dst, Register src) {
1183   (void) prefix_and_encode(dst->encoding(), src->encoding());
1184   emit_arith(0x13, 0xC0, dst, src);
1185 }
1186 
1187 void Assembler::addl(Address dst, int32_t imm32) {
1188   InstructionMark im(this);
1189   prefix(dst);
1190   emit_arith_operand(0x81, rax, dst, imm32);
1191 }
1192 
1193 void Assembler::addb(Address dst, int imm8) {
1194   InstructionMark im(this);
1195   prefix(dst);
1196   emit_int8((unsigned char)0x80);
1197   emit_operand(rax, dst, 1);
1198   emit_int8(imm8);
1199 }
1200 
1201 void Assembler::addw(Address dst, int imm16) {
1202   InstructionMark im(this);
1203   emit_int8(0x66);
1204   prefix(dst);
1205   emit_int8((unsigned char)0x81);
1206   emit_operand(rax, dst, 2);
1207   emit_int16(imm16);
1208 }
1209 
1210 void Assembler::addl(Address dst, Register src) {
1211   InstructionMark im(this);
1212   prefix(dst, src);
1213   emit_int8(0x01);
1214   emit_operand(src, dst);
1215 }
1216 
1217 void Assembler::addl(Register dst, int32_t imm32) {
1218   prefix(dst);
1219   emit_arith(0x81, 0xC0, dst, imm32);
1220 }
1221 
1222 void Assembler::addl(Register dst, Address src) {
1223   InstructionMark im(this);
1224   prefix(src, dst);
1225   emit_int8(0x03);
1226   emit_operand(dst, src);
1227 }
1228 
1229 void Assembler::addl(Register dst, Register src) {
1230   (void) prefix_and_encode(dst->encoding(), src->encoding());
1231   emit_arith(0x03, 0xC0, dst, src);
1232 }
1233 
1234 void Assembler::addr_nop_4() {
1235   assert(UseAddressNop, "no CPU support");
1236   // 4 bytes: NOP DWORD PTR [EAX+0]
1237   emit_int8(0x0F);
1238   emit_int8(0x1F);
1239   emit_int8(0x40); // emit_rm(cbuf, 0x1, EAX_enc, EAX_enc);
1240   emit_int8(0);    // 8-bits offset (1 byte)
1241 }
1242 
1243 void Assembler::addr_nop_5() {
1244   assert(UseAddressNop, "no CPU support");
1245   // 5 bytes: NOP DWORD PTR [EAX+EAX*0+0] 8-bits offset
1246   emit_int8(0x0F);
1247   emit_int8(0x1F);
1248   emit_int8(0x44); // emit_rm(cbuf, 0x1, EAX_enc, 0x4);
1249   emit_int8(0x00); // emit_rm(cbuf, 0x0, EAX_enc, EAX_enc);
1250   emit_int8(0);    // 8-bits offset (1 byte)
1251 }
1252 
1253 void Assembler::addr_nop_7() {
1254   assert(UseAddressNop, "no CPU support");
1255   // 7 bytes: NOP DWORD PTR [EAX+0] 32-bits offset
1256   emit_int8(0x0F);
1257   emit_int8(0x1F);
1258   emit_int8((unsigned char)0x80);
1259                    // emit_rm(cbuf, 0x2, EAX_enc, EAX_enc);
1260   emit_int32(0);   // 32-bits offset (4 bytes)
1261 }
1262 
1263 void Assembler::addr_nop_8() {
1264   assert(UseAddressNop, "no CPU support");
1265   // 8 bytes: NOP DWORD PTR [EAX+EAX*0+0] 32-bits offset
1266   emit_int8(0x0F);
1267   emit_int8(0x1F);
1268   emit_int8((unsigned char)0x84);
1269                    // emit_rm(cbuf, 0x2, EAX_enc, 0x4);
1270   emit_int8(0x00); // emit_rm(cbuf, 0x0, EAX_enc, EAX_enc);
1271   emit_int32(0);   // 32-bits offset (4 bytes)
1272 }
1273 
1274 void Assembler::addsd(XMMRegister dst, XMMRegister src) {
1275   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1276   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1277   attributes.set_rex_vex_w_reverted();
1278   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
1279   emit_int8(0x58);
1280   emit_int8((unsigned char)(0xC0 | encode));
1281 }
1282 
1283 void Assembler::addsd(XMMRegister dst, Address src) {
1284   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1285   InstructionMark im(this);
1286   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1287   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
1288   attributes.set_rex_vex_w_reverted();
1289   simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
1290   emit_int8(0x58);
1291   emit_operand(dst, src);
1292 }
1293 
1294 void Assembler::addss(XMMRegister dst, XMMRegister src) {
1295   NOT_LP64(assert(VM_Version::supports_sse(), ""));
1296   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1297   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
1298   emit_int8(0x58);
1299   emit_int8((unsigned char)(0xC0 | encode));
1300 }
1301 
1302 void Assembler::addss(XMMRegister dst, Address src) {
1303   NOT_LP64(assert(VM_Version::supports_sse(), ""));
1304   InstructionMark im(this);
1305   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1306   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
1307   simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
1308   emit_int8(0x58);
1309   emit_operand(dst, src);
1310 }
1311 
1312 void Assembler::aesdec(XMMRegister dst, Address src) {
1313   assert(VM_Version::supports_aes(), "");
1314   InstructionMark im(this);
1315   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
1316   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
1317   emit_int8((unsigned char)0xDE);
1318   emit_operand(dst, src);
1319 }
1320 
1321 void Assembler::aesdec(XMMRegister dst, XMMRegister src) {
1322   assert(VM_Version::supports_aes(), "");
1323   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
1324   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
1325   emit_int8((unsigned char)0xDE);
1326   emit_int8(0xC0 | encode);
1327 }
1328 
1329 void Assembler::vaesdec(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
1330   assert(VM_Version::supports_vaes(), "");
1331   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
1332   attributes.set_is_evex_instruction();
1333   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
1334   emit_int8((unsigned char)0xDE);
1335   emit_int8((unsigned char)(0xC0 | encode));
1336 }
1337 
1338 
1339 void Assembler::aesdeclast(XMMRegister dst, Address src) {
1340   assert(VM_Version::supports_aes(), "");
1341   InstructionMark im(this);
1342   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
1343   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
1344   emit_int8((unsigned char)0xDF);
1345   emit_operand(dst, src);
1346 }
1347 
1348 void Assembler::aesdeclast(XMMRegister dst, XMMRegister src) {
1349   assert(VM_Version::supports_aes(), "");
1350   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
1351   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
1352   emit_int8((unsigned char)0xDF);
1353   emit_int8((unsigned char)(0xC0 | encode));
1354 }
1355 
1356 void Assembler::vaesdeclast(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
1357   assert(VM_Version::supports_vaes(), "");
1358   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
1359   attributes.set_is_evex_instruction();
1360   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
1361   emit_int8((unsigned char)0xDF);
1362   emit_int8((unsigned char)(0xC0 | encode));
1363 }
1364 
1365 void Assembler::aesenc(XMMRegister dst, Address src) {
1366   assert(VM_Version::supports_aes(), "");
1367   InstructionMark im(this);
1368   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
1369   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
1370   emit_int8((unsigned char)0xDC);
1371   emit_operand(dst, src);
1372 }
1373 
1374 void Assembler::aesenc(XMMRegister dst, XMMRegister src) {
1375   assert(VM_Version::supports_aes(), "");
1376   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
1377   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
1378   emit_int8((unsigned char)0xDC);
1379   emit_int8(0xC0 | encode);
1380 }
1381 
1382 void Assembler::aesenclast(XMMRegister dst, Address src) {
1383   assert(VM_Version::supports_aes(), "");
1384   InstructionMark im(this);
1385   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
1386   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
1387   emit_int8((unsigned char)0xDD);
1388   emit_operand(dst, src);
1389 }
1390 
1391 void Assembler::aesenclast(XMMRegister dst, XMMRegister src) {
1392   assert(VM_Version::supports_aes(), "");
1393   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
1394   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
1395   emit_int8((unsigned char)0xDD);
1396   emit_int8((unsigned char)(0xC0 | encode));
1397 }
1398 
1399 void Assembler::andl(Address dst, int32_t imm32) {
1400   InstructionMark im(this);
1401   prefix(dst);
1402   emit_int8((unsigned char)0x81);
1403   emit_operand(rsp, dst, 4);
1404   emit_int32(imm32);
1405 }
1406 
1407 void Assembler::andl(Register dst, int32_t imm32) {
1408   prefix(dst);
1409   emit_arith(0x81, 0xE0, dst, imm32);
1410 }
1411 
1412 void Assembler::andl(Register dst, Address src) {
1413   InstructionMark im(this);
1414   prefix(src, dst);
1415   emit_int8(0x23);
1416   emit_operand(dst, src);
1417 }
1418 
1419 void Assembler::andl(Register dst, Register src) {
1420   (void) prefix_and_encode(dst->encoding(), src->encoding());
1421   emit_arith(0x23, 0xC0, dst, src);
1422 }
1423 
1424 void Assembler::andnl(Register dst, Register src1, Register src2) {
1425   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1426   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
1427   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
1428   emit_int8((unsigned char)0xF2);
1429   emit_int8((unsigned char)(0xC0 | encode));
1430 }
1431 
1432 void Assembler::andnl(Register dst, Register src1, Address src2) {
1433   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1434   InstructionMark im(this);
1435   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
1436   vex_prefix(src2, src1->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
1437   emit_int8((unsigned char)0xF2);
1438   emit_operand(dst, src2);
1439 }
1440 
1441 void Assembler::bsfl(Register dst, Register src) {
1442   int encode = prefix_and_encode(dst->encoding(), src->encoding());
1443   emit_int8(0x0F);
1444   emit_int8((unsigned char)0xBC);
1445   emit_int8((unsigned char)(0xC0 | encode));
1446 }
1447 
1448 void Assembler::bsrl(Register dst, Register src) {
1449   int encode = prefix_and_encode(dst->encoding(), src->encoding());
1450   emit_int8(0x0F);
1451   emit_int8((unsigned char)0xBD);
1452   emit_int8((unsigned char)(0xC0 | encode));
1453 }
1454 
1455 void Assembler::bswapl(Register reg) { // bswap
1456   int encode = prefix_and_encode(reg->encoding());
1457   emit_int8(0x0F);
1458   emit_int8((unsigned char)(0xC8 | encode));
1459 }
1460 
1461 void Assembler::blsil(Register dst, Register src) {
1462   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1463   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
1464   int encode = vex_prefix_and_encode(rbx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
1465   emit_int8((unsigned char)0xF3);
1466   emit_int8((unsigned char)(0xC0 | encode));
1467 }
1468 
1469 void Assembler::blsil(Register dst, Address src) {
1470   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1471   InstructionMark im(this);
1472   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
1473   vex_prefix(src, dst->encoding(), rbx->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
1474   emit_int8((unsigned char)0xF3);
1475   emit_operand(rbx, src);
1476 }
1477 
1478 void Assembler::blsmskl(Register dst, Register src) {
1479   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1480   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
1481   int encode = vex_prefix_and_encode(rdx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
1482   emit_int8((unsigned char)0xF3);
1483   emit_int8((unsigned char)(0xC0 | encode));
1484 }
1485 
1486 void Assembler::blsmskl(Register dst, Address src) {
1487   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1488   InstructionMark im(this);
1489   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
1490   vex_prefix(src, dst->encoding(), rdx->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
1491   emit_int8((unsigned char)0xF3);
1492   emit_operand(rdx, src);
1493 }
1494 
1495 void Assembler::blsrl(Register dst, Register src) {
1496   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1497   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
1498   int encode = vex_prefix_and_encode(rcx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
1499   emit_int8((unsigned char)0xF3);
1500   emit_int8((unsigned char)(0xC0 | encode));
1501 }
1502 
1503 void Assembler::blsrl(Register dst, Address src) {
1504   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1505   InstructionMark im(this);
1506   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
1507   vex_prefix(src, dst->encoding(), rcx->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
1508   emit_int8((unsigned char)0xF3);
1509   emit_operand(rcx, src);
1510 }
1511 
1512 void Assembler::call(Label& L, relocInfo::relocType rtype) {
1513   // suspect disp32 is always good
1514   int operand = LP64_ONLY(disp32_operand) NOT_LP64(imm_operand);
1515 
1516   if (L.is_bound()) {
1517     const int long_size = 5;
1518     int offs = (int)( target(L) - pc() );
1519     assert(offs <= 0, "assembler error");
1520     InstructionMark im(this);
1521     // 1110 1000 #32-bit disp
1522     emit_int8((unsigned char)0xE8);
1523     emit_data(offs - long_size, rtype, operand);
1524   } else {
1525     InstructionMark im(this);
1526     // 1110 1000 #32-bit disp
1527     L.add_patch_at(code(), locator());
1528 
1529     emit_int8((unsigned char)0xE8);
1530     emit_data(int(0), rtype, operand);
1531   }
1532 }
1533 
1534 void Assembler::call(Register dst) {
1535   int encode = prefix_and_encode(dst->encoding());
1536   emit_int8((unsigned char)0xFF);
1537   emit_int8((unsigned char)(0xD0 | encode));
1538 }
1539 
1540 
1541 void Assembler::call(Address adr) {
1542   InstructionMark im(this);
1543   prefix(adr);
1544   emit_int8((unsigned char)0xFF);
1545   emit_operand(rdx, adr);
1546 }
1547 
1548 void Assembler::call_literal(address entry, RelocationHolder const& rspec) {
1549   InstructionMark im(this);
1550   emit_int8((unsigned char)0xE8);
1551   intptr_t disp = entry - (pc() + sizeof(int32_t));
1552   // Entry is NULL in case of a scratch emit.
1553   assert(entry == NULL || is_simm32(disp), "disp=" INTPTR_FORMAT " must be 32bit offset (call2)", disp);
1554   // Technically, should use call32_operand, but this format is
1555   // implied by the fact that we're emitting a call instruction.
1556 
1557   int operand = LP64_ONLY(disp32_operand) NOT_LP64(call32_operand);
1558   emit_data((int) disp, rspec, operand);
1559 }
1560 
1561 void Assembler::cdql() {
1562   emit_int8((unsigned char)0x99);
1563 }
1564 
1565 void Assembler::cld() {
1566   emit_int8((unsigned char)0xFC);
1567 }
1568 
1569 void Assembler::cmovl(Condition cc, Register dst, Register src) {
1570   NOT_LP64(guarantee(VM_Version::supports_cmov(), "illegal instruction"));
1571   int encode = prefix_and_encode(dst->encoding(), src->encoding());
1572   emit_int8(0x0F);
1573   emit_int8(0x40 | cc);
1574   emit_int8((unsigned char)(0xC0 | encode));
1575 }
1576 
1577 
1578 void Assembler::cmovl(Condition cc, Register dst, Address src) {
1579   NOT_LP64(guarantee(VM_Version::supports_cmov(), "illegal instruction"));
1580   prefix(src, dst);
1581   emit_int8(0x0F);
1582   emit_int8(0x40 | cc);
1583   emit_operand(dst, src);
1584 }
1585 
1586 void Assembler::cmpb(Address dst, int imm8) {
1587   InstructionMark im(this);
1588   prefix(dst);
1589   emit_int8((unsigned char)0x80);
1590   emit_operand(rdi, dst, 1);
1591   emit_int8(imm8);
1592 }
1593 
1594 void Assembler::cmpl(Address dst, int32_t imm32) {
1595   InstructionMark im(this);
1596   prefix(dst);
1597   emit_int8((unsigned char)0x81);
1598   emit_operand(rdi, dst, 4);
1599   emit_int32(imm32);
1600 }
1601 
1602 void Assembler::cmpl(Register dst, int32_t imm32) {
1603   prefix(dst);
1604   emit_arith(0x81, 0xF8, dst, imm32);
1605 }
1606 
1607 void Assembler::cmpl(Register dst, Register src) {
1608   (void) prefix_and_encode(dst->encoding(), src->encoding());
1609   emit_arith(0x3B, 0xC0, dst, src);
1610 }
1611 
1612 void Assembler::cmpl(Register dst, Address  src) {
1613   InstructionMark im(this);
1614   prefix(src, dst);
1615   emit_int8((unsigned char)0x3B);
1616   emit_operand(dst, src);
1617 }
1618 
1619 void Assembler::cmpw(Address dst, int imm16) {
1620   InstructionMark im(this);
1621   assert(!dst.base_needs_rex() && !dst.index_needs_rex(), "no extended registers");
1622   emit_int8(0x66);
1623   emit_int8((unsigned char)0x81);
1624   emit_operand(rdi, dst, 2);
1625   emit_int16(imm16);
1626 }
1627 
1628 // The 32-bit cmpxchg compares the value at adr with the contents of rax,
1629 // and stores reg into adr if so; otherwise, the value at adr is loaded into rax,.
1630 // The ZF is set if the compared values were equal, and cleared otherwise.
1631 void Assembler::cmpxchgl(Register reg, Address adr) { // cmpxchg
1632   InstructionMark im(this);
1633   prefix(adr, reg);
1634   emit_int8(0x0F);
1635   emit_int8((unsigned char)0xB1);
1636   emit_operand(reg, adr);
1637 }
1638 
1639 // The 8-bit cmpxchg compares the value at adr with the contents of rax,
1640 // and stores reg into adr if so; otherwise, the value at adr is loaded into rax,.
1641 // The ZF is set if the compared values were equal, and cleared otherwise.
1642 void Assembler::cmpxchgb(Register reg, Address adr) { // cmpxchg
1643   InstructionMark im(this);
1644   prefix(adr, reg, true);
1645   emit_int8(0x0F);
1646   emit_int8((unsigned char)0xB0);
1647   emit_operand(reg, adr);
1648 }
1649 
1650 void Assembler::comisd(XMMRegister dst, Address src) {
1651   // NOTE: dbx seems to decode this as comiss even though the
1652   // 0x66 is there. Strangly ucomisd comes out correct
1653   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1654   InstructionMark im(this);
1655   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);;
1656   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
1657   attributes.set_rex_vex_w_reverted();
1658   simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
1659   emit_int8(0x2F);
1660   emit_operand(dst, src);
1661 }
1662 
1663 void Assembler::comisd(XMMRegister dst, XMMRegister src) {
1664   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1665   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1666   attributes.set_rex_vex_w_reverted();
1667   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
1668   emit_int8(0x2F);
1669   emit_int8((unsigned char)(0xC0 | encode));
1670 }
1671 
1672 void Assembler::comiss(XMMRegister dst, Address src) {
1673   NOT_LP64(assert(VM_Version::supports_sse(), ""));
1674   InstructionMark im(this);
1675   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1676   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
1677   simd_prefix(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
1678   emit_int8(0x2F);
1679   emit_operand(dst, src);
1680 }
1681 
1682 void Assembler::comiss(XMMRegister dst, XMMRegister src) {
1683   NOT_LP64(assert(VM_Version::supports_sse(), ""));
1684   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1685   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
1686   emit_int8(0x2F);
1687   emit_int8((unsigned char)(0xC0 | encode));
1688 }
1689 
1690 void Assembler::cpuid() {
1691   emit_int8(0x0F);
1692   emit_int8((unsigned char)0xA2);
1693 }
1694 
1695 // Opcode / Instruction                      Op /  En  64 - Bit Mode     Compat / Leg Mode Description                  Implemented
1696 // F2 0F 38 F0 / r       CRC32 r32, r / m8   RM        Valid             Valid             Accumulate CRC32 on r / m8.  v
1697 // F2 REX 0F 38 F0 / r   CRC32 r32, r / m8*  RM        Valid             N.E.              Accumulate CRC32 on r / m8.  -
1698 // F2 REX.W 0F 38 F0 / r CRC32 r64, r / m8   RM        Valid             N.E.              Accumulate CRC32 on r / m8.  -
1699 //
1700 // F2 0F 38 F1 / r       CRC32 r32, r / m16  RM        Valid             Valid             Accumulate CRC32 on r / m16. v
1701 //
1702 // F2 0F 38 F1 / r       CRC32 r32, r / m32  RM        Valid             Valid             Accumulate CRC32 on r / m32. v
1703 //
1704 // F2 REX.W 0F 38 F1 / r CRC32 r64, r / m64  RM        Valid             N.E.              Accumulate CRC32 on r / m64. v
1705 void Assembler::crc32(Register crc, Register v, int8_t sizeInBytes) {
1706   assert(VM_Version::supports_sse4_2(), "");
1707   int8_t w = 0x01;
1708   Prefix p = Prefix_EMPTY;
1709 
1710   emit_int8((int8_t)0xF2);
1711   switch (sizeInBytes) {
1712   case 1:
1713     w = 0;
1714     break;
1715   case 2:
1716   case 4:
1717     break;
1718   LP64_ONLY(case 8:)
1719     // This instruction is not valid in 32 bits
1720     // Note:
1721     // http://www.intel.com/content/dam/www/public/us/en/documents/manuals/64-ia-32-architectures-software-developer-instruction-set-reference-manual-325383.pdf
1722     //
1723     // Page B - 72   Vol. 2C says
1724     // qwreg2 to qwreg            1111 0010 : 0100 1R0B : 0000 1111 : 0011 1000 : 1111 0000 : 11 qwreg1 qwreg2
1725     // mem64 to qwreg             1111 0010 : 0100 1R0B : 0000 1111 : 0011 1000 : 1111 0000 : mod qwreg r / m
1726     //                                                                            F0!!!
1727     // while 3 - 208 Vol. 2A
1728     // F2 REX.W 0F 38 F1 / r       CRC32 r64, r / m64             RM         Valid      N.E.Accumulate CRC32 on r / m64.
1729     //
1730     // the 0 on a last bit is reserved for a different flavor of this instruction :
1731     // F2 REX.W 0F 38 F0 / r       CRC32 r64, r / m8              RM         Valid      N.E.Accumulate CRC32 on r / m8.
1732     p = REX_W;
1733     break;
1734   default:
1735     assert(0, "Unsupported value for a sizeInBytes argument");
1736     break;
1737   }
1738   LP64_ONLY(prefix(crc, v, p);)
1739   emit_int8((int8_t)0x0F);
1740   emit_int8(0x38);
1741   emit_int8((int8_t)(0xF0 | w));
1742   emit_int8(0xC0 | ((crc->encoding() & 0x7) << 3) | (v->encoding() & 7));
1743 }
1744 
1745 void Assembler::crc32(Register crc, Address adr, int8_t sizeInBytes) {
1746   assert(VM_Version::supports_sse4_2(), "");
1747   InstructionMark im(this);
1748   int8_t w = 0x01;
1749   Prefix p = Prefix_EMPTY;
1750 
1751   emit_int8((int8_t)0xF2);
1752   switch (sizeInBytes) {
1753   case 1:
1754     w = 0;
1755     break;
1756   case 2:
1757   case 4:
1758     break;
1759   LP64_ONLY(case 8:)
1760     // This instruction is not valid in 32 bits
1761     p = REX_W;
1762     break;
1763   default:
1764     assert(0, "Unsupported value for a sizeInBytes argument");
1765     break;
1766   }
1767   LP64_ONLY(prefix(crc, adr, p);)
1768   emit_int8((int8_t)0x0F);
1769   emit_int8(0x38);
1770   emit_int8((int8_t)(0xF0 | w));
1771   emit_operand(crc, adr);
1772 }
1773 
1774 void Assembler::cvtdq2pd(XMMRegister dst, XMMRegister src) {
1775   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1776   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
1777   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
1778   emit_int8((unsigned char)0xE6);
1779   emit_int8((unsigned char)(0xC0 | encode));
1780 }
1781 
1782 void Assembler::cvtdq2ps(XMMRegister dst, XMMRegister src) {
1783   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1784   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
1785   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
1786   emit_int8(0x5B);
1787   emit_int8((unsigned char)(0xC0 | encode));
1788 }
1789 
1790 void Assembler::cvtsd2ss(XMMRegister dst, XMMRegister src) {
1791   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1792   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1793   attributes.set_rex_vex_w_reverted();
1794   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
1795   emit_int8(0x5A);
1796   emit_int8((unsigned char)(0xC0 | encode));
1797 }
1798 
1799 void Assembler::cvtsd2ss(XMMRegister dst, Address src) {
1800   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1801   InstructionMark im(this);
1802   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1803   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
1804   attributes.set_rex_vex_w_reverted();
1805   simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
1806   emit_int8(0x5A);
1807   emit_operand(dst, src);
1808 }
1809 
1810 void Assembler::cvtsi2sdl(XMMRegister dst, Register src) {
1811   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1812   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1813   int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
1814   emit_int8(0x2A);
1815   emit_int8((unsigned char)(0xC0 | encode));
1816 }
1817 
1818 void Assembler::cvtsi2sdl(XMMRegister dst, Address src) {
1819   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1820   InstructionMark im(this);
1821   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1822   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
1823   simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
1824   emit_int8(0x2A);
1825   emit_operand(dst, src);
1826 }
1827 
1828 void Assembler::cvtsi2ssl(XMMRegister dst, Register src) {
1829   NOT_LP64(assert(VM_Version::supports_sse(), ""));
1830   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1831   int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
1832   emit_int8(0x2A);
1833   emit_int8((unsigned char)(0xC0 | encode));
1834 }
1835 
1836 void Assembler::cvtsi2ssl(XMMRegister dst, Address src) {
1837   NOT_LP64(assert(VM_Version::supports_sse(), ""));
1838   InstructionMark im(this);
1839   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1840   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
1841   simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
1842   emit_int8(0x2A);
1843   emit_operand(dst, src);
1844 }
1845 
1846 void Assembler::cvtsi2ssq(XMMRegister dst, Register src) {
1847   NOT_LP64(assert(VM_Version::supports_sse(), ""));
1848   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1849   int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
1850   emit_int8(0x2A);
1851   emit_int8((unsigned char)(0xC0 | encode));
1852 }
1853 
1854 void Assembler::cvtss2sd(XMMRegister dst, XMMRegister src) {
1855   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1856   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1857   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
1858   emit_int8(0x5A);
1859   emit_int8((unsigned char)(0xC0 | encode));
1860 }
1861 
1862 void Assembler::cvtss2sd(XMMRegister dst, Address src) {
1863   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1864   InstructionMark im(this);
1865   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1866   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
1867   simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
1868   emit_int8(0x5A);
1869   emit_operand(dst, src);
1870 }
1871 
1872 
1873 void Assembler::cvttsd2sil(Register dst, XMMRegister src) {
1874   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1875   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1876   int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
1877   emit_int8(0x2C);
1878   emit_int8((unsigned char)(0xC0 | encode));
1879 }
1880 
1881 void Assembler::cvttss2sil(Register dst, XMMRegister src) {
1882   NOT_LP64(assert(VM_Version::supports_sse(), ""));
1883   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1884   int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
1885   emit_int8(0x2C);
1886   emit_int8((unsigned char)(0xC0 | encode));
1887 }
1888 
1889 void Assembler::cvttpd2dq(XMMRegister dst, XMMRegister src) {
1890   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1891   int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_128bit;
1892   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
1893   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
1894   emit_int8((unsigned char)0xE6);
1895   emit_int8((unsigned char)(0xC0 | encode));
1896 }
1897 
1898 void Assembler::decl(Address dst) {
1899   // Don't use it directly. Use MacroAssembler::decrement() instead.
1900   InstructionMark im(this);
1901   prefix(dst);
1902   emit_int8((unsigned char)0xFF);
1903   emit_operand(rcx, dst);
1904 }
1905 
1906 void Assembler::divsd(XMMRegister dst, Address src) {
1907   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1908   InstructionMark im(this);
1909   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1910   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
1911   attributes.set_rex_vex_w_reverted();
1912   simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
1913   emit_int8(0x5E);
1914   emit_operand(dst, src);
1915 }
1916 
1917 void Assembler::divsd(XMMRegister dst, XMMRegister src) {
1918   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1919   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1920   attributes.set_rex_vex_w_reverted();
1921   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
1922   emit_int8(0x5E);
1923   emit_int8((unsigned char)(0xC0 | encode));
1924 }
1925 
1926 void Assembler::divss(XMMRegister dst, Address src) {
1927   NOT_LP64(assert(VM_Version::supports_sse(), ""));
1928   InstructionMark im(this);
1929   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1930   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
1931   simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
1932   emit_int8(0x5E);
1933   emit_operand(dst, src);
1934 }
1935 
1936 void Assembler::divss(XMMRegister dst, XMMRegister src) {
1937   NOT_LP64(assert(VM_Version::supports_sse(), ""));
1938   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1939   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
1940   emit_int8(0x5E);
1941   emit_int8((unsigned char)(0xC0 | encode));
1942 }
1943 
1944 void Assembler::emms() {
1945   NOT_LP64(assert(VM_Version::supports_mmx(), ""));
1946   emit_int8(0x0F);
1947   emit_int8(0x77);
1948 }
1949 
1950 void Assembler::hlt() {
1951   emit_int8((unsigned char)0xF4);
1952 }
1953 
1954 void Assembler::idivl(Register src) {
1955   int encode = prefix_and_encode(src->encoding());
1956   emit_int8((unsigned char)0xF7);
1957   emit_int8((unsigned char)(0xF8 | encode));
1958 }
1959 
1960 void Assembler::divl(Register src) { // Unsigned
1961   int encode = prefix_and_encode(src->encoding());
1962   emit_int8((unsigned char)0xF7);
1963   emit_int8((unsigned char)(0xF0 | encode));
1964 }
1965 
1966 void Assembler::imull(Register src) {
1967   int encode = prefix_and_encode(src->encoding());
1968   emit_int8((unsigned char)0xF7);
1969   emit_int8((unsigned char)(0xE8 | encode));
1970 }
1971 
1972 void Assembler::imull(Register dst, Register src) {
1973   int encode = prefix_and_encode(dst->encoding(), src->encoding());
1974   emit_int8(0x0F);
1975   emit_int8((unsigned char)0xAF);
1976   emit_int8((unsigned char)(0xC0 | encode));
1977 }
1978 
1979 
1980 void Assembler::imull(Register dst, Register src, int value) {
1981   int encode = prefix_and_encode(dst->encoding(), src->encoding());
1982   if (is8bit(value)) {
1983     emit_int8(0x6B);
1984     emit_int8((unsigned char)(0xC0 | encode));
1985     emit_int8(value & 0xFF);
1986   } else {
1987     emit_int8(0x69);
1988     emit_int8((unsigned char)(0xC0 | encode));
1989     emit_int32(value);
1990   }
1991 }
1992 
1993 void Assembler::imull(Register dst, Address src) {
1994   InstructionMark im(this);
1995   prefix(src, dst);
1996   emit_int8(0x0F);
1997   emit_int8((unsigned char) 0xAF);
1998   emit_operand(dst, src);
1999 }
2000 
2001 
2002 void Assembler::incl(Address dst) {
2003   // Don't use it directly. Use MacroAssembler::increment() instead.
2004   InstructionMark im(this);
2005   prefix(dst);
2006   emit_int8((unsigned char)0xFF);
2007   emit_operand(rax, dst);
2008 }
2009 
2010 void Assembler::jcc(Condition cc, Label& L, bool maybe_short) {
2011   InstructionMark im(this);
2012   assert((0 <= cc) && (cc < 16), "illegal cc");
2013   if (L.is_bound()) {
2014     address dst = target(L);
2015     assert(dst != NULL, "jcc most probably wrong");
2016 
2017     const int short_size = 2;
2018     const int long_size = 6;
2019     intptr_t offs = (intptr_t)dst - (intptr_t)pc();
2020     if (maybe_short && is8bit(offs - short_size)) {
2021       // 0111 tttn #8-bit disp
2022       emit_int8(0x70 | cc);
2023       emit_int8((offs - short_size) & 0xFF);
2024     } else {
2025       // 0000 1111 1000 tttn #32-bit disp
2026       assert(is_simm32(offs - long_size),
2027              "must be 32bit offset (call4)");
2028       emit_int8(0x0F);
2029       emit_int8((unsigned char)(0x80 | cc));
2030       emit_int32(offs - long_size);
2031     }
2032   } else {
2033     // Note: could eliminate cond. jumps to this jump if condition
2034     //       is the same however, seems to be rather unlikely case.
2035     // Note: use jccb() if label to be bound is very close to get
2036     //       an 8-bit displacement
2037     L.add_patch_at(code(), locator());
2038     emit_int8(0x0F);
2039     emit_int8((unsigned char)(0x80 | cc));
2040     emit_int32(0);
2041   }
2042 }
2043 
2044 void Assembler::jccb(Condition cc, Label& L) {
2045   if (L.is_bound()) {
2046     const int short_size = 2;
2047     address entry = target(L);
2048 #ifdef ASSERT
2049     intptr_t dist = (intptr_t)entry - ((intptr_t)pc() + short_size);
2050     intptr_t delta = short_branch_delta();
2051     if (delta != 0) {
2052       dist += (dist < 0 ? (-delta) :delta);
2053     }
2054     assert(is8bit(dist), "Dispacement too large for a short jmp");
2055 #endif
2056     intptr_t offs = (intptr_t)entry - (intptr_t)pc();
2057     // 0111 tttn #8-bit disp
2058     emit_int8(0x70 | cc);
2059     emit_int8((offs - short_size) & 0xFF);
2060   } else {
2061     InstructionMark im(this);
2062     L.add_patch_at(code(), locator());
2063     emit_int8(0x70 | cc);
2064     emit_int8(0);
2065   }
2066 }
2067 
2068 void Assembler::jmp(Address adr) {
2069   InstructionMark im(this);
2070   prefix(adr);
2071   emit_int8((unsigned char)0xFF);
2072   emit_operand(rsp, adr);
2073 }
2074 
2075 void Assembler::jmp(Label& L, bool maybe_short) {
2076   if (L.is_bound()) {
2077     address entry = target(L);
2078     assert(entry != NULL, "jmp most probably wrong");
2079     InstructionMark im(this);
2080     const int short_size = 2;
2081     const int long_size = 5;
2082     intptr_t offs = entry - pc();
2083     if (maybe_short && is8bit(offs - short_size)) {
2084       emit_int8((unsigned char)0xEB);
2085       emit_int8((offs - short_size) & 0xFF);
2086     } else {
2087       emit_int8((unsigned char)0xE9);
2088       emit_int32(offs - long_size);
2089     }
2090   } else {
2091     // By default, forward jumps are always 32-bit displacements, since
2092     // we can't yet know where the label will be bound.  If you're sure that
2093     // the forward jump will not run beyond 256 bytes, use jmpb to
2094     // force an 8-bit displacement.
2095     InstructionMark im(this);
2096     L.add_patch_at(code(), locator());
2097     emit_int8((unsigned char)0xE9);
2098     emit_int32(0);
2099   }
2100 }
2101 
2102 void Assembler::jmp(Register entry) {
2103   int encode = prefix_and_encode(entry->encoding());
2104   emit_int8((unsigned char)0xFF);
2105   emit_int8((unsigned char)(0xE0 | encode));
2106 }
2107 
2108 void Assembler::jmp_literal(address dest, RelocationHolder const& rspec) {
2109   InstructionMark im(this);
2110   emit_int8((unsigned char)0xE9);
2111   assert(dest != NULL, "must have a target");
2112   intptr_t disp = dest - (pc() + sizeof(int32_t));
2113   assert(is_simm32(disp), "must be 32bit offset (jmp)");
2114   emit_data(disp, rspec.reloc(), call32_operand);
2115 }
2116 
2117 void Assembler::jmpb(Label& L) {
2118   if (L.is_bound()) {
2119     const int short_size = 2;
2120     address entry = target(L);
2121     assert(entry != NULL, "jmp most probably wrong");
2122 #ifdef ASSERT
2123     intptr_t dist = (intptr_t)entry - ((intptr_t)pc() + short_size);
2124     intptr_t delta = short_branch_delta();
2125     if (delta != 0) {
2126       dist += (dist < 0 ? (-delta) :delta);
2127     }
2128     assert(is8bit(dist), "Dispacement too large for a short jmp");
2129 #endif
2130     intptr_t offs = entry - pc();
2131     emit_int8((unsigned char)0xEB);
2132     emit_int8((offs - short_size) & 0xFF);
2133   } else {
2134     InstructionMark im(this);
2135     L.add_patch_at(code(), locator());
2136     emit_int8((unsigned char)0xEB);
2137     emit_int8(0);
2138   }
2139 }
2140 
2141 void Assembler::ldmxcsr( Address src) {
2142   if (UseAVX > 0 ) {
2143     InstructionMark im(this);
2144     InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2145     vex_prefix(src, 0, 0, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2146     emit_int8((unsigned char)0xAE);
2147     emit_operand(as_Register(2), src);
2148   } else {
2149     NOT_LP64(assert(VM_Version::supports_sse(), ""));
2150     InstructionMark im(this);
2151     prefix(src);
2152     emit_int8(0x0F);
2153     emit_int8((unsigned char)0xAE);
2154     emit_operand(as_Register(2), src);
2155   }
2156 }
2157 
2158 void Assembler::leal(Register dst, Address src) {
2159   InstructionMark im(this);
2160 #ifdef _LP64
2161   emit_int8(0x67); // addr32
2162   prefix(src, dst);
2163 #endif // LP64
2164   emit_int8((unsigned char)0x8D);
2165   emit_operand(dst, src);
2166 }
2167 
2168 void Assembler::lfence() {
2169   emit_int8(0x0F);
2170   emit_int8((unsigned char)0xAE);
2171   emit_int8((unsigned char)0xE8);
2172 }
2173 
2174 void Assembler::lock() {
2175   emit_int8((unsigned char)0xF0);
2176 }
2177 
2178 void Assembler::lzcntl(Register dst, Register src) {
2179   assert(VM_Version::supports_lzcnt(), "encoding is treated as BSR");
2180   emit_int8((unsigned char)0xF3);
2181   int encode = prefix_and_encode(dst->encoding(), src->encoding());
2182   emit_int8(0x0F);
2183   emit_int8((unsigned char)0xBD);
2184   emit_int8((unsigned char)(0xC0 | encode));
2185 }
2186 
2187 // Emit mfence instruction
2188 void Assembler::mfence() {
2189   NOT_LP64(assert(VM_Version::supports_sse2(), "unsupported");)
2190   emit_int8(0x0F);
2191   emit_int8((unsigned char)0xAE);
2192   emit_int8((unsigned char)0xF0);
2193 }
2194 
2195 // Emit sfence instruction
2196 void Assembler::sfence() {
2197   NOT_LP64(assert(VM_Version::supports_sse2(), "unsupported");)
2198   emit_int8(0x0F);
2199   emit_int8((unsigned char)0xAE);
2200   emit_int8((unsigned char)0xF8);
2201 }
2202 
2203 void Assembler::mov(Register dst, Register src) {
2204   LP64_ONLY(movq(dst, src)) NOT_LP64(movl(dst, src));
2205 }
2206 
2207 void Assembler::movapd(XMMRegister dst, XMMRegister src) {
2208   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2209   int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_128bit;
2210   InstructionAttr attributes(vector_len, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
2211   attributes.set_rex_vex_w_reverted();
2212   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2213   emit_int8(0x28);
2214   emit_int8((unsigned char)(0xC0 | encode));
2215 }
2216 
2217 void Assembler::movaps(XMMRegister dst, XMMRegister src) {
2218   NOT_LP64(assert(VM_Version::supports_sse(), ""));
2219   int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_128bit;
2220   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
2221   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2222   emit_int8(0x28);
2223   emit_int8((unsigned char)(0xC0 | encode));
2224 }
2225 
2226 void Assembler::movlhps(XMMRegister dst, XMMRegister src) {
2227   NOT_LP64(assert(VM_Version::supports_sse(), ""));
2228   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2229   int encode = simd_prefix_and_encode(dst, src, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2230   emit_int8(0x16);
2231   emit_int8((unsigned char)(0xC0 | encode));
2232 }
2233 
2234 void Assembler::movb(Register dst, Address src) {
2235   NOT_LP64(assert(dst->has_byte_register(), "must have byte register"));
2236   InstructionMark im(this);
2237   prefix(src, dst, true);
2238   emit_int8((unsigned char)0x8A);
2239   emit_operand(dst, src);
2240 }
2241 
2242 void Assembler::movddup(XMMRegister dst, XMMRegister src) {
2243   NOT_LP64(assert(VM_Version::supports_sse3(), ""));
2244   int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_128bit;
2245   InstructionAttr attributes(vector_len, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2246   attributes.set_rex_vex_w_reverted();
2247   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2248   emit_int8(0x12);
2249   emit_int8(0xC0 | encode);
2250 }
2251 
2252 void Assembler::kmovbl(KRegister dst, Register src) {
2253   assert(VM_Version::supports_avx512dq(), "");
2254   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2255   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2256   emit_int8((unsigned char)0x92);
2257   emit_int8((unsigned char)(0xC0 | encode));
2258 }
2259 
2260 void Assembler::kmovbl(Register dst, KRegister src) {
2261   assert(VM_Version::supports_avx512dq(), "");
2262   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2263   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2264   emit_int8((unsigned char)0x93);
2265   emit_int8((unsigned char)(0xC0 | encode));
2266 }
2267 
2268 void Assembler::kmovwl(KRegister dst, Register src) {
2269   assert(VM_Version::supports_evex(), "");
2270   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2271   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2272   emit_int8((unsigned char)0x92);
2273   emit_int8((unsigned char)(0xC0 | encode));
2274 }
2275 
2276 void Assembler::kmovwl(Register dst, KRegister src) {
2277   assert(VM_Version::supports_evex(), "");
2278   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2279   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2280   emit_int8((unsigned char)0x93);
2281   emit_int8((unsigned char)(0xC0 | encode));
2282 }
2283 
2284 void Assembler::kmovwl(KRegister dst, Address src) {
2285   assert(VM_Version::supports_evex(), "");
2286   InstructionMark im(this);
2287   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2288   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2289   emit_int8((unsigned char)0x90);
2290   emit_operand((Register)dst, src);
2291 }
2292 
2293 void Assembler::kmovdl(KRegister dst, Register src) {
2294   assert(VM_Version::supports_avx512bw(), "");
2295   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2296   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2297   emit_int8((unsigned char)0x92);
2298   emit_int8((unsigned char)(0xC0 | encode));
2299 }
2300 
2301 void Assembler::kmovdl(Register dst, KRegister src) {
2302   assert(VM_Version::supports_avx512bw(), "");
2303   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2304   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2305   emit_int8((unsigned char)0x93);
2306   emit_int8((unsigned char)(0xC0 | encode));
2307 }
2308 
2309 void Assembler::kmovql(KRegister dst, KRegister src) {
2310   assert(VM_Version::supports_avx512bw(), "");
2311   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2312   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2313   emit_int8((unsigned char)0x90);
2314   emit_int8((unsigned char)(0xC0 | encode));
2315 }
2316 
2317 void Assembler::kmovql(KRegister dst, Address src) {
2318   assert(VM_Version::supports_avx512bw(), "");
2319   InstructionMark im(this);
2320   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2321   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2322   emit_int8((unsigned char)0x90);
2323   emit_operand((Register)dst, src);
2324 }
2325 
2326 void Assembler::kmovql(Address dst, KRegister src) {
2327   assert(VM_Version::supports_avx512bw(), "");
2328   InstructionMark im(this);
2329   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2330   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2331   emit_int8((unsigned char)0x90);
2332   emit_operand((Register)src, dst);
2333 }
2334 
2335 void Assembler::kmovql(KRegister dst, Register src) {
2336   assert(VM_Version::supports_avx512bw(), "");
2337   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2338   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2339   emit_int8((unsigned char)0x92);
2340   emit_int8((unsigned char)(0xC0 | encode));
2341 }
2342 
2343 void Assembler::kmovql(Register dst, KRegister src) {
2344   assert(VM_Version::supports_avx512bw(), "");
2345   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2346   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2347   emit_int8((unsigned char)0x93);
2348   emit_int8((unsigned char)(0xC0 | encode));
2349 }
2350 
2351 void Assembler::knotwl(KRegister dst, KRegister src) {
2352   assert(VM_Version::supports_evex(), "");
2353   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2354   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2355   emit_int8((unsigned char)0x44);
2356   emit_int8((unsigned char)(0xC0 | encode));
2357 }
2358 
2359 // This instruction produces ZF or CF flags
2360 void Assembler::kortestbl(KRegister src1, KRegister src2) {
2361   assert(VM_Version::supports_avx512dq(), "");
2362   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2363   int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2364   emit_int8((unsigned char)0x98);
2365   emit_int8((unsigned char)(0xC0 | encode));
2366 }
2367 
2368 // This instruction produces ZF or CF flags
2369 void Assembler::kortestwl(KRegister src1, KRegister src2) {
2370   assert(VM_Version::supports_evex(), "");
2371   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2372   int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2373   emit_int8((unsigned char)0x98);
2374   emit_int8((unsigned char)(0xC0 | encode));
2375 }
2376 
2377 // This instruction produces ZF or CF flags
2378 void Assembler::kortestdl(KRegister src1, KRegister src2) {
2379   assert(VM_Version::supports_avx512bw(), "");
2380   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2381   int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2382   emit_int8((unsigned char)0x98);
2383   emit_int8((unsigned char)(0xC0 | encode));
2384 }
2385 
2386 // This instruction produces ZF or CF flags
2387 void Assembler::kortestql(KRegister src1, KRegister src2) {
2388   assert(VM_Version::supports_avx512bw(), "");
2389   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2390   int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2391   emit_int8((unsigned char)0x98);
2392   emit_int8((unsigned char)(0xC0 | encode));
2393 }
2394 
2395 // This instruction produces ZF or CF flags
2396 void Assembler::ktestql(KRegister src1, KRegister src2) {
2397   assert(VM_Version::supports_avx512bw(), "");
2398   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2399   int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2400   emit_int8((unsigned char)0x99);
2401   emit_int8((unsigned char)(0xC0 | encode));
2402 }
2403 
2404 void Assembler::ktestq(KRegister src1, KRegister src2) {
2405   assert(VM_Version::supports_avx512bw(), "");
2406   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2407   int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2408   emit_int8((unsigned char)0x99);
2409   emit_int8((unsigned char)(0xC0 | encode));
2410 }
2411 
2412 void Assembler::ktestd(KRegister src1, KRegister src2) {
2413   assert(VM_Version::supports_avx512bw(), "");
2414   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2415   int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2416   emit_int8((unsigned char)0x99);
2417   emit_int8((unsigned char)(0xC0 | encode));
2418 }
2419 
2420 void Assembler::movb(Address dst, int imm8) {
2421   InstructionMark im(this);
2422    prefix(dst);
2423   emit_int8((unsigned char)0xC6);
2424   emit_operand(rax, dst, 1);
2425   emit_int8(imm8);
2426 }
2427 
2428 
2429 void Assembler::movb(Address dst, Register src) {
2430   assert(src->has_byte_register(), "must have byte register");
2431   InstructionMark im(this);
2432   prefix(dst, src, true);
2433   emit_int8((unsigned char)0x88);
2434   emit_operand(src, dst);
2435 }
2436 
2437 void Assembler::movdl(XMMRegister dst, Register src) {
2438   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2439   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2440   int encode = simd_prefix_and_encode(dst, xnoreg, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2441   emit_int8(0x6E);
2442   emit_int8((unsigned char)(0xC0 | encode));
2443 }
2444 
2445 void Assembler::movdl(Register dst, XMMRegister src) {
2446   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2447   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2448   // swap src/dst to get correct prefix
2449   int encode = simd_prefix_and_encode(src, xnoreg, as_XMMRegister(dst->encoding()), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2450   emit_int8(0x7E);
2451   emit_int8((unsigned char)(0xC0 | encode));
2452 }
2453 
2454 void Assembler::movdl(XMMRegister dst, Address src) {
2455   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2456   InstructionMark im(this);
2457   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2458   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
2459   simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2460   emit_int8(0x6E);
2461   emit_operand(dst, src);
2462 }
2463 
2464 void Assembler::movdl(Address dst, XMMRegister src) {
2465   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2466   InstructionMark im(this);
2467   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2468   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
2469   simd_prefix(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2470   emit_int8(0x7E);
2471   emit_operand(src, dst);
2472 }
2473 
2474 void Assembler::movdqa(XMMRegister dst, XMMRegister src) {
2475   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2476   int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_128bit;
2477   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2478   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2479   emit_int8(0x6F);
2480   emit_int8((unsigned char)(0xC0 | encode));
2481 }
2482 
2483 void Assembler::movdqa(XMMRegister dst, Address src) {
2484   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2485   InstructionMark im(this);
2486   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2487   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2488   simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2489   emit_int8(0x6F);
2490   emit_operand(dst, src);
2491 }
2492 
2493 void Assembler::movdqu(XMMRegister dst, Address src) {
2494   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2495   InstructionMark im(this);
2496   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2497   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2498   simd_prefix(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2499   emit_int8(0x6F);
2500   emit_operand(dst, src);
2501 }
2502 
2503 void Assembler::movdqu(XMMRegister dst, XMMRegister src) {
2504   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2505   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2506   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2507   emit_int8(0x6F);
2508   emit_int8((unsigned char)(0xC0 | encode));
2509 }
2510 
2511 void Assembler::movdqu(Address dst, XMMRegister src) {
2512   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2513   InstructionMark im(this);
2514   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2515   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2516   attributes.reset_is_clear_context();
2517   simd_prefix(src, xnoreg, dst, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2518   emit_int8(0x7F);
2519   emit_operand(src, dst);
2520 }
2521 
2522 // Move Unaligned 256bit Vector
2523 void Assembler::vmovdqu(XMMRegister dst, XMMRegister src) {
2524   assert(UseAVX > 0, "");
2525   InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2526   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2527   emit_int8(0x6F);
2528   emit_int8((unsigned char)(0xC0 | encode));
2529 }
2530 
2531 void Assembler::vmovdqu(XMMRegister dst, Address src) {
2532   assert(UseAVX > 0, "");
2533   InstructionMark im(this);
2534   InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2535   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2536   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2537   emit_int8(0x6F);
2538   emit_operand(dst, src);
2539 }
2540 
2541 void Assembler::vmovdqu(Address dst, XMMRegister src) {
2542   assert(UseAVX > 0, "");
2543   InstructionMark im(this);
2544   InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2545   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2546   attributes.reset_is_clear_context();
2547   // swap src<->dst for encoding
2548   assert(src != xnoreg, "sanity");
2549   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2550   emit_int8(0x7F);
2551   emit_operand(src, dst);
2552 }
2553 
2554 // Move Unaligned EVEX enabled Vector (programmable : 8,16,32,64)
2555 void Assembler::evmovdqub(XMMRegister dst, XMMRegister src, int vector_len) {
2556   assert(VM_Version::supports_evex(), "");
2557   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
2558   attributes.set_is_evex_instruction();
2559   int prefix = (_legacy_mode_bw) ? VEX_SIMD_F2 : VEX_SIMD_F3;
2560   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), (Assembler::VexSimdPrefix)prefix, VEX_OPCODE_0F, &attributes);
2561   emit_int8(0x6F);
2562   emit_int8((unsigned char)(0xC0 | encode));
2563 }
2564 
2565 void Assembler::evmovdqub(XMMRegister dst, Address src, int vector_len) {
2566   assert(VM_Version::supports_evex(), "");
2567   InstructionMark im(this);
2568   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
2569   int prefix = (_legacy_mode_bw) ? VEX_SIMD_F2 : VEX_SIMD_F3;
2570   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2571   attributes.set_is_evex_instruction();
2572   vex_prefix(src, 0, dst->encoding(), (Assembler::VexSimdPrefix)prefix, VEX_OPCODE_0F, &attributes);
2573   emit_int8(0x6F);
2574   emit_operand(dst, src);
2575 }
2576 
2577 void Assembler::evmovdqub(Address dst, XMMRegister src, int vector_len) {
2578   assert(VM_Version::supports_evex(), "");
2579   assert(src != xnoreg, "sanity");
2580   InstructionMark im(this);
2581   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
2582   int prefix = (_legacy_mode_bw) ? VEX_SIMD_F2 : VEX_SIMD_F3;
2583   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2584   attributes.set_is_evex_instruction();
2585   vex_prefix(dst, 0, src->encoding(), (Assembler::VexSimdPrefix)prefix, VEX_OPCODE_0F, &attributes);
2586   emit_int8(0x7F);
2587   emit_operand(src, dst);
2588 }
2589 
2590 void Assembler::evmovdqub(XMMRegister dst, KRegister mask, Address src, int vector_len) {
2591   assert(VM_Version::supports_avx512vlbw(), "");
2592   assert(is_vector_masking(), "");    // For stub code use only
2593   InstructionMark im(this);
2594   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
2595   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2596   attributes.set_embedded_opmask_register_specifier(mask);
2597   attributes.set_is_evex_instruction();
2598   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2599   emit_int8(0x6F);
2600   emit_operand(dst, src);
2601 }
2602 
2603 void Assembler::evmovdquw(XMMRegister dst, Address src, int vector_len) {
2604   assert(VM_Version::supports_evex(), "");
2605   InstructionMark im(this);
2606   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
2607   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2608   attributes.set_is_evex_instruction();
2609   int prefix = (_legacy_mode_bw) ? VEX_SIMD_F2 : VEX_SIMD_F3;
2610   vex_prefix(src, 0, dst->encoding(), (Assembler::VexSimdPrefix)prefix, VEX_OPCODE_0F, &attributes);
2611   emit_int8(0x6F);
2612   emit_operand(dst, src);
2613 }
2614 
2615 void Assembler::evmovdquw(XMMRegister dst, KRegister mask, Address src, int vector_len) {
2616   assert(is_vector_masking(), "");
2617   assert(VM_Version::supports_avx512vlbw(), "");
2618   InstructionMark im(this);
2619   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
2620   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2621   attributes.set_embedded_opmask_register_specifier(mask);
2622   attributes.set_is_evex_instruction();
2623   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2624   emit_int8(0x6F);
2625   emit_operand(dst, src);
2626 }
2627 
2628 void Assembler::evmovdquw(Address dst, XMMRegister src, int vector_len) {
2629   assert(VM_Version::supports_evex(), "");
2630   assert(src != xnoreg, "sanity");
2631   InstructionMark im(this);
2632   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
2633   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2634   attributes.set_is_evex_instruction();
2635   int prefix = (_legacy_mode_bw) ? VEX_SIMD_F2 : VEX_SIMD_F3;
2636   vex_prefix(dst, 0, src->encoding(), (Assembler::VexSimdPrefix)prefix, VEX_OPCODE_0F, &attributes);
2637   emit_int8(0x7F);
2638   emit_operand(src, dst);
2639 }
2640 
2641 void Assembler::evmovdquw(Address dst, KRegister mask, XMMRegister src, int vector_len) {
2642   assert(VM_Version::supports_avx512vlbw(), "");
2643   assert(src != xnoreg, "sanity");
2644   InstructionMark im(this);
2645   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2646   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2647   attributes.reset_is_clear_context();
2648   attributes.set_embedded_opmask_register_specifier(mask);
2649   attributes.set_is_evex_instruction();
2650   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2651   emit_int8(0x7F);
2652   emit_operand(src, dst);
2653 }
2654 
2655 void Assembler::evmovdqul(XMMRegister dst, XMMRegister src, int vector_len) {
2656   assert(VM_Version::supports_evex(), "");
2657   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2658   attributes.set_is_evex_instruction();
2659   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2660   emit_int8(0x6F);
2661   emit_int8((unsigned char)(0xC0 | encode));
2662 }
2663 
2664 void Assembler::evmovdqul(XMMRegister dst, Address src, int vector_len) {
2665   assert(VM_Version::supports_evex(), "");
2666   InstructionMark im(this);
2667   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false , /* uses_vl */ true);
2668   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2669   attributes.set_is_evex_instruction();
2670   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2671   emit_int8(0x6F);
2672   emit_operand(dst, src);
2673 }
2674 
2675 void Assembler::evmovdqul(Address dst, XMMRegister src, int vector_len) {
2676   assert(VM_Version::supports_evex(), "");
2677   assert(src != xnoreg, "sanity");
2678   InstructionMark im(this);
2679   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2680   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2681   attributes.reset_is_clear_context();
2682   attributes.set_is_evex_instruction();
2683   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2684   emit_int8(0x7F);
2685   emit_operand(src, dst);
2686 }
2687 
2688 void Assembler::evmovdquq(XMMRegister dst, XMMRegister src, int vector_len) {
2689   assert(VM_Version::supports_evex(), "");
2690   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2691   attributes.set_is_evex_instruction();
2692   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2693   emit_int8(0x6F);
2694   emit_int8((unsigned char)(0xC0 | encode));
2695 }
2696 
2697 void Assembler::evmovdquq(XMMRegister dst, Address src, int vector_len) {
2698   assert(VM_Version::supports_evex(), "");
2699   InstructionMark im(this);
2700   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2701   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2702   attributes.set_is_evex_instruction();
2703   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2704   emit_int8(0x6F);
2705   emit_operand(dst, src);
2706 }
2707 
2708 void Assembler::evmovdquq(Address dst, XMMRegister src, int vector_len) {
2709   assert(VM_Version::supports_evex(), "");
2710   assert(src != xnoreg, "sanity");
2711   InstructionMark im(this);
2712   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2713   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2714   attributes.reset_is_clear_context();
2715   attributes.set_is_evex_instruction();
2716   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2717   emit_int8(0x7F);
2718   emit_operand(src, dst);
2719 }
2720 
2721 // Uses zero extension on 64bit
2722 
2723 void Assembler::movl(Register dst, int32_t imm32) {
2724   int encode = prefix_and_encode(dst->encoding());
2725   emit_int8((unsigned char)(0xB8 | encode));
2726   emit_int32(imm32);
2727 }
2728 
2729 void Assembler::movl(Register dst, Register src) {
2730   int encode = prefix_and_encode(dst->encoding(), src->encoding());
2731   emit_int8((unsigned char)0x8B);
2732   emit_int8((unsigned char)(0xC0 | encode));
2733 }
2734 
2735 void Assembler::movl(Register dst, Address src) {
2736   InstructionMark im(this);
2737   prefix(src, dst);
2738   emit_int8((unsigned char)0x8B);
2739   emit_operand(dst, src);
2740 }
2741 
2742 void Assembler::movl(Address dst, int32_t imm32) {
2743   InstructionMark im(this);
2744   prefix(dst);
2745   emit_int8((unsigned char)0xC7);
2746   emit_operand(rax, dst, 4);
2747   emit_int32(imm32);
2748 }
2749 
2750 void Assembler::movl(Address dst, Register src) {
2751   InstructionMark im(this);
2752   prefix(dst, src);
2753   emit_int8((unsigned char)0x89);
2754   emit_operand(src, dst);
2755 }
2756 
2757 // New cpus require to use movsd and movss to avoid partial register stall
2758 // when loading from memory. But for old Opteron use movlpd instead of movsd.
2759 // The selection is done in MacroAssembler::movdbl() and movflt().
2760 void Assembler::movlpd(XMMRegister dst, Address src) {
2761   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2762   InstructionMark im(this);
2763   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2764   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
2765   attributes.set_rex_vex_w_reverted();
2766   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2767   emit_int8(0x12);
2768   emit_operand(dst, src);
2769 }
2770 
2771 void Assembler::movq( MMXRegister dst, Address src ) {
2772   assert( VM_Version::supports_mmx(), "" );
2773   emit_int8(0x0F);
2774   emit_int8(0x6F);
2775   emit_operand(dst, src);
2776 }
2777 
2778 void Assembler::movq( Address dst, MMXRegister src ) {
2779   assert( VM_Version::supports_mmx(), "" );
2780   emit_int8(0x0F);
2781   emit_int8(0x7F);
2782   // workaround gcc (3.2.1-7a) bug
2783   // In that version of gcc with only an emit_operand(MMX, Address)
2784   // gcc will tail jump and try and reverse the parameters completely
2785   // obliterating dst in the process. By having a version available
2786   // that doesn't need to swap the args at the tail jump the bug is
2787   // avoided.
2788   emit_operand(dst, src);
2789 }
2790 
2791 void Assembler::movq(XMMRegister dst, Address src) {
2792   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2793   InstructionMark im(this);
2794   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2795   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
2796   attributes.set_rex_vex_w_reverted();
2797   simd_prefix(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2798   emit_int8(0x7E);
2799   emit_operand(dst, src);
2800 }
2801 
2802 void Assembler::movq(Address dst, XMMRegister src) {
2803   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2804   InstructionMark im(this);
2805   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2806   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
2807   attributes.set_rex_vex_w_reverted();
2808   simd_prefix(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2809   emit_int8((unsigned char)0xD6);
2810   emit_operand(src, dst);
2811 }
2812 
2813 void Assembler::movsbl(Register dst, Address src) { // movsxb
2814   InstructionMark im(this);
2815   prefix(src, dst);
2816   emit_int8(0x0F);
2817   emit_int8((unsigned char)0xBE);
2818   emit_operand(dst, src);
2819 }
2820 
2821 void Assembler::movsbl(Register dst, Register src) { // movsxb
2822   NOT_LP64(assert(src->has_byte_register(), "must have byte register"));
2823   int encode = prefix_and_encode(dst->encoding(), false, src->encoding(), true);
2824   emit_int8(0x0F);
2825   emit_int8((unsigned char)0xBE);
2826   emit_int8((unsigned char)(0xC0 | encode));
2827 }
2828 
2829 void Assembler::movsd(XMMRegister dst, XMMRegister src) {
2830   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2831   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2832   attributes.set_rex_vex_w_reverted();
2833   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2834   emit_int8(0x10);
2835   emit_int8((unsigned char)(0xC0 | encode));
2836 }
2837 
2838 void Assembler::movsd(XMMRegister dst, Address src) {
2839   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2840   InstructionMark im(this);
2841   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2842   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
2843   attributes.set_rex_vex_w_reverted();
2844   simd_prefix(dst, xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2845   emit_int8(0x10);
2846   emit_operand(dst, src);
2847 }
2848 
2849 void Assembler::movsd(Address dst, XMMRegister src) {
2850   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2851   InstructionMark im(this);
2852   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2853   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
2854   attributes.reset_is_clear_context();
2855   attributes.set_rex_vex_w_reverted();
2856   simd_prefix(src, xnoreg, dst, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2857   emit_int8(0x11);
2858   emit_operand(src, dst);
2859 }
2860 
2861 void Assembler::movss(XMMRegister dst, XMMRegister src) {
2862   NOT_LP64(assert(VM_Version::supports_sse(), ""));
2863   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2864   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2865   emit_int8(0x10);
2866   emit_int8((unsigned char)(0xC0 | encode));
2867 }
2868 
2869 void Assembler::movss(XMMRegister dst, Address src) {
2870   NOT_LP64(assert(VM_Version::supports_sse(), ""));
2871   InstructionMark im(this);
2872   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2873   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
2874   simd_prefix(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2875   emit_int8(0x10);
2876   emit_operand(dst, src);
2877 }
2878 
2879 void Assembler::movss(Address dst, XMMRegister src) {
2880   NOT_LP64(assert(VM_Version::supports_sse(), ""));
2881   InstructionMark im(this);
2882   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2883   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
2884   attributes.reset_is_clear_context();
2885   simd_prefix(src, xnoreg, dst, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2886   emit_int8(0x11);
2887   emit_operand(src, dst);
2888 }
2889 
2890 void Assembler::movswl(Register dst, Address src) { // movsxw
2891   InstructionMark im(this);
2892   prefix(src, dst);
2893   emit_int8(0x0F);
2894   emit_int8((unsigned char)0xBF);
2895   emit_operand(dst, src);
2896 }
2897 
2898 void Assembler::movswl(Register dst, Register src) { // movsxw
2899   int encode = prefix_and_encode(dst->encoding(), src->encoding());
2900   emit_int8(0x0F);
2901   emit_int8((unsigned char)0xBF);
2902   emit_int8((unsigned char)(0xC0 | encode));
2903 }
2904 
2905 void Assembler::movw(Address dst, int imm16) {
2906   InstructionMark im(this);
2907 
2908   emit_int8(0x66); // switch to 16-bit mode
2909   prefix(dst);
2910   emit_int8((unsigned char)0xC7);
2911   emit_operand(rax, dst, 2);
2912   emit_int16(imm16);
2913 }
2914 
2915 void Assembler::movw(Register dst, Address src) {
2916   InstructionMark im(this);
2917   emit_int8(0x66);
2918   prefix(src, dst);
2919   emit_int8((unsigned char)0x8B);
2920   emit_operand(dst, src);
2921 }
2922 
2923 void Assembler::movw(Address dst, Register src) {
2924   InstructionMark im(this);
2925   emit_int8(0x66);
2926   prefix(dst, src);
2927   emit_int8((unsigned char)0x89);
2928   emit_operand(src, dst);
2929 }
2930 
2931 void Assembler::movzbl(Register dst, Address src) { // movzxb
2932   InstructionMark im(this);
2933   prefix(src, dst);
2934   emit_int8(0x0F);
2935   emit_int8((unsigned char)0xB6);
2936   emit_operand(dst, src);
2937 }
2938 
2939 void Assembler::movzbl(Register dst, Register src) { // movzxb
2940   NOT_LP64(assert(src->has_byte_register(), "must have byte register"));
2941   int encode = prefix_and_encode(dst->encoding(), false, src->encoding(), true);
2942   emit_int8(0x0F);
2943   emit_int8((unsigned char)0xB6);
2944   emit_int8(0xC0 | encode);
2945 }
2946 
2947 void Assembler::movzwl(Register dst, Address src) { // movzxw
2948   InstructionMark im(this);
2949   prefix(src, dst);
2950   emit_int8(0x0F);
2951   emit_int8((unsigned char)0xB7);
2952   emit_operand(dst, src);
2953 }
2954 
2955 void Assembler::movzwl(Register dst, Register src) { // movzxw
2956   int encode = prefix_and_encode(dst->encoding(), src->encoding());
2957   emit_int8(0x0F);
2958   emit_int8((unsigned char)0xB7);
2959   emit_int8(0xC0 | encode);
2960 }
2961 
2962 void Assembler::mull(Address src) {
2963   InstructionMark im(this);
2964   prefix(src);
2965   emit_int8((unsigned char)0xF7);
2966   emit_operand(rsp, src);
2967 }
2968 
2969 void Assembler::mull(Register src) {
2970   int encode = prefix_and_encode(src->encoding());
2971   emit_int8((unsigned char)0xF7);
2972   emit_int8((unsigned char)(0xE0 | encode));
2973 }
2974 
2975 void Assembler::mulsd(XMMRegister dst, Address src) {
2976   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2977   InstructionMark im(this);
2978   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2979   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
2980   attributes.set_rex_vex_w_reverted();
2981   simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2982   emit_int8(0x59);
2983   emit_operand(dst, src);
2984 }
2985 
2986 void Assembler::mulsd(XMMRegister dst, XMMRegister src) {
2987   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2988   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2989   attributes.set_rex_vex_w_reverted();
2990   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2991   emit_int8(0x59);
2992   emit_int8((unsigned char)(0xC0 | encode));
2993 }
2994 
2995 void Assembler::mulss(XMMRegister dst, Address src) {
2996   NOT_LP64(assert(VM_Version::supports_sse(), ""));
2997   InstructionMark im(this);
2998   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2999   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
3000   simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
3001   emit_int8(0x59);
3002   emit_operand(dst, src);
3003 }
3004 
3005 void Assembler::mulss(XMMRegister dst, XMMRegister src) {
3006   NOT_LP64(assert(VM_Version::supports_sse(), ""));
3007   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3008   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
3009   emit_int8(0x59);
3010   emit_int8((unsigned char)(0xC0 | encode));
3011 }
3012 
3013 void Assembler::negl(Register dst) {
3014   int encode = prefix_and_encode(dst->encoding());
3015   emit_int8((unsigned char)0xF7);
3016   emit_int8((unsigned char)(0xD8 | encode));
3017 }
3018 
3019 void Assembler::nop(int i) {
3020 #ifdef ASSERT
3021   assert(i > 0, " ");
3022   // The fancy nops aren't currently recognized by debuggers making it a
3023   // pain to disassemble code while debugging. If asserts are on clearly
3024   // speed is not an issue so simply use the single byte traditional nop
3025   // to do alignment.
3026 
3027   for (; i > 0 ; i--) emit_int8((unsigned char)0x90);
3028   return;
3029 
3030 #endif // ASSERT
3031 
3032   if (UseAddressNop && VM_Version::is_intel()) {
3033     //
3034     // Using multi-bytes nops "0x0F 0x1F [address]" for Intel
3035     //  1: 0x90
3036     //  2: 0x66 0x90
3037     //  3: 0x66 0x66 0x90 (don't use "0x0F 0x1F 0x00" - need patching safe padding)
3038     //  4: 0x0F 0x1F 0x40 0x00
3039     //  5: 0x0F 0x1F 0x44 0x00 0x00
3040     //  6: 0x66 0x0F 0x1F 0x44 0x00 0x00
3041     //  7: 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00
3042     //  8: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
3043     //  9: 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
3044     // 10: 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
3045     // 11: 0x66 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
3046 
3047     // The rest coding is Intel specific - don't use consecutive address nops
3048 
3049     // 12: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
3050     // 13: 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
3051     // 14: 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
3052     // 15: 0x66 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
3053 
3054     while(i >= 15) {
3055       // For Intel don't generate consecutive addess nops (mix with regular nops)
3056       i -= 15;
3057       emit_int8(0x66);   // size prefix
3058       emit_int8(0x66);   // size prefix
3059       emit_int8(0x66);   // size prefix
3060       addr_nop_8();
3061       emit_int8(0x66);   // size prefix
3062       emit_int8(0x66);   // size prefix
3063       emit_int8(0x66);   // size prefix
3064       emit_int8((unsigned char)0x90);
3065                          // nop
3066     }
3067     switch (i) {
3068       case 14:
3069         emit_int8(0x66); // size prefix
3070       case 13:
3071         emit_int8(0x66); // size prefix
3072       case 12:
3073         addr_nop_8();
3074         emit_int8(0x66); // size prefix
3075         emit_int8(0x66); // size prefix
3076         emit_int8(0x66); // size prefix
3077         emit_int8((unsigned char)0x90);
3078                          // nop
3079         break;
3080       case 11:
3081         emit_int8(0x66); // size prefix
3082       case 10:
3083         emit_int8(0x66); // size prefix
3084       case 9:
3085         emit_int8(0x66); // size prefix
3086       case 8:
3087         addr_nop_8();
3088         break;
3089       case 7:
3090         addr_nop_7();
3091         break;
3092       case 6:
3093         emit_int8(0x66); // size prefix
3094       case 5:
3095         addr_nop_5();
3096         break;
3097       case 4:
3098         addr_nop_4();
3099         break;
3100       case 3:
3101         // Don't use "0x0F 0x1F 0x00" - need patching safe padding
3102         emit_int8(0x66); // size prefix
3103       case 2:
3104         emit_int8(0x66); // size prefix
3105       case 1:
3106         emit_int8((unsigned char)0x90);
3107                          // nop
3108         break;
3109       default:
3110         assert(i == 0, " ");
3111     }
3112     return;
3113   }
3114   if (UseAddressNop && VM_Version::is_amd()) {
3115     //
3116     // Using multi-bytes nops "0x0F 0x1F [address]" for AMD.
3117     //  1: 0x90
3118     //  2: 0x66 0x90
3119     //  3: 0x66 0x66 0x90 (don't use "0x0F 0x1F 0x00" - need patching safe padding)
3120     //  4: 0x0F 0x1F 0x40 0x00
3121     //  5: 0x0F 0x1F 0x44 0x00 0x00
3122     //  6: 0x66 0x0F 0x1F 0x44 0x00 0x00
3123     //  7: 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00
3124     //  8: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
3125     //  9: 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
3126     // 10: 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
3127     // 11: 0x66 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
3128 
3129     // The rest coding is AMD specific - use consecutive address nops
3130 
3131     // 12: 0x66 0x0F 0x1F 0x44 0x00 0x00 0x66 0x0F 0x1F 0x44 0x00 0x00
3132     // 13: 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00 0x66 0x0F 0x1F 0x44 0x00 0x00
3133     // 14: 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00
3134     // 15: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00
3135     // 16: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
3136     //     Size prefixes (0x66) are added for larger sizes
3137 
3138     while(i >= 22) {
3139       i -= 11;
3140       emit_int8(0x66); // size prefix
3141       emit_int8(0x66); // size prefix
3142       emit_int8(0x66); // size prefix
3143       addr_nop_8();
3144     }
3145     // Generate first nop for size between 21-12
3146     switch (i) {
3147       case 21:
3148         i -= 1;
3149         emit_int8(0x66); // size prefix
3150       case 20:
3151       case 19:
3152         i -= 1;
3153         emit_int8(0x66); // size prefix
3154       case 18:
3155       case 17:
3156         i -= 1;
3157         emit_int8(0x66); // size prefix
3158       case 16:
3159       case 15:
3160         i -= 8;
3161         addr_nop_8();
3162         break;
3163       case 14:
3164       case 13:
3165         i -= 7;
3166         addr_nop_7();
3167         break;
3168       case 12:
3169         i -= 6;
3170         emit_int8(0x66); // size prefix
3171         addr_nop_5();
3172         break;
3173       default:
3174         assert(i < 12, " ");
3175     }
3176 
3177     // Generate second nop for size between 11-1
3178     switch (i) {
3179       case 11:
3180         emit_int8(0x66); // size prefix
3181       case 10:
3182         emit_int8(0x66); // size prefix
3183       case 9:
3184         emit_int8(0x66); // size prefix
3185       case 8:
3186         addr_nop_8();
3187         break;
3188       case 7:
3189         addr_nop_7();
3190         break;
3191       case 6:
3192         emit_int8(0x66); // size prefix
3193       case 5:
3194         addr_nop_5();
3195         break;
3196       case 4:
3197         addr_nop_4();
3198         break;
3199       case 3:
3200         // Don't use "0x0F 0x1F 0x00" - need patching safe padding
3201         emit_int8(0x66); // size prefix
3202       case 2:
3203         emit_int8(0x66); // size prefix
3204       case 1:
3205         emit_int8((unsigned char)0x90);
3206                          // nop
3207         break;
3208       default:
3209         assert(i == 0, " ");
3210     }
3211     return;
3212   }
3213 
3214   if (UseAddressNop && VM_Version::is_zx()) {
3215     //
3216     // Using multi-bytes nops "0x0F 0x1F [address]" for ZX
3217     //  1: 0x90
3218     //  2: 0x66 0x90
3219     //  3: 0x66 0x66 0x90 (don't use "0x0F 0x1F 0x00" - need patching safe padding)
3220     //  4: 0x0F 0x1F 0x40 0x00
3221     //  5: 0x0F 0x1F 0x44 0x00 0x00
3222     //  6: 0x66 0x0F 0x1F 0x44 0x00 0x00
3223     //  7: 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00
3224     //  8: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
3225     //  9: 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
3226     // 10: 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
3227     // 11: 0x66 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
3228 
3229     // The rest coding is ZX specific - don't use consecutive address nops
3230 
3231     // 12: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
3232     // 13: 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
3233     // 14: 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
3234     // 15: 0x66 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
3235 
3236     while (i >= 15) {
3237       // For ZX don't generate consecutive addess nops (mix with regular nops)
3238       i -= 15;
3239       emit_int8(0x66);   // size prefix
3240       emit_int8(0x66);   // size prefix
3241       emit_int8(0x66);   // size prefix
3242       addr_nop_8();
3243       emit_int8(0x66);   // size prefix
3244       emit_int8(0x66);   // size prefix
3245       emit_int8(0x66);   // size prefix
3246       emit_int8((unsigned char)0x90);
3247                          // nop
3248     }
3249     switch (i) {
3250       case 14:
3251         emit_int8(0x66); // size prefix
3252       case 13:
3253         emit_int8(0x66); // size prefix
3254       case 12:
3255         addr_nop_8();
3256         emit_int8(0x66); // size prefix
3257         emit_int8(0x66); // size prefix
3258         emit_int8(0x66); // size prefix
3259         emit_int8((unsigned char)0x90);
3260                          // nop
3261         break;
3262       case 11:
3263         emit_int8(0x66); // size prefix
3264       case 10:
3265         emit_int8(0x66); // size prefix
3266       case 9:
3267         emit_int8(0x66); // size prefix
3268       case 8:
3269         addr_nop_8();
3270         break;
3271       case 7:
3272         addr_nop_7();
3273         break;
3274       case 6:
3275         emit_int8(0x66); // size prefix
3276       case 5:
3277         addr_nop_5();
3278         break;
3279       case 4:
3280         addr_nop_4();
3281         break;
3282       case 3:
3283         // Don't use "0x0F 0x1F 0x00" - need patching safe padding
3284         emit_int8(0x66); // size prefix
3285       case 2:
3286         emit_int8(0x66); // size prefix
3287       case 1:
3288         emit_int8((unsigned char)0x90);
3289                          // nop
3290         break;
3291       default:
3292         assert(i == 0, " ");
3293     }
3294     return;
3295   }
3296 
3297   // Using nops with size prefixes "0x66 0x90".
3298   // From AMD Optimization Guide:
3299   //  1: 0x90
3300   //  2: 0x66 0x90
3301   //  3: 0x66 0x66 0x90
3302   //  4: 0x66 0x66 0x66 0x90
3303   //  5: 0x66 0x66 0x90 0x66 0x90
3304   //  6: 0x66 0x66 0x90 0x66 0x66 0x90
3305   //  7: 0x66 0x66 0x66 0x90 0x66 0x66 0x90
3306   //  8: 0x66 0x66 0x66 0x90 0x66 0x66 0x66 0x90
3307   //  9: 0x66 0x66 0x90 0x66 0x66 0x90 0x66 0x66 0x90
3308   // 10: 0x66 0x66 0x66 0x90 0x66 0x66 0x90 0x66 0x66 0x90
3309   //
3310   while(i > 12) {
3311     i -= 4;
3312     emit_int8(0x66); // size prefix
3313     emit_int8(0x66);
3314     emit_int8(0x66);
3315     emit_int8((unsigned char)0x90);
3316                      // nop
3317   }
3318   // 1 - 12 nops
3319   if(i > 8) {
3320     if(i > 9) {
3321       i -= 1;
3322       emit_int8(0x66);
3323     }
3324     i -= 3;
3325     emit_int8(0x66);
3326     emit_int8(0x66);
3327     emit_int8((unsigned char)0x90);
3328   }
3329   // 1 - 8 nops
3330   if(i > 4) {
3331     if(i > 6) {
3332       i -= 1;
3333       emit_int8(0x66);
3334     }
3335     i -= 3;
3336     emit_int8(0x66);
3337     emit_int8(0x66);
3338     emit_int8((unsigned char)0x90);
3339   }
3340   switch (i) {
3341     case 4:
3342       emit_int8(0x66);
3343     case 3:
3344       emit_int8(0x66);
3345     case 2:
3346       emit_int8(0x66);
3347     case 1:
3348       emit_int8((unsigned char)0x90);
3349       break;
3350     default:
3351       assert(i == 0, " ");
3352   }
3353 }
3354 
3355 void Assembler::notl(Register dst) {
3356   int encode = prefix_and_encode(dst->encoding());
3357   emit_int8((unsigned char)0xF7);
3358   emit_int8((unsigned char)(0xD0 | encode));
3359 }
3360 
3361 void Assembler::orl(Address dst, int32_t imm32) {
3362   InstructionMark im(this);
3363   prefix(dst);
3364   emit_arith_operand(0x81, rcx, dst, imm32);
3365 }
3366 
3367 void Assembler::orl(Register dst, int32_t imm32) {
3368   prefix(dst);
3369   emit_arith(0x81, 0xC8, dst, imm32);
3370 }
3371 
3372 void Assembler::orl(Register dst, Address src) {
3373   InstructionMark im(this);
3374   prefix(src, dst);
3375   emit_int8(0x0B);
3376   emit_operand(dst, src);
3377 }
3378 
3379 void Assembler::orl(Register dst, Register src) {
3380   (void) prefix_and_encode(dst->encoding(), src->encoding());
3381   emit_arith(0x0B, 0xC0, dst, src);
3382 }
3383 
3384 void Assembler::orl(Address dst, Register src) {
3385   InstructionMark im(this);
3386   prefix(dst, src);
3387   emit_int8(0x09);
3388   emit_operand(src, dst);
3389 }
3390 
3391 void Assembler::orb(Address dst, int imm8) {
3392   InstructionMark im(this);
3393   prefix(dst);
3394   emit_int8((unsigned char)0x80);
3395   emit_operand(rcx, dst, 1);
3396   emit_int8(imm8);
3397 }
3398 
3399 void Assembler::packuswb(XMMRegister dst, Address src) {
3400   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3401   assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
3402   InstructionMark im(this);
3403   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
3404   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
3405   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3406   emit_int8(0x67);
3407   emit_operand(dst, src);
3408 }
3409 
3410 void Assembler::packuswb(XMMRegister dst, XMMRegister src) {
3411   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3412   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
3413   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3414   emit_int8(0x67);
3415   emit_int8((unsigned char)(0xC0 | encode));
3416 }
3417 
3418 void Assembler::vpackuswb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
3419   assert(UseAVX > 0, "some form of AVX must be enabled");
3420   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
3421   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3422   emit_int8(0x67);
3423   emit_int8((unsigned char)(0xC0 | encode));
3424 }
3425 
3426 void Assembler::vpermq(XMMRegister dst, XMMRegister src, int imm8, int vector_len) {
3427   assert(VM_Version::supports_avx2(), "");
3428   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
3429   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3430   emit_int8(0x00);
3431   emit_int8(0xC0 | encode);
3432   emit_int8(imm8);
3433 }
3434 
3435 void Assembler::vperm2i128(XMMRegister dst,  XMMRegister nds, XMMRegister src, int imm8) {
3436   assert(VM_Version::supports_avx2(), "");
3437   InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3438   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3439   emit_int8(0x46);
3440   emit_int8(0xC0 | encode);
3441   emit_int8(imm8);
3442 }
3443 
3444 void Assembler::vperm2f128(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8) {
3445   assert(VM_Version::supports_avx(), "");
3446   InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3447   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3448   emit_int8(0x06);
3449   emit_int8(0xC0 | encode);
3450   emit_int8(imm8);
3451 }
3452 
3453 void Assembler::evpermi2q(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
3454   assert(VM_Version::supports_evex(), "");
3455   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
3456   attributes.set_is_evex_instruction();
3457   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3458   emit_int8(0x76);
3459   emit_int8((unsigned char)(0xC0 | encode));
3460 }
3461 
3462 
3463 void Assembler::pause() {
3464   emit_int8((unsigned char)0xF3);
3465   emit_int8((unsigned char)0x90);
3466 }
3467 
3468 void Assembler::ud2() {
3469   emit_int8(0x0F);
3470   emit_int8(0x0B);
3471 }
3472 
3473 void Assembler::pcmpestri(XMMRegister dst, Address src, int imm8) {
3474   assert(VM_Version::supports_sse4_2(), "");
3475   InstructionMark im(this);
3476   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3477   simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3478   emit_int8(0x61);
3479   emit_operand(dst, src);
3480   emit_int8(imm8);
3481 }
3482 
3483 void Assembler::pcmpestri(XMMRegister dst, XMMRegister src, int imm8) {
3484   assert(VM_Version::supports_sse4_2(), "");
3485   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3486   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3487   emit_int8(0x61);
3488   emit_int8((unsigned char)(0xC0 | encode));
3489   emit_int8(imm8);
3490 }
3491 
3492 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
3493 void Assembler::pcmpeqb(XMMRegister dst, XMMRegister src) {
3494   assert(VM_Version::supports_sse2(), "");
3495   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3496   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3497   emit_int8(0x74);
3498   emit_int8((unsigned char)(0xC0 | encode));
3499 }
3500 
3501 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
3502 void Assembler::vpcmpeqb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
3503   assert(VM_Version::supports_avx(), "");
3504   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3505   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3506   emit_int8(0x74);
3507   emit_int8((unsigned char)(0xC0 | encode));
3508 }
3509 
3510 // In this context, kdst is written the mask used to process the equal components
3511 void Assembler::evpcmpeqb(KRegister kdst, XMMRegister nds, XMMRegister src, int vector_len) {
3512   assert(VM_Version::supports_avx512bw(), "");
3513   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
3514   attributes.set_is_evex_instruction();
3515   int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3516   emit_int8(0x74);
3517   emit_int8((unsigned char)(0xC0 | encode));
3518 }
3519 
3520 void Assembler::evpcmpgtb(KRegister kdst, XMMRegister nds, Address src, int vector_len) {
3521   assert(VM_Version::supports_avx512vlbw(), "");
3522   InstructionMark im(this);
3523   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3524   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
3525   attributes.set_is_evex_instruction();
3526   int dst_enc = kdst->encoding();
3527   vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3528   emit_int8(0x64);
3529   emit_operand(as_Register(dst_enc), src);
3530 }
3531 
3532 void Assembler::evpcmpgtb(KRegister kdst, KRegister mask, XMMRegister nds, Address src, int vector_len) {
3533   assert(is_vector_masking(), "");
3534   assert(VM_Version::supports_avx512vlbw(), "");
3535   InstructionMark im(this);
3536   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
3537   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
3538   attributes.reset_is_clear_context();
3539   attributes.set_embedded_opmask_register_specifier(mask);
3540   attributes.set_is_evex_instruction();
3541   int dst_enc = kdst->encoding();
3542   vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3543   emit_int8(0x64);
3544   emit_operand(as_Register(dst_enc), src);
3545 }
3546 
3547 void Assembler::evpcmpuw(KRegister kdst, XMMRegister nds, XMMRegister src, ComparisonPredicate vcc, int vector_len) {
3548   assert(VM_Version::supports_avx512vlbw(), "");
3549   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3550   attributes.set_is_evex_instruction();
3551   int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3552   emit_int8(0x3E);
3553   emit_int8((unsigned char)(0xC0 | encode));
3554   emit_int8(vcc);
3555 }
3556 
3557 void Assembler::evpcmpuw(KRegister kdst, KRegister mask, XMMRegister nds, XMMRegister src, ComparisonPredicate vcc, int vector_len) {
3558   assert(is_vector_masking(), "");
3559   assert(VM_Version::supports_avx512vlbw(), "");
3560   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
3561   attributes.reset_is_clear_context();
3562   attributes.set_embedded_opmask_register_specifier(mask);
3563   attributes.set_is_evex_instruction();
3564   int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3565   emit_int8(0x3E);
3566   emit_int8((unsigned char)(0xC0 | encode));
3567   emit_int8(vcc);
3568 }
3569 
3570 void Assembler::evpcmpuw(KRegister kdst, XMMRegister nds, Address src, ComparisonPredicate vcc, int vector_len) {
3571   assert(VM_Version::supports_avx512vlbw(), "");
3572   InstructionMark im(this);
3573   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3574   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
3575   attributes.set_is_evex_instruction();
3576   int dst_enc = kdst->encoding();
3577   vex_prefix(src, nds->encoding(), kdst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3578   emit_int8(0x3E);
3579   emit_operand(as_Register(dst_enc), src);
3580   emit_int8(vcc);
3581 }
3582 
3583 void Assembler::evpcmpeqb(KRegister kdst, XMMRegister nds, Address src, int vector_len) {
3584   assert(VM_Version::supports_avx512bw(), "");
3585   InstructionMark im(this);
3586   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3587   attributes.set_is_evex_instruction();
3588   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
3589   int dst_enc = kdst->encoding();
3590   vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3591   emit_int8(0x74);
3592   emit_operand(as_Register(dst_enc), src);
3593 }
3594 
3595 void Assembler::evpcmpeqb(KRegister kdst, KRegister mask, XMMRegister nds, Address src, int vector_len) {
3596   assert(VM_Version::supports_avx512vlbw(), "");
3597   assert(is_vector_masking(), "");    // For stub code use only
3598   InstructionMark im(this);
3599   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_reg_mask */ false, /* uses_vl */ false);
3600   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
3601   attributes.reset_is_clear_context();
3602   attributes.set_embedded_opmask_register_specifier(mask);
3603   attributes.set_is_evex_instruction();
3604   vex_prefix(src, nds->encoding(), kdst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3605   emit_int8(0x74);
3606   emit_operand(as_Register(kdst->encoding()), src);
3607 }
3608 
3609 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
3610 void Assembler::pcmpeqw(XMMRegister dst, XMMRegister src) {
3611   assert(VM_Version::supports_sse2(), "");
3612   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3613   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3614   emit_int8(0x75);
3615   emit_int8((unsigned char)(0xC0 | encode));
3616 }
3617 
3618 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
3619 void Assembler::vpcmpeqw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
3620   assert(VM_Version::supports_avx(), "");
3621   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3622   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3623   emit_int8(0x75);
3624   emit_int8((unsigned char)(0xC0 | encode));
3625 }
3626 
3627 // In this context, kdst is written the mask used to process the equal components
3628 void Assembler::evpcmpeqw(KRegister kdst, XMMRegister nds, XMMRegister src, int vector_len) {
3629   assert(VM_Version::supports_avx512bw(), "");
3630   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
3631   attributes.set_is_evex_instruction();
3632   int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3633   emit_int8(0x75);
3634   emit_int8((unsigned char)(0xC0 | encode));
3635 }
3636 
3637 void Assembler::evpcmpeqw(KRegister kdst, XMMRegister nds, Address src, int vector_len) {
3638   assert(VM_Version::supports_avx512bw(), "");
3639   InstructionMark im(this);
3640   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
3641   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
3642   attributes.set_is_evex_instruction();
3643   int dst_enc = kdst->encoding();
3644   vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3645   emit_int8(0x75);
3646   emit_operand(as_Register(dst_enc), src);
3647 }
3648 
3649 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
3650 void Assembler::pcmpeqd(XMMRegister dst, XMMRegister src) {
3651   assert(VM_Version::supports_sse2(), "");
3652   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3653   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3654   emit_int8(0x76);
3655   emit_int8((unsigned char)(0xC0 | encode));
3656 }
3657 
3658 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
3659 void Assembler::vpcmpeqd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
3660   assert(VM_Version::supports_avx(), "");
3661   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3662   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3663   emit_int8(0x76);
3664   emit_int8((unsigned char)(0xC0 | encode));
3665 }
3666 
3667 // In this context, kdst is written the mask used to process the equal components
3668 void Assembler::evpcmpeqd(KRegister kdst, XMMRegister nds, XMMRegister src, int vector_len) {
3669   assert(VM_Version::supports_evex(), "");
3670   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
3671   attributes.set_is_evex_instruction();
3672   attributes.reset_is_clear_context();
3673   int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3674   emit_int8(0x76);
3675   emit_int8((unsigned char)(0xC0 | encode));
3676 }
3677 
3678 void Assembler::evpcmpeqd(KRegister kdst, XMMRegister nds, Address src, int vector_len) {
3679   assert(VM_Version::supports_evex(), "");
3680   InstructionMark im(this);
3681   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
3682   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
3683   attributes.reset_is_clear_context();
3684   attributes.set_is_evex_instruction();
3685   int dst_enc = kdst->encoding();
3686   vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3687   emit_int8(0x76);
3688   emit_operand(as_Register(dst_enc), src);
3689 }
3690 
3691 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
3692 void Assembler::pcmpeqq(XMMRegister dst, XMMRegister src) {
3693   assert(VM_Version::supports_sse4_1(), "");
3694   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3695   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3696   emit_int8(0x29);
3697   emit_int8((unsigned char)(0xC0 | encode));
3698 }
3699 
3700 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
3701 void Assembler::vpcmpeqq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
3702   assert(VM_Version::supports_avx(), "");
3703   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3704   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3705   emit_int8(0x29);
3706   emit_int8((unsigned char)(0xC0 | encode));
3707 }
3708 
3709 // In this context, kdst is written the mask used to process the equal components
3710 void Assembler::evpcmpeqq(KRegister kdst, XMMRegister nds, XMMRegister src, int vector_len) {
3711   assert(VM_Version::supports_evex(), "");
3712   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
3713   attributes.reset_is_clear_context();
3714   attributes.set_is_evex_instruction();
3715   int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3716   emit_int8(0x29);
3717   emit_int8((unsigned char)(0xC0 | encode));
3718 }
3719 
3720 // In this context, kdst is written the mask used to process the equal components
3721 void Assembler::evpcmpeqq(KRegister kdst, XMMRegister nds, Address src, int vector_len) {
3722   assert(VM_Version::supports_evex(), "");
3723   InstructionMark im(this);
3724   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
3725   attributes.reset_is_clear_context();
3726   attributes.set_is_evex_instruction();
3727   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
3728   int dst_enc = kdst->encoding();
3729   vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3730   emit_int8(0x29);
3731   emit_operand(as_Register(dst_enc), src);
3732 }
3733 
3734 void Assembler::pmovmskb(Register dst, XMMRegister src) {
3735   assert(VM_Version::supports_sse2(), "");
3736   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3737   int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3738   emit_int8((unsigned char)0xD7);
3739   emit_int8((unsigned char)(0xC0 | encode));
3740 }
3741 
3742 void Assembler::vpmovmskb(Register dst, XMMRegister src) {
3743   assert(VM_Version::supports_avx2(), "");
3744   InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3745   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3746   emit_int8((unsigned char)0xD7);
3747   emit_int8((unsigned char)(0xC0 | encode));
3748 }
3749 
3750 void Assembler::pextrd(Register dst, XMMRegister src, int imm8) {
3751   assert(VM_Version::supports_sse4_1(), "");
3752   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
3753   int encode = simd_prefix_and_encode(src, xnoreg, as_XMMRegister(dst->encoding()), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3754   emit_int8(0x16);
3755   emit_int8((unsigned char)(0xC0 | encode));
3756   emit_int8(imm8);
3757 }
3758 
3759 void Assembler::pextrd(Address dst, XMMRegister src, int imm8) {
3760   assert(VM_Version::supports_sse4_1(), "");
3761   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
3762   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
3763   simd_prefix(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3764   emit_int8(0x16);
3765   emit_operand(src, dst);
3766   emit_int8(imm8);
3767 }
3768 
3769 void Assembler::pextrq(Register dst, XMMRegister src, int imm8) {
3770   assert(VM_Version::supports_sse4_1(), "");
3771   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
3772   int encode = simd_prefix_and_encode(src, xnoreg, as_XMMRegister(dst->encoding()), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3773   emit_int8(0x16);
3774   emit_int8((unsigned char)(0xC0 | encode));
3775   emit_int8(imm8);
3776 }
3777 
3778 void Assembler::pextrq(Address dst, XMMRegister src, int imm8) {
3779   assert(VM_Version::supports_sse4_1(), "");
3780   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
3781   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
3782   simd_prefix(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3783   emit_int8(0x16);
3784   emit_operand(src, dst);
3785   emit_int8(imm8);
3786 }
3787 
3788 void Assembler::pextrw(Register dst, XMMRegister src, int imm8) {
3789   assert(VM_Version::supports_sse2(), "");
3790   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
3791   int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3792   emit_int8((unsigned char)0xC5);
3793   emit_int8((unsigned char)(0xC0 | encode));
3794   emit_int8(imm8);
3795 }
3796 
3797 void Assembler::pextrw(Address dst, XMMRegister src, int imm8) {
3798   assert(VM_Version::supports_sse4_1(), "");
3799   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
3800   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_16bit);
3801   simd_prefix(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3802   emit_int8((unsigned char)0x15);
3803   emit_operand(src, dst);
3804   emit_int8(imm8);
3805 }
3806 
3807 void Assembler::pextrb(Address dst, XMMRegister src, int imm8) {
3808   assert(VM_Version::supports_sse4_1(), "");
3809   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
3810   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_8bit);
3811   simd_prefix(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3812   emit_int8(0x14);
3813   emit_operand(src, dst);
3814   emit_int8(imm8);
3815 }
3816 
3817 void Assembler::pinsrd(XMMRegister dst, Register src, int imm8) {
3818   assert(VM_Version::supports_sse4_1(), "");
3819   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
3820   int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3821   emit_int8(0x22);
3822   emit_int8((unsigned char)(0xC0 | encode));
3823   emit_int8(imm8);
3824 }
3825 
3826 void Assembler::pinsrd(XMMRegister dst, Address src, int imm8) {
3827   assert(VM_Version::supports_sse4_1(), "");
3828   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
3829   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
3830   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3831   emit_int8(0x22);
3832   emit_operand(dst,src);
3833   emit_int8(imm8);
3834 }
3835 
3836 void Assembler::pinsrq(XMMRegister dst, Register src, int imm8) {
3837   assert(VM_Version::supports_sse4_1(), "");
3838   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
3839   int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3840   emit_int8(0x22);
3841   emit_int8((unsigned char)(0xC0 | encode));
3842   emit_int8(imm8);
3843 }
3844 
3845 void Assembler::pinsrq(XMMRegister dst, Address src, int imm8) {
3846   assert(VM_Version::supports_sse4_1(), "");
3847   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
3848   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
3849   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3850   emit_int8(0x22);
3851   emit_operand(dst, src);
3852   emit_int8(imm8);
3853 }
3854 
3855 void Assembler::pinsrw(XMMRegister dst, Register src, int imm8) {
3856   assert(VM_Version::supports_sse2(), "");
3857   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
3858   int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3859   emit_int8((unsigned char)0xC4);
3860   emit_int8((unsigned char)(0xC0 | encode));
3861   emit_int8(imm8);
3862 }
3863 
3864 void Assembler::pinsrw(XMMRegister dst, Address src, int imm8) {
3865   assert(VM_Version::supports_sse2(), "");
3866   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
3867   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_16bit);
3868   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3869   emit_int8((unsigned char)0xC4);
3870   emit_operand(dst, src);
3871   emit_int8(imm8);
3872 }
3873 
3874 void Assembler::pinsrb(XMMRegister dst, Address src, int imm8) {
3875   assert(VM_Version::supports_sse4_1(), "");
3876   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
3877   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_8bit);
3878   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3879   emit_int8(0x20);
3880   emit_operand(dst, src);
3881   emit_int8(imm8);
3882 }
3883 
3884 void Assembler::pmovzxbw(XMMRegister dst, Address src) {
3885   assert(VM_Version::supports_sse4_1(), "");
3886   InstructionMark im(this);
3887   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
3888   attributes.set_address_attributes(/* tuple_type */ EVEX_HVM, /* input_size_in_bits */ EVEX_NObit);
3889   simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3890   emit_int8(0x30);
3891   emit_operand(dst, src);
3892 }
3893 
3894 void Assembler::pmovzxbw(XMMRegister dst, XMMRegister src) {
3895   assert(VM_Version::supports_sse4_1(), "");
3896   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
3897   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3898   emit_int8(0x30);
3899   emit_int8((unsigned char)(0xC0 | encode));
3900 }
3901 
3902 void Assembler::vpmovzxbw(XMMRegister dst, Address src, int vector_len) {
3903   assert(VM_Version::supports_avx(), "");
3904   InstructionMark im(this);
3905   assert(dst != xnoreg, "sanity");
3906   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
3907   attributes.set_address_attributes(/* tuple_type */ EVEX_HVM, /* input_size_in_bits */ EVEX_NObit);
3908   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3909   emit_int8(0x30);
3910   emit_operand(dst, src);
3911 }
3912 
3913 void Assembler::vpmovzxbw(XMMRegister dst, XMMRegister src, int vector_len) {
3914   assert(vector_len == AVX_128bit? VM_Version::supports_avx() :
3915   vector_len == AVX_256bit? VM_Version::supports_avx2() :
3916   vector_len == AVX_512bit? VM_Version::supports_avx512bw() : 0, "");
3917   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
3918   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3919   emit_int8(0x30);
3920   emit_int8((unsigned char) (0xC0 | encode));
3921 }
3922 
3923 
3924 void Assembler::evpmovzxbw(XMMRegister dst, KRegister mask, Address src, int vector_len) {
3925   assert(is_vector_masking(), "");
3926   assert(VM_Version::supports_avx512vlbw(), "");
3927   assert(dst != xnoreg, "sanity");
3928   InstructionMark im(this);
3929   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
3930   attributes.set_address_attributes(/* tuple_type */ EVEX_HVM, /* input_size_in_bits */ EVEX_NObit);
3931   attributes.set_embedded_opmask_register_specifier(mask);
3932   attributes.set_is_evex_instruction();
3933   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3934   emit_int8(0x30);
3935   emit_operand(dst, src);
3936 }
3937 void Assembler::evpmovwb(Address dst, XMMRegister src, int vector_len) {
3938   assert(VM_Version::supports_avx512vlbw(), "");
3939   assert(src != xnoreg, "sanity");
3940   InstructionMark im(this);
3941   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3942   attributes.set_address_attributes(/* tuple_type */ EVEX_HVM, /* input_size_in_bits */ EVEX_NObit);
3943   attributes.set_is_evex_instruction();
3944   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
3945   emit_int8(0x30);
3946   emit_operand(src, dst);
3947 }
3948 
3949 void Assembler::evpmovwb(Address dst, KRegister mask, XMMRegister src, int vector_len) {
3950   assert(is_vector_masking(), "");
3951   assert(VM_Version::supports_avx512vlbw(), "");
3952   assert(src != xnoreg, "sanity");
3953   InstructionMark im(this);
3954   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
3955   attributes.set_address_attributes(/* tuple_type */ EVEX_HVM, /* input_size_in_bits */ EVEX_NObit);
3956   attributes.reset_is_clear_context();
3957   attributes.set_embedded_opmask_register_specifier(mask);
3958   attributes.set_is_evex_instruction();
3959   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
3960   emit_int8(0x30);
3961   emit_operand(src, dst);
3962 }
3963 
3964 void Assembler::evpmovdb(Address dst, XMMRegister src, int vector_len) {
3965   assert(VM_Version::supports_evex(), "");
3966   assert(src != xnoreg, "sanity");
3967   InstructionMark im(this);
3968   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3969   attributes.set_address_attributes(/* tuple_type */ EVEX_QVM, /* input_size_in_bits */ EVEX_NObit);
3970   attributes.set_is_evex_instruction();
3971   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
3972   emit_int8(0x31);
3973   emit_operand(src, dst);
3974 }
3975 
3976 void Assembler::vpmovzxwd(XMMRegister dst, XMMRegister src, int vector_len) {
3977   assert(vector_len == AVX_128bit? VM_Version::supports_avx() :
3978   vector_len == AVX_256bit? VM_Version::supports_avx2() :
3979   vector_len == AVX_512bit? VM_Version::supports_evex() : 0, " ");
3980   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
3981   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3982   emit_int8(0x33);
3983   emit_int8((unsigned char)(0xC0 | encode));
3984 }
3985 
3986 // generic
3987 void Assembler::pop(Register dst) {
3988   int encode = prefix_and_encode(dst->encoding());
3989   emit_int8(0x58 | encode);
3990 }
3991 
3992 void Assembler::popcntl(Register dst, Address src) {
3993   assert(VM_Version::supports_popcnt(), "must support");
3994   InstructionMark im(this);
3995   emit_int8((unsigned char)0xF3);
3996   prefix(src, dst);
3997   emit_int8(0x0F);
3998   emit_int8((unsigned char)0xB8);
3999   emit_operand(dst, src);
4000 }
4001 
4002 void Assembler::popcntl(Register dst, Register src) {
4003   assert(VM_Version::supports_popcnt(), "must support");
4004   emit_int8((unsigned char)0xF3);
4005   int encode = prefix_and_encode(dst->encoding(), src->encoding());
4006   emit_int8(0x0F);
4007   emit_int8((unsigned char)0xB8);
4008   emit_int8((unsigned char)(0xC0 | encode));
4009 }
4010 
4011 void Assembler::vpopcntd(XMMRegister dst, XMMRegister src, int vector_len) {
4012   assert(VM_Version::supports_vpopcntdq(), "must support vpopcntdq feature");
4013   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4014   attributes.set_is_evex_instruction();
4015   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4016   emit_int8(0x55);
4017   emit_int8((unsigned char)(0xC0 | encode));
4018 }
4019 
4020 void Assembler::popf() {
4021   emit_int8((unsigned char)0x9D);
4022 }
4023 
4024 #ifndef _LP64 // no 32bit push/pop on amd64
4025 void Assembler::popl(Address dst) {
4026   // NOTE: this will adjust stack by 8byte on 64bits
4027   InstructionMark im(this);
4028   prefix(dst);
4029   emit_int8((unsigned char)0x8F);
4030   emit_operand(rax, dst);
4031 }
4032 #endif
4033 
4034 void Assembler::prefetch_prefix(Address src) {
4035   prefix(src);
4036   emit_int8(0x0F);
4037 }
4038 
4039 void Assembler::prefetchnta(Address src) {
4040   NOT_LP64(assert(VM_Version::supports_sse(), "must support"));
4041   InstructionMark im(this);
4042   prefetch_prefix(src);
4043   emit_int8(0x18);
4044   emit_operand(rax, src); // 0, src
4045 }
4046 
4047 void Assembler::prefetchr(Address src) {
4048   assert(VM_Version::supports_3dnow_prefetch(), "must support");
4049   InstructionMark im(this);
4050   prefetch_prefix(src);
4051   emit_int8(0x0D);
4052   emit_operand(rax, src); // 0, src
4053 }
4054 
4055 void Assembler::prefetcht0(Address src) {
4056   NOT_LP64(assert(VM_Version::supports_sse(), "must support"));
4057   InstructionMark im(this);
4058   prefetch_prefix(src);
4059   emit_int8(0x18);
4060   emit_operand(rcx, src); // 1, src
4061 }
4062 
4063 void Assembler::prefetcht1(Address src) {
4064   NOT_LP64(assert(VM_Version::supports_sse(), "must support"));
4065   InstructionMark im(this);
4066   prefetch_prefix(src);
4067   emit_int8(0x18);
4068   emit_operand(rdx, src); // 2, src
4069 }
4070 
4071 void Assembler::prefetcht2(Address src) {
4072   NOT_LP64(assert(VM_Version::supports_sse(), "must support"));
4073   InstructionMark im(this);
4074   prefetch_prefix(src);
4075   emit_int8(0x18);
4076   emit_operand(rbx, src); // 3, src
4077 }
4078 
4079 void Assembler::prefetchw(Address src) {
4080   assert(VM_Version::supports_3dnow_prefetch(), "must support");
4081   InstructionMark im(this);
4082   prefetch_prefix(src);
4083   emit_int8(0x0D);
4084   emit_operand(rcx, src); // 1, src
4085 }
4086 
4087 void Assembler::prefix(Prefix p) {
4088   emit_int8(p);
4089 }
4090 
4091 void Assembler::pshufb(XMMRegister dst, XMMRegister src) {
4092   assert(VM_Version::supports_ssse3(), "");
4093   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
4094   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4095   emit_int8(0x00);
4096   emit_int8((unsigned char)(0xC0 | encode));
4097 }
4098 
4099 void Assembler::vpshufb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4100   assert(vector_len == AVX_128bit? VM_Version::supports_avx() :
4101          vector_len == AVX_256bit? VM_Version::supports_avx2() :
4102          0, "");
4103   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
4104   int encode = simd_prefix_and_encode(dst, nds, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4105   emit_int8(0x00);
4106   emit_int8((unsigned char)(0xC0 | encode));
4107 }
4108 
4109 void Assembler::pshufb(XMMRegister dst, Address src) {
4110   assert(VM_Version::supports_ssse3(), "");
4111   InstructionMark im(this);
4112   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
4113   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
4114   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4115   emit_int8(0x00);
4116   emit_operand(dst, src);
4117 }
4118 
4119 void Assembler::pshufd(XMMRegister dst, XMMRegister src, int mode) {
4120   assert(isByte(mode), "invalid value");
4121   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4122   int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_128bit;
4123   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4124   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4125   emit_int8(0x70);
4126   emit_int8((unsigned char)(0xC0 | encode));
4127   emit_int8(mode & 0xFF);
4128 }
4129 
4130 void Assembler::vpshufd(XMMRegister dst, XMMRegister src, int mode, int vector_len) {
4131   assert(vector_len == AVX_128bit? VM_Version::supports_avx() :
4132          vector_len == AVX_256bit? VM_Version::supports_avx2() :
4133          0, "");
4134   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4135   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4136   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4137   emit_int8(0x70);
4138   emit_int8((unsigned char)(0xC0 | encode));
4139   emit_int8(mode & 0xFF);
4140 }
4141 
4142 void Assembler::pshufd(XMMRegister dst, Address src, int mode) {
4143   assert(isByte(mode), "invalid value");
4144   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4145   assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
4146   InstructionMark im(this);
4147   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4148   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
4149   simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4150   emit_int8(0x70);
4151   emit_operand(dst, src);
4152   emit_int8(mode & 0xFF);
4153 }
4154 
4155 void Assembler::pshuflw(XMMRegister dst, XMMRegister src, int mode) {
4156   assert(isByte(mode), "invalid value");
4157   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4158   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
4159   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
4160   emit_int8(0x70);
4161   emit_int8((unsigned char)(0xC0 | encode));
4162   emit_int8(mode & 0xFF);
4163 }
4164 
4165 void Assembler::pshuflw(XMMRegister dst, Address src, int mode) {
4166   assert(isByte(mode), "invalid value");
4167   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4168   assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
4169   InstructionMark im(this);
4170   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
4171   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
4172   simd_prefix(dst, xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
4173   emit_int8(0x70);
4174   emit_operand(dst, src);
4175   emit_int8(mode & 0xFF);
4176 }
4177 void Assembler::evshufi64x2(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8, int vector_len) {
4178   assert(VM_Version::supports_evex(), "requires EVEX support");
4179   assert(vector_len == Assembler::AVX_256bit || vector_len == Assembler::AVX_512bit, "");
4180   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
4181   attributes.set_is_evex_instruction();
4182   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
4183   emit_int8(0x43);
4184   emit_int8((unsigned char)(0xC0 | encode));
4185   emit_int8(imm8 & 0xFF);
4186 }
4187 
4188 void Assembler::psrldq(XMMRegister dst, int shift) {
4189   // Shift left 128 bit value in dst XMMRegister by shift number of bytes.
4190   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4191   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
4192   int encode = simd_prefix_and_encode(xmm3, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4193   emit_int8(0x73);
4194   emit_int8((unsigned char)(0xC0 | encode));
4195   emit_int8(shift);
4196 }
4197 
4198 void Assembler::pslldq(XMMRegister dst, int shift) {
4199   // Shift left 128 bit value in dst XMMRegister by shift number of bytes.
4200   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4201   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
4202   // XMM7 is for /7 encoding: 66 0F 73 /7 ib
4203   int encode = simd_prefix_and_encode(xmm7, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4204   emit_int8(0x73);
4205   emit_int8((unsigned char)(0xC0 | encode));
4206   emit_int8(shift);
4207 }
4208 
4209 void Assembler::ptest(XMMRegister dst, Address src) {
4210   assert(VM_Version::supports_sse4_1(), "");
4211   assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
4212   InstructionMark im(this);
4213   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
4214   simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4215   emit_int8(0x17);
4216   emit_operand(dst, src);
4217 }
4218 
4219 void Assembler::ptest(XMMRegister dst, XMMRegister src) {
4220   assert(VM_Version::supports_sse4_1(), "");
4221   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
4222   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4223   emit_int8(0x17);
4224   emit_int8((unsigned char)(0xC0 | encode));
4225 }
4226 
4227 void Assembler::vptest(XMMRegister dst, Address src) {
4228   assert(VM_Version::supports_avx(), "");
4229   InstructionMark im(this);
4230   InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
4231   assert(dst != xnoreg, "sanity");
4232   // swap src<->dst for encoding
4233   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4234   emit_int8(0x17);
4235   emit_operand(dst, src);
4236 }
4237 
4238 void Assembler::vptest(XMMRegister dst, XMMRegister src) {
4239   assert(VM_Version::supports_avx(), "");
4240   InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
4241   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4242   emit_int8(0x17);
4243   emit_int8((unsigned char)(0xC0 | encode));
4244 }
4245 
4246 void Assembler::punpcklbw(XMMRegister dst, Address src) {
4247   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4248   assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
4249   InstructionMark im(this);
4250   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_vlbw, /* no_mask_reg */ false, /* uses_vl */ true);
4251   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
4252   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4253   emit_int8(0x60);
4254   emit_operand(dst, src);
4255 }
4256 
4257 void Assembler::punpcklbw(XMMRegister dst, XMMRegister src) {
4258   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4259   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_vlbw, /* no_mask_reg */ false, /* uses_vl */ true);
4260   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4261   emit_int8(0x60);
4262   emit_int8((unsigned char)(0xC0 | encode));
4263 }
4264 
4265 void Assembler::punpckldq(XMMRegister dst, Address src) {
4266   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4267   assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
4268   InstructionMark im(this);
4269   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4270   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
4271   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4272   emit_int8(0x62);
4273   emit_operand(dst, src);
4274 }
4275 
4276 void Assembler::punpckldq(XMMRegister dst, XMMRegister src) {
4277   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4278   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4279   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4280   emit_int8(0x62);
4281   emit_int8((unsigned char)(0xC0 | encode));
4282 }
4283 
4284 void Assembler::punpcklqdq(XMMRegister dst, XMMRegister src) {
4285   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4286   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4287   attributes.set_rex_vex_w_reverted();
4288   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4289   emit_int8(0x6C);
4290   emit_int8((unsigned char)(0xC0 | encode));
4291 }
4292 
4293 void Assembler::push(int32_t imm32) {
4294   // in 64bits we push 64bits onto the stack but only
4295   // take a 32bit immediate
4296   emit_int8(0x68);
4297   emit_int32(imm32);
4298 }
4299 
4300 void Assembler::push(Register src) {
4301   int encode = prefix_and_encode(src->encoding());
4302 
4303   emit_int8(0x50 | encode);
4304 }
4305 
4306 void Assembler::pushf() {
4307   emit_int8((unsigned char)0x9C);
4308 }
4309 
4310 #ifndef _LP64 // no 32bit push/pop on amd64
4311 void Assembler::pushl(Address src) {
4312   // Note this will push 64bit on 64bit
4313   InstructionMark im(this);
4314   prefix(src);
4315   emit_int8((unsigned char)0xFF);
4316   emit_operand(rsi, src);
4317 }
4318 #endif
4319 
4320 void Assembler::rcll(Register dst, int imm8) {
4321   assert(isShiftCount(imm8), "illegal shift count");
4322   int encode = prefix_and_encode(dst->encoding());
4323   if (imm8 == 1) {
4324     emit_int8((unsigned char)0xD1);
4325     emit_int8((unsigned char)(0xD0 | encode));
4326   } else {
4327     emit_int8((unsigned char)0xC1);
4328     emit_int8((unsigned char)0xD0 | encode);
4329     emit_int8(imm8);
4330   }
4331 }
4332 
4333 void Assembler::rcpps(XMMRegister dst, XMMRegister src) {
4334   NOT_LP64(assert(VM_Version::supports_sse(), ""));
4335   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
4336   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
4337   emit_int8(0x53);
4338   emit_int8((unsigned char)(0xC0 | encode));
4339 }
4340 
4341 void Assembler::rcpss(XMMRegister dst, XMMRegister src) {
4342   NOT_LP64(assert(VM_Version::supports_sse(), ""));
4343   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
4344   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
4345   emit_int8(0x53);
4346   emit_int8((unsigned char)(0xC0 | encode));
4347 }
4348 
4349 void Assembler::rdtsc() {
4350   emit_int8((unsigned char)0x0F);
4351   emit_int8((unsigned char)0x31);
4352 }
4353 
4354 // copies data from [esi] to [edi] using rcx pointer sized words
4355 // generic
4356 void Assembler::rep_mov() {
4357   emit_int8((unsigned char)0xF3);
4358   // MOVSQ
4359   LP64_ONLY(prefix(REX_W));
4360   emit_int8((unsigned char)0xA5);
4361 }
4362 
4363 // sets rcx bytes with rax, value at [edi]
4364 void Assembler::rep_stosb() {
4365   emit_int8((unsigned char)0xF3); // REP
4366   LP64_ONLY(prefix(REX_W));
4367   emit_int8((unsigned char)0xAA); // STOSB
4368 }
4369 
4370 // sets rcx pointer sized words with rax, value at [edi]
4371 // generic
4372 void Assembler::rep_stos() {
4373   emit_int8((unsigned char)0xF3); // REP
4374   LP64_ONLY(prefix(REX_W));       // LP64:STOSQ, LP32:STOSD
4375   emit_int8((unsigned char)0xAB);
4376 }
4377 
4378 // scans rcx pointer sized words at [edi] for occurance of rax,
4379 // generic
4380 void Assembler::repne_scan() { // repne_scan
4381   emit_int8((unsigned char)0xF2);
4382   // SCASQ
4383   LP64_ONLY(prefix(REX_W));
4384   emit_int8((unsigned char)0xAF);
4385 }
4386 
4387 #ifdef _LP64
4388 // scans rcx 4 byte words at [edi] for occurance of rax,
4389 // generic
4390 void Assembler::repne_scanl() { // repne_scan
4391   emit_int8((unsigned char)0xF2);
4392   // SCASL
4393   emit_int8((unsigned char)0xAF);
4394 }
4395 #endif
4396 
4397 void Assembler::ret(int imm16) {
4398   if (imm16 == 0) {
4399     emit_int8((unsigned char)0xC3);
4400   } else {
4401     emit_int8((unsigned char)0xC2);
4402     emit_int16(imm16);
4403   }
4404 }
4405 
4406 void Assembler::sahf() {
4407 #ifdef _LP64
4408   // Not supported in 64bit mode
4409   ShouldNotReachHere();
4410 #endif
4411   emit_int8((unsigned char)0x9E);
4412 }
4413 
4414 void Assembler::sarl(Register dst, int imm8) {
4415   int encode = prefix_and_encode(dst->encoding());
4416   assert(isShiftCount(imm8), "illegal shift count");
4417   if (imm8 == 1) {
4418     emit_int8((unsigned char)0xD1);
4419     emit_int8((unsigned char)(0xF8 | encode));
4420   } else {
4421     emit_int8((unsigned char)0xC1);
4422     emit_int8((unsigned char)(0xF8 | encode));
4423     emit_int8(imm8);
4424   }
4425 }
4426 
4427 void Assembler::sarl(Register dst) {
4428   int encode = prefix_and_encode(dst->encoding());
4429   emit_int8((unsigned char)0xD3);
4430   emit_int8((unsigned char)(0xF8 | encode));
4431 }
4432 
4433 void Assembler::sbbl(Address dst, int32_t imm32) {
4434   InstructionMark im(this);
4435   prefix(dst);
4436   emit_arith_operand(0x81, rbx, dst, imm32);
4437 }
4438 
4439 void Assembler::sbbl(Register dst, int32_t imm32) {
4440   prefix(dst);
4441   emit_arith(0x81, 0xD8, dst, imm32);
4442 }
4443 
4444 
4445 void Assembler::sbbl(Register dst, Address src) {
4446   InstructionMark im(this);
4447   prefix(src, dst);
4448   emit_int8(0x1B);
4449   emit_operand(dst, src);
4450 }
4451 
4452 void Assembler::sbbl(Register dst, Register src) {
4453   (void) prefix_and_encode(dst->encoding(), src->encoding());
4454   emit_arith(0x1B, 0xC0, dst, src);
4455 }
4456 
4457 void Assembler::setb(Condition cc, Register dst) {
4458   assert(0 <= cc && cc < 16, "illegal cc");
4459   int encode = prefix_and_encode(dst->encoding(), true);
4460   emit_int8(0x0F);
4461   emit_int8((unsigned char)0x90 | cc);
4462   emit_int8((unsigned char)(0xC0 | encode));
4463 }
4464 
4465 void Assembler::palignr(XMMRegister dst, XMMRegister src, int imm8) {
4466   assert(VM_Version::supports_ssse3(), "");
4467   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ false);
4468   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
4469   emit_int8((unsigned char)0x0F);
4470   emit_int8((unsigned char)(0xC0 | encode));
4471   emit_int8(imm8);
4472 }
4473 
4474 void Assembler::vpalignr(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8, int vector_len) {
4475   assert(vector_len == AVX_128bit? VM_Version::supports_avx() :
4476          vector_len == AVX_256bit? VM_Version::supports_avx2() :
4477          0, "");
4478   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
4479   int encode = simd_prefix_and_encode(dst, nds, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
4480   emit_int8((unsigned char)0x0F);
4481   emit_int8((unsigned char)(0xC0 | encode));
4482   emit_int8(imm8);
4483 }
4484 
4485 void Assembler::evalignq(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) {
4486   assert(VM_Version::supports_evex(), "");
4487   InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4488   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
4489   emit_int8(0x3);
4490   emit_int8((unsigned char)(0xC0 | encode));
4491   emit_int8(imm8);
4492 }
4493 
4494 void Assembler::pblendw(XMMRegister dst, XMMRegister src, int imm8) {
4495   assert(VM_Version::supports_sse4_1(), "");
4496   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
4497   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
4498   emit_int8((unsigned char)0x0E);
4499   emit_int8((unsigned char)(0xC0 | encode));
4500   emit_int8(imm8);
4501 }
4502 
4503 void Assembler::sha1rnds4(XMMRegister dst, XMMRegister src, int imm8) {
4504   assert(VM_Version::supports_sha(), "");
4505   int encode = rex_prefix_and_encode(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3A, /* rex_w */ false);
4506   emit_int8((unsigned char)0xCC);
4507   emit_int8((unsigned char)(0xC0 | encode));
4508   emit_int8((unsigned char)imm8);
4509 }
4510 
4511 void Assembler::sha1nexte(XMMRegister dst, XMMRegister src) {
4512   assert(VM_Version::supports_sha(), "");
4513   int encode = rex_prefix_and_encode(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, /* rex_w */ false);
4514   emit_int8((unsigned char)0xC8);
4515   emit_int8((unsigned char)(0xC0 | encode));
4516 }
4517 
4518 void Assembler::sha1msg1(XMMRegister dst, XMMRegister src) {
4519   assert(VM_Version::supports_sha(), "");
4520   int encode = rex_prefix_and_encode(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, /* rex_w */ false);
4521   emit_int8((unsigned char)0xC9);
4522   emit_int8((unsigned char)(0xC0 | encode));
4523 }
4524 
4525 void Assembler::sha1msg2(XMMRegister dst, XMMRegister src) {
4526   assert(VM_Version::supports_sha(), "");
4527   int encode = rex_prefix_and_encode(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, /* rex_w */ false);
4528   emit_int8((unsigned char)0xCA);
4529   emit_int8((unsigned char)(0xC0 | encode));
4530 }
4531 
4532 // xmm0 is implicit additional source to this instruction.
4533 void Assembler::sha256rnds2(XMMRegister dst, XMMRegister src) {
4534   assert(VM_Version::supports_sha(), "");
4535   int encode = rex_prefix_and_encode(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, /* rex_w */ false);
4536   emit_int8((unsigned char)0xCB);
4537   emit_int8((unsigned char)(0xC0 | encode));
4538 }
4539 
4540 void Assembler::sha256msg1(XMMRegister dst, XMMRegister src) {
4541   assert(VM_Version::supports_sha(), "");
4542   int encode = rex_prefix_and_encode(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, /* rex_w */ false);
4543   emit_int8((unsigned char)0xCC);
4544   emit_int8((unsigned char)(0xC0 | encode));
4545 }
4546 
4547 void Assembler::sha256msg2(XMMRegister dst, XMMRegister src) {
4548   assert(VM_Version::supports_sha(), "");
4549   int encode = rex_prefix_and_encode(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, /* rex_w */ false);
4550   emit_int8((unsigned char)0xCD);
4551   emit_int8((unsigned char)(0xC0 | encode));
4552 }
4553 
4554 
4555 void Assembler::shll(Register dst, int imm8) {
4556   assert(isShiftCount(imm8), "illegal shift count");
4557   int encode = prefix_and_encode(dst->encoding());
4558   if (imm8 == 1 ) {
4559     emit_int8((unsigned char)0xD1);
4560     emit_int8((unsigned char)(0xE0 | encode));
4561   } else {
4562     emit_int8((unsigned char)0xC1);
4563     emit_int8((unsigned char)(0xE0 | encode));
4564     emit_int8(imm8);
4565   }
4566 }
4567 
4568 void Assembler::shll(Register dst) {
4569   int encode = prefix_and_encode(dst->encoding());
4570   emit_int8((unsigned char)0xD3);
4571   emit_int8((unsigned char)(0xE0 | encode));
4572 }
4573 
4574 void Assembler::shrl(Register dst, int imm8) {
4575   assert(isShiftCount(imm8), "illegal shift count");
4576   int encode = prefix_and_encode(dst->encoding());
4577   emit_int8((unsigned char)0xC1);
4578   emit_int8((unsigned char)(0xE8 | encode));
4579   emit_int8(imm8);
4580 }
4581 
4582 void Assembler::shrl(Register dst) {
4583   int encode = prefix_and_encode(dst->encoding());
4584   emit_int8((unsigned char)0xD3);
4585   emit_int8((unsigned char)(0xE8 | encode));
4586 }
4587 
4588 // copies a single word from [esi] to [edi]
4589 void Assembler::smovl() {
4590   emit_int8((unsigned char)0xA5);
4591 }
4592 
4593 void Assembler::sqrtsd(XMMRegister dst, XMMRegister src) {
4594   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4595   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4596   attributes.set_rex_vex_w_reverted();
4597   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
4598   emit_int8(0x51);
4599   emit_int8((unsigned char)(0xC0 | encode));
4600 }
4601 
4602 void Assembler::sqrtsd(XMMRegister dst, Address src) {
4603   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4604   InstructionMark im(this);
4605   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4606   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
4607   attributes.set_rex_vex_w_reverted();
4608   simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
4609   emit_int8(0x51);
4610   emit_operand(dst, src);
4611 }
4612 
4613 void Assembler::sqrtss(XMMRegister dst, XMMRegister src) {
4614   NOT_LP64(assert(VM_Version::supports_sse(), ""));
4615   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4616   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
4617   emit_int8(0x51);
4618   emit_int8((unsigned char)(0xC0 | encode));
4619 }
4620 
4621 void Assembler::std() {
4622   emit_int8((unsigned char)0xFD);
4623 }
4624 
4625 void Assembler::sqrtss(XMMRegister dst, Address src) {
4626   NOT_LP64(assert(VM_Version::supports_sse(), ""));
4627   InstructionMark im(this);
4628   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4629   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
4630   simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
4631   emit_int8(0x51);
4632   emit_operand(dst, src);
4633 }
4634 
4635 void Assembler::stmxcsr( Address dst) {
4636   if (UseAVX > 0 ) {
4637     assert(VM_Version::supports_avx(), "");
4638     InstructionMark im(this);
4639     InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
4640     vex_prefix(dst, 0, 0, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
4641     emit_int8((unsigned char)0xAE);
4642     emit_operand(as_Register(3), dst);
4643   } else {
4644     NOT_LP64(assert(VM_Version::supports_sse(), ""));
4645     InstructionMark im(this);
4646     prefix(dst);
4647     emit_int8(0x0F);
4648     emit_int8((unsigned char)0xAE);
4649     emit_operand(as_Register(3), dst);
4650   }
4651 }
4652 
4653 void Assembler::subl(Address dst, int32_t imm32) {
4654   InstructionMark im(this);
4655   prefix(dst);
4656   emit_arith_operand(0x81, rbp, dst, imm32);
4657 }
4658 
4659 void Assembler::subl(Address dst, Register src) {
4660   InstructionMark im(this);
4661   prefix(dst, src);
4662   emit_int8(0x29);
4663   emit_operand(src, dst);
4664 }
4665 
4666 void Assembler::subl(Register dst, int32_t imm32) {
4667   prefix(dst);
4668   emit_arith(0x81, 0xE8, dst, imm32);
4669 }
4670 
4671 // Force generation of a 4 byte immediate value even if it fits into 8bit
4672 void Assembler::subl_imm32(Register dst, int32_t imm32) {
4673   prefix(dst);
4674   emit_arith_imm32(0x81, 0xE8, dst, imm32);
4675 }
4676 
4677 void Assembler::subl(Register dst, Address src) {
4678   InstructionMark im(this);
4679   prefix(src, dst);
4680   emit_int8(0x2B);
4681   emit_operand(dst, src);
4682 }
4683 
4684 void Assembler::subl(Register dst, Register src) {
4685   (void) prefix_and_encode(dst->encoding(), src->encoding());
4686   emit_arith(0x2B, 0xC0, dst, src);
4687 }
4688 
4689 void Assembler::subsd(XMMRegister dst, XMMRegister src) {
4690   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4691   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4692   attributes.set_rex_vex_w_reverted();
4693   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
4694   emit_int8(0x5C);
4695   emit_int8((unsigned char)(0xC0 | encode));
4696 }
4697 
4698 void Assembler::subsd(XMMRegister dst, Address src) {
4699   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4700   InstructionMark im(this);
4701   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4702   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
4703   attributes.set_rex_vex_w_reverted();
4704   simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
4705   emit_int8(0x5C);
4706   emit_operand(dst, src);
4707 }
4708 
4709 void Assembler::subss(XMMRegister dst, XMMRegister src) {
4710   NOT_LP64(assert(VM_Version::supports_sse(), ""));
4711   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true , /* uses_vl */ false);
4712   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
4713   emit_int8(0x5C);
4714   emit_int8((unsigned char)(0xC0 | encode));
4715 }
4716 
4717 void Assembler::subss(XMMRegister dst, Address src) {
4718   NOT_LP64(assert(VM_Version::supports_sse(), ""));
4719   InstructionMark im(this);
4720   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4721   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
4722   simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
4723   emit_int8(0x5C);
4724   emit_operand(dst, src);
4725 }
4726 
4727 void Assembler::testb(Register dst, int imm8) {
4728   NOT_LP64(assert(dst->has_byte_register(), "must have byte register"));
4729   (void) prefix_and_encode(dst->encoding(), true);
4730   emit_arith_b(0xF6, 0xC0, dst, imm8);
4731 }
4732 
4733 void Assembler::testb(Address dst, int imm8) {
4734   InstructionMark im(this);
4735   prefix(dst);
4736   emit_int8((unsigned char)0xF6);
4737   emit_operand(rax, dst, 1);
4738   emit_int8(imm8);
4739 }
4740 
4741 void Assembler::testl(Register dst, int32_t imm32) {
4742   // not using emit_arith because test
4743   // doesn't support sign-extension of
4744   // 8bit operands
4745   int encode = dst->encoding();
4746   if (encode == 0) {
4747     emit_int8((unsigned char)0xA9);
4748   } else {
4749     encode = prefix_and_encode(encode);
4750     emit_int8((unsigned char)0xF7);
4751     emit_int8((unsigned char)(0xC0 | encode));
4752   }
4753   emit_int32(imm32);
4754 }
4755 
4756 void Assembler::testl(Register dst, Register src) {
4757   (void) prefix_and_encode(dst->encoding(), src->encoding());
4758   emit_arith(0x85, 0xC0, dst, src);
4759 }
4760 
4761 void Assembler::testl(Register dst, Address src) {
4762   InstructionMark im(this);
4763   prefix(src, dst);
4764   emit_int8((unsigned char)0x85);
4765   emit_operand(dst, src);
4766 }
4767 
4768 void Assembler::tzcntl(Register dst, Register src) {
4769   assert(VM_Version::supports_bmi1(), "tzcnt instruction not supported");
4770   emit_int8((unsigned char)0xF3);
4771   int encode = prefix_and_encode(dst->encoding(), src->encoding());
4772   emit_int8(0x0F);
4773   emit_int8((unsigned char)0xBC);
4774   emit_int8((unsigned char)0xC0 | encode);
4775 }
4776 
4777 void Assembler::tzcntq(Register dst, Register src) {
4778   assert(VM_Version::supports_bmi1(), "tzcnt instruction not supported");
4779   emit_int8((unsigned char)0xF3);
4780   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
4781   emit_int8(0x0F);
4782   emit_int8((unsigned char)0xBC);
4783   emit_int8((unsigned char)(0xC0 | encode));
4784 }
4785 
4786 void Assembler::ucomisd(XMMRegister dst, Address src) {
4787   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4788   InstructionMark im(this);
4789   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4790   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
4791   attributes.set_rex_vex_w_reverted();
4792   simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4793   emit_int8(0x2E);
4794   emit_operand(dst, src);
4795 }
4796 
4797 void Assembler::ucomisd(XMMRegister dst, XMMRegister src) {
4798   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4799   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4800   attributes.set_rex_vex_w_reverted();
4801   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4802   emit_int8(0x2E);
4803   emit_int8((unsigned char)(0xC0 | encode));
4804 }
4805 
4806 void Assembler::ucomiss(XMMRegister dst, Address src) {
4807   NOT_LP64(assert(VM_Version::supports_sse(), ""));
4808   InstructionMark im(this);
4809   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4810   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
4811   simd_prefix(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
4812   emit_int8(0x2E);
4813   emit_operand(dst, src);
4814 }
4815 
4816 void Assembler::ucomiss(XMMRegister dst, XMMRegister src) {
4817   NOT_LP64(assert(VM_Version::supports_sse(), ""));
4818   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4819   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
4820   emit_int8(0x2E);
4821   emit_int8((unsigned char)(0xC0 | encode));
4822 }
4823 
4824 void Assembler::xabort(int8_t imm8) {
4825   emit_int8((unsigned char)0xC6);
4826   emit_int8((unsigned char)0xF8);
4827   emit_int8((unsigned char)(imm8 & 0xFF));
4828 }
4829 
4830 void Assembler::xaddb(Address dst, Register src) {
4831   InstructionMark im(this);
4832   prefix(dst, src, true);
4833   emit_int8(0x0F);
4834   emit_int8((unsigned char)0xC0);
4835   emit_operand(src, dst);
4836 }
4837 
4838 void Assembler::xaddw(Address dst, Register src) {
4839   InstructionMark im(this);
4840   emit_int8(0x66);
4841   prefix(dst, src);
4842   emit_int8(0x0F);
4843   emit_int8((unsigned char)0xC1);
4844   emit_operand(src, dst);
4845 }
4846 
4847 void Assembler::xaddl(Address dst, Register src) {
4848   InstructionMark im(this);
4849   prefix(dst, src);
4850   emit_int8(0x0F);
4851   emit_int8((unsigned char)0xC1);
4852   emit_operand(src, dst);
4853 }
4854 
4855 void Assembler::xbegin(Label& abort, relocInfo::relocType rtype) {
4856   InstructionMark im(this);
4857   relocate(rtype);
4858   if (abort.is_bound()) {
4859     address entry = target(abort);
4860     assert(entry != NULL, "abort entry NULL");
4861     intptr_t offset = entry - pc();
4862     emit_int8((unsigned char)0xC7);
4863     emit_int8((unsigned char)0xF8);
4864     emit_int32(offset - 6); // 2 opcode + 4 address
4865   } else {
4866     abort.add_patch_at(code(), locator());
4867     emit_int8((unsigned char)0xC7);
4868     emit_int8((unsigned char)0xF8);
4869     emit_int32(0);
4870   }
4871 }
4872 
4873 void Assembler::xchgb(Register dst, Address src) { // xchg
4874   InstructionMark im(this);
4875   prefix(src, dst, true);
4876   emit_int8((unsigned char)0x86);
4877   emit_operand(dst, src);
4878 }
4879 
4880 void Assembler::xchgw(Register dst, Address src) { // xchg
4881   InstructionMark im(this);
4882   emit_int8(0x66);
4883   prefix(src, dst);
4884   emit_int8((unsigned char)0x87);
4885   emit_operand(dst, src);
4886 }
4887 
4888 void Assembler::xchgl(Register dst, Address src) { // xchg
4889   InstructionMark im(this);
4890   prefix(src, dst);
4891   emit_int8((unsigned char)0x87);
4892   emit_operand(dst, src);
4893 }
4894 
4895 void Assembler::xchgl(Register dst, Register src) {
4896   int encode = prefix_and_encode(dst->encoding(), src->encoding());
4897   emit_int8((unsigned char)0x87);
4898   emit_int8((unsigned char)(0xC0 | encode));
4899 }
4900 
4901 void Assembler::xend() {
4902   emit_int8((unsigned char)0x0F);
4903   emit_int8((unsigned char)0x01);
4904   emit_int8((unsigned char)0xD5);
4905 }
4906 
4907 void Assembler::xgetbv() {
4908   emit_int8(0x0F);
4909   emit_int8(0x01);
4910   emit_int8((unsigned char)0xD0);
4911 }
4912 
4913 void Assembler::xorl(Register dst, int32_t imm32) {
4914   prefix(dst);
4915   emit_arith(0x81, 0xF0, dst, imm32);
4916 }
4917 
4918 void Assembler::xorl(Register dst, Address src) {
4919   InstructionMark im(this);
4920   prefix(src, dst);
4921   emit_int8(0x33);
4922   emit_operand(dst, src);
4923 }
4924 
4925 void Assembler::xorl(Register dst, Register src) {
4926   (void) prefix_and_encode(dst->encoding(), src->encoding());
4927   emit_arith(0x33, 0xC0, dst, src);
4928 }
4929 
4930 void Assembler::xorb(Register dst, Address src) {
4931   InstructionMark im(this);
4932   prefix(src, dst);
4933   emit_int8(0x32);
4934   emit_operand(dst, src);
4935 }
4936 
4937 // AVX 3-operands scalar float-point arithmetic instructions
4938 
4939 void Assembler::vaddsd(XMMRegister dst, XMMRegister nds, Address src) {
4940   assert(VM_Version::supports_avx(), "");
4941   InstructionMark im(this);
4942   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4943   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
4944   attributes.set_rex_vex_w_reverted();
4945   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
4946   emit_int8(0x58);
4947   emit_operand(dst, src);
4948 }
4949 
4950 void Assembler::vaddsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
4951   assert(VM_Version::supports_avx(), "");
4952   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4953   attributes.set_rex_vex_w_reverted();
4954   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
4955   emit_int8(0x58);
4956   emit_int8((unsigned char)(0xC0 | encode));
4957 }
4958 
4959 void Assembler::vaddss(XMMRegister dst, XMMRegister nds, Address src) {
4960   assert(VM_Version::supports_avx(), "");
4961   InstructionMark im(this);
4962   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4963   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
4964   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
4965   emit_int8(0x58);
4966   emit_operand(dst, src);
4967 }
4968 
4969 void Assembler::vaddss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
4970   assert(VM_Version::supports_avx(), "");
4971   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4972   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
4973   emit_int8(0x58);
4974   emit_int8((unsigned char)(0xC0 | encode));
4975 }
4976 
4977 void Assembler::vdivsd(XMMRegister dst, XMMRegister nds, Address src) {
4978   assert(VM_Version::supports_avx(), "");
4979   InstructionMark im(this);
4980   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4981   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
4982   attributes.set_rex_vex_w_reverted();
4983   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
4984   emit_int8(0x5E);
4985   emit_operand(dst, src);
4986 }
4987 
4988 void Assembler::vdivsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
4989   assert(VM_Version::supports_avx(), "");
4990   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4991   attributes.set_rex_vex_w_reverted();
4992   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
4993   emit_int8(0x5E);
4994   emit_int8((unsigned char)(0xC0 | encode));
4995 }
4996 
4997 void Assembler::vdivss(XMMRegister dst, XMMRegister nds, Address src) {
4998   assert(VM_Version::supports_avx(), "");
4999   InstructionMark im(this);
5000   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
5001   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
5002   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
5003   emit_int8(0x5E);
5004   emit_operand(dst, src);
5005 }
5006 
5007 void Assembler::vdivss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
5008   assert(VM_Version::supports_avx(), "");
5009   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
5010   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
5011   emit_int8(0x5E);
5012   emit_int8((unsigned char)(0xC0 | encode));
5013 }
5014 
5015 void Assembler::vfmadd231sd(XMMRegister dst, XMMRegister src1, XMMRegister src2) {
5016   assert(VM_Version::supports_fma(), "");
5017   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
5018   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5019   emit_int8((unsigned char)0xB9);
5020   emit_int8((unsigned char)(0xC0 | encode));
5021 }
5022 
5023 void Assembler::vfmadd231ss(XMMRegister dst, XMMRegister src1, XMMRegister src2) {
5024   assert(VM_Version::supports_fma(), "");
5025   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
5026   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5027   emit_int8((unsigned char)0xB9);
5028   emit_int8((unsigned char)(0xC0 | encode));
5029 }
5030 
5031 void Assembler::vmulsd(XMMRegister dst, XMMRegister nds, Address src) {
5032   assert(VM_Version::supports_avx(), "");
5033   InstructionMark im(this);
5034   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
5035   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
5036   attributes.set_rex_vex_w_reverted();
5037   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
5038   emit_int8(0x59);
5039   emit_operand(dst, src);
5040 }
5041 
5042 void Assembler::vmulsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
5043   assert(VM_Version::supports_avx(), "");
5044   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
5045   attributes.set_rex_vex_w_reverted();
5046   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
5047   emit_int8(0x59);
5048   emit_int8((unsigned char)(0xC0 | encode));
5049 }
5050 
5051 void Assembler::vmulss(XMMRegister dst, XMMRegister nds, Address src) {
5052   assert(VM_Version::supports_avx(), "");
5053   InstructionMark im(this);
5054   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
5055   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
5056   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
5057   emit_int8(0x59);
5058   emit_operand(dst, src);
5059 }
5060 
5061 void Assembler::vmulss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
5062   assert(VM_Version::supports_avx(), "");
5063   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
5064   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
5065   emit_int8(0x59);
5066   emit_int8((unsigned char)(0xC0 | encode));
5067 }
5068 
5069 void Assembler::vsubsd(XMMRegister dst, XMMRegister nds, Address src) {
5070   assert(VM_Version::supports_avx(), "");
5071   InstructionMark im(this);
5072   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
5073   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
5074   attributes.set_rex_vex_w_reverted();
5075   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
5076   emit_int8(0x5C);
5077   emit_operand(dst, src);
5078 }
5079 
5080 void Assembler::vsubsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
5081   assert(VM_Version::supports_avx(), "");
5082   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
5083   attributes.set_rex_vex_w_reverted();
5084   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
5085   emit_int8(0x5C);
5086   emit_int8((unsigned char)(0xC0 | encode));
5087 }
5088 
5089 void Assembler::vsubss(XMMRegister dst, XMMRegister nds, Address src) {
5090   assert(VM_Version::supports_avx(), "");
5091   InstructionMark im(this);
5092   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
5093   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
5094   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
5095   emit_int8(0x5C);
5096   emit_operand(dst, src);
5097 }
5098 
5099 void Assembler::vsubss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
5100   assert(VM_Version::supports_avx(), "");
5101   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
5102   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
5103   emit_int8(0x5C);
5104   emit_int8((unsigned char)(0xC0 | encode));
5105 }
5106 
5107 //====================VECTOR ARITHMETIC=====================================
5108 
5109 // Float-point vector arithmetic
5110 
5111 void Assembler::addpd(XMMRegister dst, XMMRegister src) {
5112   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5113   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5114   attributes.set_rex_vex_w_reverted();
5115   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5116   emit_int8(0x58);
5117   emit_int8((unsigned char)(0xC0 | encode));
5118 }
5119 
5120 void Assembler::addpd(XMMRegister dst, Address src) {
5121   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5122   InstructionMark im(this);
5123   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5124   attributes.set_rex_vex_w_reverted();
5125   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
5126   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5127   emit_int8(0x58);
5128   emit_operand(dst, src);
5129 }
5130 
5131 
5132 void Assembler::addps(XMMRegister dst, XMMRegister src) {
5133   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5134   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5135   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
5136   emit_int8(0x58);
5137   emit_int8((unsigned char)(0xC0 | encode));
5138 }
5139 
5140 void Assembler::vaddpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5141   assert(VM_Version::supports_avx(), "");
5142   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5143   attributes.set_rex_vex_w_reverted();
5144   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5145   emit_int8(0x58);
5146   emit_int8((unsigned char)(0xC0 | encode));
5147 }
5148 
5149 void Assembler::vaddps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5150   assert(VM_Version::supports_avx(), "");
5151   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5152   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
5153   emit_int8(0x58);
5154   emit_int8((unsigned char)(0xC0 | encode));
5155 }
5156 
5157 void Assembler::vaddpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5158   assert(VM_Version::supports_avx(), "");
5159   InstructionMark im(this);
5160   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5161   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
5162   attributes.set_rex_vex_w_reverted();
5163   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5164   emit_int8(0x58);
5165   emit_operand(dst, src);
5166 }
5167 
5168 void Assembler::vaddps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5169   assert(VM_Version::supports_avx(), "");
5170   InstructionMark im(this);
5171   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5172   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
5173   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
5174   emit_int8(0x58);
5175   emit_operand(dst, src);
5176 }
5177 
5178 void Assembler::subpd(XMMRegister dst, XMMRegister src) {
5179   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5180   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5181   attributes.set_rex_vex_w_reverted();
5182   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5183   emit_int8(0x5C);
5184   emit_int8((unsigned char)(0xC0 | encode));
5185 }
5186 
5187 void Assembler::subps(XMMRegister dst, XMMRegister src) {
5188   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5189   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5190   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
5191   emit_int8(0x5C);
5192   emit_int8((unsigned char)(0xC0 | encode));
5193 }
5194 
5195 void Assembler::vsubpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5196   assert(VM_Version::supports_avx(), "");
5197   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5198   attributes.set_rex_vex_w_reverted();
5199   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5200   emit_int8(0x5C);
5201   emit_int8((unsigned char)(0xC0 | encode));
5202 }
5203 
5204 void Assembler::vsubps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5205   assert(VM_Version::supports_avx(), "");
5206   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5207   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
5208   emit_int8(0x5C);
5209   emit_int8((unsigned char)(0xC0 | encode));
5210 }
5211 
5212 void Assembler::vsubpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5213   assert(VM_Version::supports_avx(), "");
5214   InstructionMark im(this);
5215   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5216   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
5217   attributes.set_rex_vex_w_reverted();
5218   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5219   emit_int8(0x5C);
5220   emit_operand(dst, src);
5221 }
5222 
5223 void Assembler::vsubps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5224   assert(VM_Version::supports_avx(), "");
5225   InstructionMark im(this);
5226   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5227   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
5228   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
5229   emit_int8(0x5C);
5230   emit_operand(dst, src);
5231 }
5232 
5233 void Assembler::mulpd(XMMRegister dst, XMMRegister src) {
5234   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5235   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5236   attributes.set_rex_vex_w_reverted();
5237   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5238   emit_int8(0x59);
5239   emit_int8((unsigned char)(0xC0 | encode));
5240 }
5241 
5242 void Assembler::mulpd(XMMRegister dst, Address src) {
5243   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5244   InstructionMark im(this);
5245   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5246   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
5247   attributes.set_rex_vex_w_reverted();
5248   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5249   emit_int8(0x59);
5250   emit_operand(dst, src);
5251 }
5252 
5253 void Assembler::mulps(XMMRegister dst, XMMRegister src) {
5254   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5255   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5256   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
5257   emit_int8(0x59);
5258   emit_int8((unsigned char)(0xC0 | encode));
5259 }
5260 
5261 void Assembler::vmulpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5262   assert(VM_Version::supports_avx(), "");
5263   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5264   attributes.set_rex_vex_w_reverted();
5265   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5266   emit_int8(0x59);
5267   emit_int8((unsigned char)(0xC0 | encode));
5268 }
5269 
5270 void Assembler::vmulps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5271   assert(VM_Version::supports_avx(), "");
5272   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5273   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
5274   emit_int8(0x59);
5275   emit_int8((unsigned char)(0xC0 | encode));
5276 }
5277 
5278 void Assembler::vmulpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5279   assert(VM_Version::supports_avx(), "");
5280   InstructionMark im(this);
5281   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5282   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
5283   attributes.set_rex_vex_w_reverted();
5284   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5285   emit_int8(0x59);
5286   emit_operand(dst, src);
5287 }
5288 
5289 void Assembler::vmulps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5290   assert(VM_Version::supports_avx(), "");
5291   InstructionMark im(this);
5292   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5293   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
5294   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
5295   emit_int8(0x59);
5296   emit_operand(dst, src);
5297 }
5298 
5299 void Assembler::vfmadd231pd(XMMRegister dst, XMMRegister src1, XMMRegister src2, int vector_len) {
5300   assert(VM_Version::supports_fma(), "");
5301   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5302   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5303   emit_int8((unsigned char)0xB8);
5304   emit_int8((unsigned char)(0xC0 | encode));
5305 }
5306 
5307 void Assembler::vfmadd231ps(XMMRegister dst, XMMRegister src1, XMMRegister src2, int vector_len) {
5308   assert(VM_Version::supports_fma(), "");
5309   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5310   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5311   emit_int8((unsigned char)0xB8);
5312   emit_int8((unsigned char)(0xC0 | encode));
5313 }
5314 
5315 void Assembler::vfmadd231pd(XMMRegister dst, XMMRegister src1, Address src2, int vector_len) {
5316   assert(VM_Version::supports_fma(), "");
5317   InstructionMark im(this);
5318   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5319   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
5320   vex_prefix(src2, src1->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5321   emit_int8((unsigned char)0xB8);
5322   emit_operand(dst, src2);
5323 }
5324 
5325 void Assembler::vfmadd231ps(XMMRegister dst, XMMRegister src1, Address src2, int vector_len) {
5326   assert(VM_Version::supports_fma(), "");
5327   InstructionMark im(this);
5328   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5329   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
5330   vex_prefix(src2, src1->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5331   emit_int8((unsigned char)0xB8);
5332   emit_operand(dst, src2);
5333 }
5334 
5335 void Assembler::divpd(XMMRegister dst, XMMRegister src) {
5336   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5337   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5338   attributes.set_rex_vex_w_reverted();
5339   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5340   emit_int8(0x5E);
5341   emit_int8((unsigned char)(0xC0 | encode));
5342 }
5343 
5344 void Assembler::divps(XMMRegister dst, XMMRegister src) {
5345   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5346   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5347   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
5348   emit_int8(0x5E);
5349   emit_int8((unsigned char)(0xC0 | encode));
5350 }
5351 
5352 void Assembler::vdivpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5353   assert(VM_Version::supports_avx(), "");
5354   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5355   attributes.set_rex_vex_w_reverted();
5356   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5357   emit_int8(0x5E);
5358   emit_int8((unsigned char)(0xC0 | encode));
5359 }
5360 
5361 void Assembler::vdivps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5362   assert(VM_Version::supports_avx(), "");
5363   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5364   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
5365   emit_int8(0x5E);
5366   emit_int8((unsigned char)(0xC0 | encode));
5367 }
5368 
5369 void Assembler::vdivpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5370   assert(VM_Version::supports_avx(), "");
5371   InstructionMark im(this);
5372   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5373   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
5374   attributes.set_rex_vex_w_reverted();
5375   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5376   emit_int8(0x5E);
5377   emit_operand(dst, src);
5378 }
5379 
5380 void Assembler::vdivps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5381   assert(VM_Version::supports_avx(), "");
5382   InstructionMark im(this);
5383   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5384   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
5385   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
5386   emit_int8(0x5E);
5387   emit_operand(dst, src);
5388 }
5389 
5390 void Assembler::vsqrtpd(XMMRegister dst, XMMRegister src, int vector_len) {
5391   assert(VM_Version::supports_avx(), "");
5392   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5393   attributes.set_rex_vex_w_reverted();
5394   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5395   emit_int8(0x51);
5396   emit_int8((unsigned char)(0xC0 | encode));
5397 }
5398 
5399 void Assembler::vsqrtpd(XMMRegister dst, Address src, int vector_len) {
5400   assert(VM_Version::supports_avx(), "");
5401   InstructionMark im(this);
5402   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5403   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
5404   attributes.set_rex_vex_w_reverted();
5405   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5406   emit_int8(0x51);
5407   emit_operand(dst, src);
5408 }
5409 
5410 void Assembler::vsqrtps(XMMRegister dst, XMMRegister src, int vector_len) {
5411   assert(VM_Version::supports_avx(), "");
5412   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5413   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
5414   emit_int8(0x51);
5415   emit_int8((unsigned char)(0xC0 | encode));
5416 }
5417 
5418 void Assembler::vsqrtps(XMMRegister dst, Address src, int vector_len) {
5419   assert(VM_Version::supports_avx(), "");
5420   InstructionMark im(this);
5421   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5422   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
5423   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
5424   emit_int8(0x51);
5425   emit_operand(dst, src);
5426 }
5427 
5428 void Assembler::andpd(XMMRegister dst, XMMRegister src) {
5429   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5430   InstructionAttr attributes(AVX_128bit, /* rex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
5431   attributes.set_rex_vex_w_reverted();
5432   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5433   emit_int8(0x54);
5434   emit_int8((unsigned char)(0xC0 | encode));
5435 }
5436 
5437 void Assembler::andps(XMMRegister dst, XMMRegister src) {
5438   NOT_LP64(assert(VM_Version::supports_sse(), ""));
5439   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
5440   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
5441   emit_int8(0x54);
5442   emit_int8((unsigned char)(0xC0 | encode));
5443 }
5444 
5445 void Assembler::andps(XMMRegister dst, Address src) {
5446   NOT_LP64(assert(VM_Version::supports_sse(), ""));
5447   InstructionMark im(this);
5448   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
5449   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
5450   simd_prefix(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
5451   emit_int8(0x54);
5452   emit_operand(dst, src);
5453 }
5454 
5455 void Assembler::andpd(XMMRegister dst, Address src) {
5456   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5457   InstructionMark im(this);
5458   InstructionAttr attributes(AVX_128bit, /* rex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
5459   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
5460   attributes.set_rex_vex_w_reverted();
5461   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5462   emit_int8(0x54);
5463   emit_operand(dst, src);
5464 }
5465 
5466 void Assembler::vandpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5467   assert(VM_Version::supports_avx(), "");
5468   InstructionAttr attributes(vector_len, /* vex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
5469   attributes.set_rex_vex_w_reverted();
5470   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5471   emit_int8(0x54);
5472   emit_int8((unsigned char)(0xC0 | encode));
5473 }
5474 
5475 void Assembler::vandps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5476   assert(VM_Version::supports_avx(), "");
5477   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
5478   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
5479   emit_int8(0x54);
5480   emit_int8((unsigned char)(0xC0 | encode));
5481 }
5482 
5483 void Assembler::vandpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5484   assert(VM_Version::supports_avx(), "");
5485   InstructionMark im(this);
5486   InstructionAttr attributes(vector_len, /* vex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
5487   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
5488   attributes.set_rex_vex_w_reverted();
5489   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5490   emit_int8(0x54);
5491   emit_operand(dst, src);
5492 }
5493 
5494 void Assembler::vandps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5495   assert(VM_Version::supports_avx(), "");
5496   InstructionMark im(this);
5497   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
5498   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
5499   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
5500   emit_int8(0x54);
5501   emit_operand(dst, src);
5502 }
5503 
5504 void Assembler::unpckhpd(XMMRegister dst, XMMRegister src) {
5505   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5506   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5507   attributes.set_rex_vex_w_reverted();
5508   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5509   emit_int8(0x15);
5510   emit_int8((unsigned char)(0xC0 | encode));
5511 }
5512 
5513 void Assembler::unpcklpd(XMMRegister dst, XMMRegister src) {
5514   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5515   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5516   attributes.set_rex_vex_w_reverted();
5517   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5518   emit_int8(0x14);
5519   emit_int8((unsigned char)(0xC0 | encode));
5520 }
5521 
5522 void Assembler::xorpd(XMMRegister dst, XMMRegister src) {
5523   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5524   InstructionAttr attributes(AVX_128bit, /* rex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
5525   attributes.set_rex_vex_w_reverted();
5526   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5527   emit_int8(0x57);
5528   emit_int8((unsigned char)(0xC0 | encode));
5529 }
5530 
5531 void Assembler::xorps(XMMRegister dst, XMMRegister src) {
5532   NOT_LP64(assert(VM_Version::supports_sse(), ""));
5533   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
5534   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
5535   emit_int8(0x57);
5536   emit_int8((unsigned char)(0xC0 | encode));
5537 }
5538 
5539 void Assembler::xorpd(XMMRegister dst, Address src) {
5540   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5541   InstructionMark im(this);
5542   InstructionAttr attributes(AVX_128bit, /* rex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
5543   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
5544   attributes.set_rex_vex_w_reverted();
5545   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5546   emit_int8(0x57);
5547   emit_operand(dst, src);
5548 }
5549 
5550 void Assembler::xorps(XMMRegister dst, Address src) {
5551   NOT_LP64(assert(VM_Version::supports_sse(), ""));
5552   InstructionMark im(this);
5553   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
5554   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
5555   simd_prefix(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
5556   emit_int8(0x57);
5557   emit_operand(dst, src);
5558 }
5559 
5560 void Assembler::vxorpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5561   assert(VM_Version::supports_avx(), "");
5562   InstructionAttr attributes(vector_len, /* vex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
5563   attributes.set_rex_vex_w_reverted();
5564   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5565   emit_int8(0x57);
5566   emit_int8((unsigned char)(0xC0 | encode));
5567 }
5568 
5569 void Assembler::vxorps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5570   assert(VM_Version::supports_avx(), "");
5571   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
5572   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
5573   emit_int8(0x57);
5574   emit_int8((unsigned char)(0xC0 | encode));
5575 }
5576 
5577 void Assembler::vxorpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5578   assert(VM_Version::supports_avx(), "");
5579   InstructionMark im(this);
5580   InstructionAttr attributes(vector_len, /* vex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
5581   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
5582   attributes.set_rex_vex_w_reverted();
5583   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5584   emit_int8(0x57);
5585   emit_operand(dst, src);
5586 }
5587 
5588 void Assembler::vxorps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5589   assert(VM_Version::supports_avx(), "");
5590   InstructionMark im(this);
5591   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
5592   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
5593   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
5594   emit_int8(0x57);
5595   emit_operand(dst, src);
5596 }
5597 
5598 // Integer vector arithmetic
5599 void Assembler::vphaddw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5600   assert(VM_Version::supports_avx() && (vector_len == 0) ||
5601          VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
5602   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
5603   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5604   emit_int8(0x01);
5605   emit_int8((unsigned char)(0xC0 | encode));
5606 }
5607 
5608 void Assembler::vphaddd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5609   assert(VM_Version::supports_avx() && (vector_len == 0) ||
5610          VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
5611   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
5612   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5613   emit_int8(0x02);
5614   emit_int8((unsigned char)(0xC0 | encode));
5615 }
5616 
5617 void Assembler::paddb(XMMRegister dst, XMMRegister src) {
5618   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5619   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5620   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5621   emit_int8((unsigned char)0xFC);
5622   emit_int8((unsigned char)(0xC0 | encode));
5623 }
5624 
5625 void Assembler::paddw(XMMRegister dst, XMMRegister src) {
5626   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5627   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5628   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5629   emit_int8((unsigned char)0xFD);
5630   emit_int8((unsigned char)(0xC0 | encode));
5631 }
5632 
5633 void Assembler::paddd(XMMRegister dst, XMMRegister src) {
5634   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5635   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5636   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5637   emit_int8((unsigned char)0xFE);
5638   emit_int8((unsigned char)(0xC0 | encode));
5639 }
5640 
5641 void Assembler::paddd(XMMRegister dst, Address src) {
5642   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5643   InstructionMark im(this);
5644   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5645   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5646   emit_int8((unsigned char)0xFE);
5647   emit_operand(dst, src);
5648 }
5649 
5650 void Assembler::paddq(XMMRegister dst, XMMRegister src) {
5651   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5652   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5653   attributes.set_rex_vex_w_reverted();
5654   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5655   emit_int8((unsigned char)0xD4);
5656   emit_int8((unsigned char)(0xC0 | encode));
5657 }
5658 
5659 void Assembler::phaddw(XMMRegister dst, XMMRegister src) {
5660   assert(VM_Version::supports_sse3(), "");
5661   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
5662   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5663   emit_int8(0x01);
5664   emit_int8((unsigned char)(0xC0 | encode));
5665 }
5666 
5667 void Assembler::phaddd(XMMRegister dst, XMMRegister src) {
5668   assert(VM_Version::supports_sse3(), "");
5669   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
5670   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5671   emit_int8(0x02);
5672   emit_int8((unsigned char)(0xC0 | encode));
5673 }
5674 
5675 void Assembler::vpaddb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5676   assert(UseAVX > 0, "requires some form of AVX");
5677   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5678   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5679   emit_int8((unsigned char)0xFC);
5680   emit_int8((unsigned char)(0xC0 | encode));
5681 }
5682 
5683 void Assembler::vpaddw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5684   assert(UseAVX > 0, "requires some form of AVX");
5685   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5686   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5687   emit_int8((unsigned char)0xFD);
5688   emit_int8((unsigned char)(0xC0 | encode));
5689 }
5690 
5691 void Assembler::vpaddd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5692   assert(UseAVX > 0, "requires some form of AVX");
5693   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5694   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5695   emit_int8((unsigned char)0xFE);
5696   emit_int8((unsigned char)(0xC0 | encode));
5697 }
5698 
5699 void Assembler::vpaddq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5700   assert(UseAVX > 0, "requires some form of AVX");
5701   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5702   attributes.set_rex_vex_w_reverted();
5703   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5704   emit_int8((unsigned char)0xD4);
5705   emit_int8((unsigned char)(0xC0 | encode));
5706 }
5707 
5708 void Assembler::vpaddb(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5709   assert(UseAVX > 0, "requires some form of AVX");
5710   InstructionMark im(this);
5711   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5712   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
5713   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5714   emit_int8((unsigned char)0xFC);
5715   emit_operand(dst, src);
5716 }
5717 
5718 void Assembler::vpaddw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5719   assert(UseAVX > 0, "requires some form of AVX");
5720   InstructionMark im(this);
5721   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5722   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
5723   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5724   emit_int8((unsigned char)0xFD);
5725   emit_operand(dst, src);
5726 }
5727 
5728 void Assembler::vpaddd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5729   assert(UseAVX > 0, "requires some form of AVX");
5730   InstructionMark im(this);
5731   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5732   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
5733   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5734   emit_int8((unsigned char)0xFE);
5735   emit_operand(dst, src);
5736 }
5737 
5738 void Assembler::vpaddq(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5739   assert(UseAVX > 0, "requires some form of AVX");
5740   InstructionMark im(this);
5741   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5742   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
5743   attributes.set_rex_vex_w_reverted();
5744   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5745   emit_int8((unsigned char)0xD4);
5746   emit_operand(dst, src);
5747 }
5748 
5749 void Assembler::psubb(XMMRegister dst, XMMRegister src) {
5750   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5751   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5752   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5753   emit_int8((unsigned char)0xF8);
5754   emit_int8((unsigned char)(0xC0 | encode));
5755 }
5756 
5757 void Assembler::psubw(XMMRegister dst, XMMRegister src) {
5758   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5759   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5760   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5761   emit_int8((unsigned char)0xF9);
5762   emit_int8((unsigned char)(0xC0 | encode));
5763 }
5764 
5765 void Assembler::psubd(XMMRegister dst, XMMRegister src) {
5766   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5767   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5768   emit_int8((unsigned char)0xFA);
5769   emit_int8((unsigned char)(0xC0 | encode));
5770 }
5771 
5772 void Assembler::psubq(XMMRegister dst, XMMRegister src) {
5773   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5774   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5775   attributes.set_rex_vex_w_reverted();
5776   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5777   emit_int8((unsigned char)0xFB);
5778   emit_int8((unsigned char)(0xC0 | encode));
5779 }
5780 
5781 void Assembler::vpsubb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5782   assert(UseAVX > 0, "requires some form of AVX");
5783   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5784   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5785   emit_int8((unsigned char)0xF8);
5786   emit_int8((unsigned char)(0xC0 | encode));
5787 }
5788 
5789 void Assembler::vpsubw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5790   assert(UseAVX > 0, "requires some form of AVX");
5791   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5792   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5793   emit_int8((unsigned char)0xF9);
5794   emit_int8((unsigned char)(0xC0 | encode));
5795 }
5796 
5797 void Assembler::vpsubd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5798   assert(UseAVX > 0, "requires some form of AVX");
5799   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5800   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5801   emit_int8((unsigned char)0xFA);
5802   emit_int8((unsigned char)(0xC0 | encode));
5803 }
5804 
5805 void Assembler::vpsubq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5806   assert(UseAVX > 0, "requires some form of AVX");
5807   InstructionAttr attributes(vector_len, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5808   attributes.set_rex_vex_w_reverted();
5809   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5810   emit_int8((unsigned char)0xFB);
5811   emit_int8((unsigned char)(0xC0 | encode));
5812 }
5813 
5814 void Assembler::vpsubb(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5815   assert(UseAVX > 0, "requires some form of AVX");
5816   InstructionMark im(this);
5817   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5818   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
5819   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5820   emit_int8((unsigned char)0xF8);
5821   emit_operand(dst, src);
5822 }
5823 
5824 void Assembler::vpsubw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5825   assert(UseAVX > 0, "requires some form of AVX");
5826   InstructionMark im(this);
5827   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5828   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
5829   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5830   emit_int8((unsigned char)0xF9);
5831   emit_operand(dst, src);
5832 }
5833 
5834 void Assembler::vpsubd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5835   assert(UseAVX > 0, "requires some form of AVX");
5836   InstructionMark im(this);
5837   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5838   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
5839   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5840   emit_int8((unsigned char)0xFA);
5841   emit_operand(dst, src);
5842 }
5843 
5844 void Assembler::vpsubq(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5845   assert(UseAVX > 0, "requires some form of AVX");
5846   InstructionMark im(this);
5847   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5848   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
5849   attributes.set_rex_vex_w_reverted();
5850   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5851   emit_int8((unsigned char)0xFB);
5852   emit_operand(dst, src);
5853 }
5854 
5855 void Assembler::pmullw(XMMRegister dst, XMMRegister src) {
5856   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5857   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5858   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5859   emit_int8((unsigned char)0xD5);
5860   emit_int8((unsigned char)(0xC0 | encode));
5861 }
5862 
5863 void Assembler::pmulld(XMMRegister dst, XMMRegister src) {
5864   assert(VM_Version::supports_sse4_1(), "");
5865   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5866   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5867   emit_int8(0x40);
5868   emit_int8((unsigned char)(0xC0 | encode));
5869 }
5870 
5871 void Assembler::vpmullw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5872   assert(UseAVX > 0, "requires some form of AVX");
5873   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5874   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5875   emit_int8((unsigned char)0xD5);
5876   emit_int8((unsigned char)(0xC0 | encode));
5877 }
5878 
5879 void Assembler::vpmulld(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5880   assert(UseAVX > 0, "requires some form of AVX");
5881   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5882   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5883   emit_int8(0x40);
5884   emit_int8((unsigned char)(0xC0 | encode));
5885 }
5886 
5887 void Assembler::vpmullq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5888   assert(UseAVX > 2, "requires some form of EVEX");
5889   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
5890   attributes.set_is_evex_instruction();
5891   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5892   emit_int8(0x40);
5893   emit_int8((unsigned char)(0xC0 | encode));
5894 }
5895 
5896 void Assembler::vpmullw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5897   assert(UseAVX > 0, "requires some form of AVX");
5898   InstructionMark im(this);
5899   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5900   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
5901   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5902   emit_int8((unsigned char)0xD5);
5903   emit_operand(dst, src);
5904 }
5905 
5906 void Assembler::vpmulld(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5907   assert(UseAVX > 0, "requires some form of AVX");
5908   InstructionMark im(this);
5909   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5910   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
5911   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5912   emit_int8(0x40);
5913   emit_operand(dst, src);
5914 }
5915 
5916 void Assembler::vpmullq(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5917   assert(UseAVX > 2, "requires some form of EVEX");
5918   InstructionMark im(this);
5919   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
5920   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
5921   attributes.set_is_evex_instruction();
5922   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5923   emit_int8(0x40);
5924   emit_operand(dst, src);
5925 }
5926 
5927 // Shift packed integers left by specified number of bits.
5928 void Assembler::psllw(XMMRegister dst, int shift) {
5929   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5930   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5931   // XMM6 is for /6 encoding: 66 0F 71 /6 ib
5932   int encode = simd_prefix_and_encode(xmm6, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5933   emit_int8(0x71);
5934   emit_int8((unsigned char)(0xC0 | encode));
5935   emit_int8(shift & 0xFF);
5936 }
5937 
5938 void Assembler::pslld(XMMRegister dst, int shift) {
5939   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5940   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5941   // XMM6 is for /6 encoding: 66 0F 72 /6 ib
5942   int encode = simd_prefix_and_encode(xmm6, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5943   emit_int8(0x72);
5944   emit_int8((unsigned char)(0xC0 | encode));
5945   emit_int8(shift & 0xFF);
5946 }
5947 
5948 void Assembler::psllq(XMMRegister dst, int shift) {
5949   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5950   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5951   // XMM6 is for /6 encoding: 66 0F 73 /6 ib
5952   int encode = simd_prefix_and_encode(xmm6, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5953   emit_int8(0x73);
5954   emit_int8((unsigned char)(0xC0 | encode));
5955   emit_int8(shift & 0xFF);
5956 }
5957 
5958 void Assembler::psllw(XMMRegister dst, XMMRegister shift) {
5959   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5960   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5961   int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5962   emit_int8((unsigned char)0xF1);
5963   emit_int8((unsigned char)(0xC0 | encode));
5964 }
5965 
5966 void Assembler::pslld(XMMRegister dst, XMMRegister shift) {
5967   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5968   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5969   int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5970   emit_int8((unsigned char)0xF2);
5971   emit_int8((unsigned char)(0xC0 | encode));
5972 }
5973 
5974 void Assembler::psllq(XMMRegister dst, XMMRegister shift) {
5975   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5976   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5977   attributes.set_rex_vex_w_reverted();
5978   int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5979   emit_int8((unsigned char)0xF3);
5980   emit_int8((unsigned char)(0xC0 | encode));
5981 }
5982 
5983 void Assembler::vpsllw(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
5984   assert(UseAVX > 0, "requires some form of AVX");
5985   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5986   // XMM6 is for /6 encoding: 66 0F 71 /6 ib
5987   int encode = vex_prefix_and_encode(xmm6->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5988   emit_int8(0x71);
5989   emit_int8((unsigned char)(0xC0 | encode));
5990   emit_int8(shift & 0xFF);
5991 }
5992 
5993 void Assembler::vpslld(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
5994   assert(UseAVX > 0, "requires some form of AVX");
5995   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5996   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5997   // XMM6 is for /6 encoding: 66 0F 72 /6 ib
5998   int encode = vex_prefix_and_encode(xmm6->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5999   emit_int8(0x72);
6000   emit_int8((unsigned char)(0xC0 | encode));
6001   emit_int8(shift & 0xFF);
6002 }
6003 
6004 void Assembler::vpsllq(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
6005   assert(UseAVX > 0, "requires some form of AVX");
6006   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6007   attributes.set_rex_vex_w_reverted();
6008   // XMM6 is for /6 encoding: 66 0F 73 /6 ib
6009   int encode = vex_prefix_and_encode(xmm6->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6010   emit_int8(0x73);
6011   emit_int8((unsigned char)(0xC0 | encode));
6012   emit_int8(shift & 0xFF);
6013 }
6014 
6015 void Assembler::vpsllw(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
6016   assert(UseAVX > 0, "requires some form of AVX");
6017   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
6018   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6019   emit_int8((unsigned char)0xF1);
6020   emit_int8((unsigned char)(0xC0 | encode));
6021 }
6022 
6023 void Assembler::vpslld(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
6024   assert(UseAVX > 0, "requires some form of AVX");
6025   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6026   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6027   emit_int8((unsigned char)0xF2);
6028   emit_int8((unsigned char)(0xC0 | encode));
6029 }
6030 
6031 void Assembler::vpsllq(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
6032   assert(UseAVX > 0, "requires some form of AVX");
6033   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6034   attributes.set_rex_vex_w_reverted();
6035   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6036   emit_int8((unsigned char)0xF3);
6037   emit_int8((unsigned char)(0xC0 | encode));
6038 }
6039 
6040 // Shift packed integers logically right by specified number of bits.
6041 void Assembler::psrlw(XMMRegister dst, int shift) {
6042   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
6043   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
6044   // XMM2 is for /2 encoding: 66 0F 71 /2 ib
6045   int encode = simd_prefix_and_encode(xmm2, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6046   emit_int8(0x71);
6047   emit_int8((unsigned char)(0xC0 | encode));
6048   emit_int8(shift & 0xFF);
6049 }
6050 
6051 void Assembler::psrld(XMMRegister dst, int shift) {
6052   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
6053   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6054   // XMM2 is for /2 encoding: 66 0F 72 /2 ib
6055   int encode = simd_prefix_and_encode(xmm2, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6056   emit_int8(0x72);
6057   emit_int8((unsigned char)(0xC0 | encode));
6058   emit_int8(shift & 0xFF);
6059 }
6060 
6061 void Assembler::psrlq(XMMRegister dst, int shift) {
6062   // Do not confuse it with psrldq SSE2 instruction which
6063   // shifts 128 bit value in xmm register by number of bytes.
6064   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
6065   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6066   attributes.set_rex_vex_w_reverted();
6067   // XMM2 is for /2 encoding: 66 0F 73 /2 ib
6068   int encode = simd_prefix_and_encode(xmm2, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6069   emit_int8(0x73);
6070   emit_int8((unsigned char)(0xC0 | encode));
6071   emit_int8(shift & 0xFF);
6072 }
6073 
6074 void Assembler::psrlw(XMMRegister dst, XMMRegister shift) {
6075   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
6076   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
6077   int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6078   emit_int8((unsigned char)0xD1);
6079   emit_int8((unsigned char)(0xC0 | encode));
6080 }
6081 
6082 void Assembler::psrld(XMMRegister dst, XMMRegister shift) {
6083   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
6084   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6085   int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6086   emit_int8((unsigned char)0xD2);
6087   emit_int8((unsigned char)(0xC0 | encode));
6088 }
6089 
6090 void Assembler::psrlq(XMMRegister dst, XMMRegister shift) {
6091   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
6092   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6093   attributes.set_rex_vex_w_reverted();
6094   int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6095   emit_int8((unsigned char)0xD3);
6096   emit_int8((unsigned char)(0xC0 | encode));
6097 }
6098 
6099 void Assembler::vpsrlw(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
6100   assert(UseAVX > 0, "requires some form of AVX");
6101   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
6102   // XMM2 is for /2 encoding: 66 0F 71 /2 ib
6103   int encode = vex_prefix_and_encode(xmm2->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6104   emit_int8(0x71);
6105   emit_int8((unsigned char)(0xC0 | encode));
6106   emit_int8(shift & 0xFF);
6107 }
6108 
6109 void Assembler::vpsrld(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
6110   assert(UseAVX > 0, "requires some form of AVX");
6111   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6112   // XMM2 is for /2 encoding: 66 0F 72 /2 ib
6113   int encode = vex_prefix_and_encode(xmm2->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6114   emit_int8(0x72);
6115   emit_int8((unsigned char)(0xC0 | encode));
6116   emit_int8(shift & 0xFF);
6117 }
6118 
6119 void Assembler::vpsrlq(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
6120   assert(UseAVX > 0, "requires some form of AVX");
6121   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6122   attributes.set_rex_vex_w_reverted();
6123   // XMM2 is for /2 encoding: 66 0F 73 /2 ib
6124   int encode = vex_prefix_and_encode(xmm2->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6125   emit_int8(0x73);
6126   emit_int8((unsigned char)(0xC0 | encode));
6127   emit_int8(shift & 0xFF);
6128 }
6129 
6130 void Assembler::vpsrlw(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
6131   assert(UseAVX > 0, "requires some form of AVX");
6132   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
6133   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6134   emit_int8((unsigned char)0xD1);
6135   emit_int8((unsigned char)(0xC0 | encode));
6136 }
6137 
6138 void Assembler::vpsrld(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
6139   assert(UseAVX > 0, "requires some form of AVX");
6140   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6141   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6142   emit_int8((unsigned char)0xD2);
6143   emit_int8((unsigned char)(0xC0 | encode));
6144 }
6145 
6146 void Assembler::vpsrlq(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
6147   assert(UseAVX > 0, "requires some form of AVX");
6148   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6149   attributes.set_rex_vex_w_reverted();
6150   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6151   emit_int8((unsigned char)0xD3);
6152   emit_int8((unsigned char)(0xC0 | encode));
6153 }
6154 
6155 void Assembler::evpsrlvw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
6156   assert(VM_Version::supports_avx512bw(), "");
6157   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6158   attributes.set_is_evex_instruction();
6159   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6160   emit_int8(0x10);
6161   emit_int8((unsigned char)(0xC0 | encode));
6162 }
6163 
6164 void Assembler::evpsllvw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
6165   assert(VM_Version::supports_avx512bw(), "");
6166   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6167   attributes.set_is_evex_instruction();
6168   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6169   emit_int8(0x12);
6170   emit_int8((unsigned char)(0xC0 | encode));
6171 }
6172 
6173 // Shift packed integers arithmetically right by specified number of bits.
6174 void Assembler::psraw(XMMRegister dst, int shift) {
6175   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
6176   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
6177   // XMM4 is for /4 encoding: 66 0F 71 /4 ib
6178   int encode = simd_prefix_and_encode(xmm4, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6179   emit_int8(0x71);
6180   emit_int8((unsigned char)(0xC0 | encode));
6181   emit_int8(shift & 0xFF);
6182 }
6183 
6184 void Assembler::psrad(XMMRegister dst, int shift) {
6185   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
6186   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6187   // XMM4 is for /4 encoding: 66 0F 72 /4 ib
6188   int encode = simd_prefix_and_encode(xmm4, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6189   emit_int8(0x72);
6190   emit_int8((unsigned char)(0xC0 | encode));
6191   emit_int8(shift & 0xFF);
6192 }
6193 
6194 void Assembler::psraw(XMMRegister dst, XMMRegister shift) {
6195   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
6196   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
6197   int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6198   emit_int8((unsigned char)0xE1);
6199   emit_int8((unsigned char)(0xC0 | encode));
6200 }
6201 
6202 void Assembler::psrad(XMMRegister dst, XMMRegister shift) {
6203   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
6204   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6205   int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6206   emit_int8((unsigned char)0xE2);
6207   emit_int8((unsigned char)(0xC0 | encode));
6208 }
6209 
6210 void Assembler::vpsraw(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
6211   assert(UseAVX > 0, "requires some form of AVX");
6212   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
6213   // XMM4 is for /4 encoding: 66 0F 71 /4 ib
6214   int encode = vex_prefix_and_encode(xmm4->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6215   emit_int8(0x71);
6216   emit_int8((unsigned char)(0xC0 | encode));
6217   emit_int8(shift & 0xFF);
6218 }
6219 
6220 void Assembler::vpsrad(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
6221   assert(UseAVX > 0, "requires some form of AVX");
6222   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6223   // XMM4 is for /4 encoding: 66 0F 71 /4 ib
6224   int encode = vex_prefix_and_encode(xmm4->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6225   emit_int8(0x72);
6226   emit_int8((unsigned char)(0xC0 | encode));
6227   emit_int8(shift & 0xFF);
6228 }
6229 
6230 void Assembler::vpsraw(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
6231   assert(UseAVX > 0, "requires some form of AVX");
6232   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
6233   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6234   emit_int8((unsigned char)0xE1);
6235   emit_int8((unsigned char)(0xC0 | encode));
6236 }
6237 
6238 void Assembler::vpsrad(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
6239   assert(UseAVX > 0, "requires some form of AVX");
6240   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6241   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6242   emit_int8((unsigned char)0xE2);
6243   emit_int8((unsigned char)(0xC0 | encode));
6244 }
6245 
6246 
6247 // logical operations packed integers
6248 void Assembler::pand(XMMRegister dst, XMMRegister src) {
6249   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
6250   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6251   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6252   emit_int8((unsigned char)0xDB);
6253   emit_int8((unsigned char)(0xC0 | encode));
6254 }
6255 
6256 void Assembler::vpand(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
6257   assert(UseAVX > 0, "requires some form of AVX");
6258   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6259   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6260   emit_int8((unsigned char)0xDB);
6261   emit_int8((unsigned char)(0xC0 | encode));
6262 }
6263 
6264 void Assembler::vpand(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
6265   assert(UseAVX > 0, "requires some form of AVX");
6266   InstructionMark im(this);
6267   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6268   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
6269   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6270   emit_int8((unsigned char)0xDB);
6271   emit_operand(dst, src);
6272 }
6273 
6274 void Assembler::vpandq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
6275   assert(VM_Version::supports_evex(), "");
6276   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6277   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6278   emit_int8((unsigned char)0xDB);
6279   emit_int8((unsigned char)(0xC0 | encode));
6280 }
6281 
6282 
6283 void Assembler::pandn(XMMRegister dst, XMMRegister src) {
6284   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
6285   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6286   attributes.set_rex_vex_w_reverted();
6287   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6288   emit_int8((unsigned char)0xDF);
6289   emit_int8((unsigned char)(0xC0 | encode));
6290 }
6291 
6292 void Assembler::por(XMMRegister dst, XMMRegister src) {
6293   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
6294   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6295   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6296   emit_int8((unsigned char)0xEB);
6297   emit_int8((unsigned char)(0xC0 | encode));
6298 }
6299 
6300 void Assembler::vpor(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
6301   assert(UseAVX > 0, "requires some form of AVX");
6302   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6303   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6304   emit_int8((unsigned char)0xEB);
6305   emit_int8((unsigned char)(0xC0 | encode));
6306 }
6307 
6308 void Assembler::vpor(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
6309   assert(UseAVX > 0, "requires some form of AVX");
6310   InstructionMark im(this);
6311   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6312   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
6313   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6314   emit_int8((unsigned char)0xEB);
6315   emit_operand(dst, src);
6316 }
6317 
6318 void Assembler::vporq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
6319   assert(VM_Version::supports_evex(), "");
6320   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6321   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6322   emit_int8((unsigned char)0xEB);
6323   emit_int8((unsigned char)(0xC0 | encode));
6324 }
6325 
6326 
6327 void Assembler::pxor(XMMRegister dst, XMMRegister src) {
6328   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
6329   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6330   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6331   emit_int8((unsigned char)0xEF);
6332   emit_int8((unsigned char)(0xC0 | encode));
6333 }
6334 
6335 void Assembler::vpxor(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
6336   assert(UseAVX > 0, "requires some form of AVX");
6337   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6338   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6339   emit_int8((unsigned char)0xEF);
6340   emit_int8((unsigned char)(0xC0 | encode));
6341 }
6342 
6343 void Assembler::vpxor(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
6344   assert(UseAVX > 0, "requires some form of AVX");
6345   InstructionMark im(this);
6346   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6347   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
6348   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6349   emit_int8((unsigned char)0xEF);
6350   emit_operand(dst, src);
6351 }
6352 
6353 void Assembler::evpxorq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
6354   assert(VM_Version::supports_evex(), "requires EVEX support");
6355   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6356   attributes.set_is_evex_instruction();
6357   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6358   emit_int8((unsigned char)0xEF);
6359   emit_int8((unsigned char)(0xC0 | encode));
6360 }
6361 
6362 void Assembler::evpxorq(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
6363   assert(VM_Version::supports_evex(), "requires EVEX support");
6364   assert(dst != xnoreg, "sanity");
6365   InstructionMark im(this);
6366   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6367   attributes.set_is_evex_instruction();
6368   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
6369   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6370   emit_int8((unsigned char)0xEF);
6371   emit_operand(dst, src);
6372 }
6373 
6374 
6375 // vinserti forms
6376 
6377 void Assembler::vinserti128(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) {
6378   assert(VM_Version::supports_avx2(), "");
6379   assert(imm8 <= 0x01, "imm8: %u", imm8);
6380   int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_256bit;
6381   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6382   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6383   emit_int8(0x38);
6384   emit_int8((unsigned char)(0xC0 | encode));
6385   // 0x00 - insert into lower 128 bits
6386   // 0x01 - insert into upper 128 bits
6387   emit_int8(imm8 & 0x01);
6388 }
6389 
6390 void Assembler::vinserti128(XMMRegister dst, XMMRegister nds, Address src, uint8_t imm8) {
6391   assert(VM_Version::supports_avx2(), "");
6392   assert(dst != xnoreg, "sanity");
6393   assert(imm8 <= 0x01, "imm8: %u", imm8);
6394   int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_256bit;
6395   InstructionMark im(this);
6396   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6397   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
6398   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6399   emit_int8(0x38);
6400   emit_operand(dst, src);
6401   // 0x00 - insert into lower 128 bits
6402   // 0x01 - insert into upper 128 bits
6403   emit_int8(imm8 & 0x01);
6404 }
6405 
6406 void Assembler::vinserti32x4(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) {
6407   assert(VM_Version::supports_evex(), "");
6408   assert(imm8 <= 0x03, "imm8: %u", imm8);
6409   InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6410   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6411   emit_int8(0x38);
6412   emit_int8((unsigned char)(0xC0 | encode));
6413   // 0x00 - insert into q0 128 bits (0..127)
6414   // 0x01 - insert into q1 128 bits (128..255)
6415   // 0x02 - insert into q2 128 bits (256..383)
6416   // 0x03 - insert into q3 128 bits (384..511)
6417   emit_int8(imm8 & 0x03);
6418 }
6419 
6420 void Assembler::vinserti32x4(XMMRegister dst, XMMRegister nds, Address src, uint8_t imm8) {
6421   assert(VM_Version::supports_avx(), "");
6422   assert(dst != xnoreg, "sanity");
6423   assert(imm8 <= 0x03, "imm8: %u", imm8);
6424   int vector_len = VM_Version::supports_evex() ? AVX_512bit : AVX_256bit;
6425   InstructionMark im(this);
6426   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6427   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
6428   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6429   emit_int8(0x18);
6430   emit_operand(dst, src);
6431   // 0x00 - insert into q0 128 bits (0..127)
6432   // 0x01 - insert into q1 128 bits (128..255)
6433   // 0x02 - insert into q2 128 bits (256..383)
6434   // 0x03 - insert into q3 128 bits (384..511)
6435   emit_int8(imm8 & 0x03);
6436 }
6437 
6438 void Assembler::vinserti64x4(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) {
6439   assert(VM_Version::supports_evex(), "");
6440   assert(imm8 <= 0x01, "imm8: %u", imm8);
6441   InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6442   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6443   emit_int8(0x38);
6444   emit_int8((unsigned char)(0xC0 | encode));
6445   // 0x00 - insert into lower 256 bits
6446   // 0x01 - insert into upper 256 bits
6447   emit_int8(imm8 & 0x01);
6448 }
6449 
6450 
6451 // vinsertf forms
6452 
6453 void Assembler::vinsertf128(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) {
6454   assert(VM_Version::supports_avx(), "");
6455   assert(imm8 <= 0x01, "imm8: %u", imm8);
6456   int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_256bit;
6457   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6458   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6459   emit_int8(0x18);
6460   emit_int8((unsigned char)(0xC0 | encode));
6461   // 0x00 - insert into lower 128 bits
6462   // 0x01 - insert into upper 128 bits
6463   emit_int8(imm8 & 0x01);
6464 }
6465 
6466 void Assembler::vinsertf128(XMMRegister dst, XMMRegister nds, Address src, uint8_t imm8) {
6467   assert(VM_Version::supports_avx(), "");
6468   assert(dst != xnoreg, "sanity");
6469   assert(imm8 <= 0x01, "imm8: %u", imm8);
6470   int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_256bit;
6471   InstructionMark im(this);
6472   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6473   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
6474   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6475   emit_int8(0x18);
6476   emit_operand(dst, src);
6477   // 0x00 - insert into lower 128 bits
6478   // 0x01 - insert into upper 128 bits
6479   emit_int8(imm8 & 0x01);
6480 }
6481 
6482 void Assembler::vinsertf32x4(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) {
6483   assert(VM_Version::supports_evex(), "");
6484   assert(imm8 <= 0x03, "imm8: %u", imm8);
6485   InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6486   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6487   emit_int8(0x18);
6488   emit_int8((unsigned char)(0xC0 | encode));
6489   // 0x00 - insert into q0 128 bits (0..127)
6490   // 0x01 - insert into q1 128 bits (128..255)
6491   // 0x02 - insert into q2 128 bits (256..383)
6492   // 0x03 - insert into q3 128 bits (384..511)
6493   emit_int8(imm8 & 0x03);
6494 }
6495 
6496 void Assembler::vinsertf32x4(XMMRegister dst, XMMRegister nds, Address src, uint8_t imm8) {
6497   assert(VM_Version::supports_avx(), "");
6498   assert(dst != xnoreg, "sanity");
6499   assert(imm8 <= 0x03, "imm8: %u", imm8);
6500   int vector_len = VM_Version::supports_evex() ? AVX_512bit : AVX_256bit;
6501   InstructionMark im(this);
6502   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6503   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
6504   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6505   emit_int8(0x18);
6506   emit_operand(dst, src);
6507   // 0x00 - insert into q0 128 bits (0..127)
6508   // 0x01 - insert into q1 128 bits (128..255)
6509   // 0x02 - insert into q2 128 bits (256..383)
6510   // 0x03 - insert into q3 128 bits (384..511)
6511   emit_int8(imm8 & 0x03);
6512 }
6513 
6514 void Assembler::vinsertf64x4(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) {
6515   assert(VM_Version::supports_evex(), "");
6516   assert(imm8 <= 0x01, "imm8: %u", imm8);
6517   InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6518   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6519   emit_int8(0x1A);
6520   emit_int8((unsigned char)(0xC0 | encode));
6521   // 0x00 - insert into lower 256 bits
6522   // 0x01 - insert into upper 256 bits
6523   emit_int8(imm8 & 0x01);
6524 }
6525 
6526 void Assembler::vinsertf64x4(XMMRegister dst, XMMRegister nds, Address src, uint8_t imm8) {
6527   assert(VM_Version::supports_evex(), "");
6528   assert(dst != xnoreg, "sanity");
6529   assert(imm8 <= 0x01, "imm8: %u", imm8);
6530   InstructionMark im(this);
6531   InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6532   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_64bit);
6533   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6534   emit_int8(0x1A);
6535   emit_operand(dst, src);
6536   // 0x00 - insert into lower 256 bits
6537   // 0x01 - insert into upper 256 bits
6538   emit_int8(imm8 & 0x01);
6539 }
6540 
6541 
6542 // vextracti forms
6543 
6544 void Assembler::vextracti128(XMMRegister dst, XMMRegister src, uint8_t imm8) {
6545   assert(VM_Version::supports_avx(), "");
6546   assert(imm8 <= 0x01, "imm8: %u", imm8);
6547   int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_256bit;
6548   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6549   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6550   emit_int8(0x39);
6551   emit_int8((unsigned char)(0xC0 | encode));
6552   // 0x00 - extract from lower 128 bits
6553   // 0x01 - extract from upper 128 bits
6554   emit_int8(imm8 & 0x01);
6555 }
6556 
6557 void Assembler::vextracti128(Address dst, XMMRegister src, uint8_t imm8) {
6558   assert(VM_Version::supports_avx2(), "");
6559   assert(src != xnoreg, "sanity");
6560   assert(imm8 <= 0x01, "imm8: %u", imm8);
6561   int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_256bit;
6562   InstructionMark im(this);
6563   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6564   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
6565   attributes.reset_is_clear_context();
6566   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6567   emit_int8(0x39);
6568   emit_operand(src, dst);
6569   // 0x00 - extract from lower 128 bits
6570   // 0x01 - extract from upper 128 bits
6571   emit_int8(imm8 & 0x01);
6572 }
6573 
6574 void Assembler::vextracti32x4(XMMRegister dst, XMMRegister src, uint8_t imm8) {
6575   assert(VM_Version::supports_avx(), "");
6576   assert(imm8 <= 0x03, "imm8: %u", imm8);
6577   int vector_len = VM_Version::supports_evex() ? AVX_512bit : AVX_256bit;
6578   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6579   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6580   emit_int8(0x39);
6581   emit_int8((unsigned char)(0xC0 | encode));
6582   // 0x00 - extract from bits 127:0
6583   // 0x01 - extract from bits 255:128
6584   // 0x02 - extract from bits 383:256
6585   // 0x03 - extract from bits 511:384
6586   emit_int8(imm8 & 0x03);
6587 }
6588 
6589 void Assembler::vextracti32x4(Address dst, XMMRegister src, uint8_t imm8) {
6590   assert(VM_Version::supports_evex(), "");
6591   assert(src != xnoreg, "sanity");
6592   assert(imm8 <= 0x03, "imm8: %u", imm8);
6593   InstructionMark im(this);
6594   InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6595   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
6596   attributes.reset_is_clear_context();
6597   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6598   emit_int8(0x39);
6599   emit_operand(src, dst);
6600   // 0x00 - extract from bits 127:0
6601   // 0x01 - extract from bits 255:128
6602   // 0x02 - extract from bits 383:256
6603   // 0x03 - extract from bits 511:384
6604   emit_int8(imm8 & 0x03);
6605 }
6606 
6607 void Assembler::vextracti64x2(XMMRegister dst, XMMRegister src, uint8_t imm8) {
6608   assert(VM_Version::supports_avx512dq(), "");
6609   assert(imm8 <= 0x03, "imm8: %u", imm8);
6610   InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6611   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6612   emit_int8(0x39);
6613   emit_int8((unsigned char)(0xC0 | encode));
6614   // 0x00 - extract from bits 127:0
6615   // 0x01 - extract from bits 255:128
6616   // 0x02 - extract from bits 383:256
6617   // 0x03 - extract from bits 511:384
6618   emit_int8(imm8 & 0x03);
6619 }
6620 
6621 void Assembler::vextracti64x4(XMMRegister dst, XMMRegister src, uint8_t imm8) {
6622   assert(VM_Version::supports_evex(), "");
6623   assert(imm8 <= 0x01, "imm8: %u", imm8);
6624   InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6625   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6626   emit_int8(0x3B);
6627   emit_int8((unsigned char)(0xC0 | encode));
6628   // 0x00 - extract from lower 256 bits
6629   // 0x01 - extract from upper 256 bits
6630   emit_int8(imm8 & 0x01);
6631 }
6632 
6633 
6634 // vextractf forms
6635 
6636 void Assembler::vextractf128(XMMRegister dst, XMMRegister src, uint8_t imm8) {
6637   assert(VM_Version::supports_avx(), "");
6638   assert(imm8 <= 0x01, "imm8: %u", imm8);
6639   int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_256bit;
6640   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6641   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6642   emit_int8(0x19);
6643   emit_int8((unsigned char)(0xC0 | encode));
6644   // 0x00 - extract from lower 128 bits
6645   // 0x01 - extract from upper 128 bits
6646   emit_int8(imm8 & 0x01);
6647 }
6648 
6649 void Assembler::vextractf128(Address dst, XMMRegister src, uint8_t imm8) {
6650   assert(VM_Version::supports_avx(), "");
6651   assert(src != xnoreg, "sanity");
6652   assert(imm8 <= 0x01, "imm8: %u", imm8);
6653   int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_256bit;
6654   InstructionMark im(this);
6655   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6656   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
6657   attributes.reset_is_clear_context();
6658   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6659   emit_int8(0x19);
6660   emit_operand(src, dst);
6661   // 0x00 - extract from lower 128 bits
6662   // 0x01 - extract from upper 128 bits
6663   emit_int8(imm8 & 0x01);
6664 }
6665 
6666 void Assembler::vextractf32x4(XMMRegister dst, XMMRegister src, uint8_t imm8) {
6667   assert(VM_Version::supports_avx(), "");
6668   assert(imm8 <= 0x03, "imm8: %u", imm8);
6669   int vector_len = VM_Version::supports_evex() ? AVX_512bit : AVX_256bit;
6670   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6671   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6672   emit_int8(0x19);
6673   emit_int8((unsigned char)(0xC0 | encode));
6674   // 0x00 - extract from bits 127:0
6675   // 0x01 - extract from bits 255:128
6676   // 0x02 - extract from bits 383:256
6677   // 0x03 - extract from bits 511:384
6678   emit_int8(imm8 & 0x03);
6679 }
6680 
6681 void Assembler::vextractf32x4(Address dst, XMMRegister src, uint8_t imm8) {
6682   assert(VM_Version::supports_evex(), "");
6683   assert(src != xnoreg, "sanity");
6684   assert(imm8 <= 0x03, "imm8: %u", imm8);
6685   InstructionMark im(this);
6686   InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6687   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
6688   attributes.reset_is_clear_context();
6689   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6690   emit_int8(0x19);
6691   emit_operand(src, dst);
6692   // 0x00 - extract from bits 127:0
6693   // 0x01 - extract from bits 255:128
6694   // 0x02 - extract from bits 383:256
6695   // 0x03 - extract from bits 511:384
6696   emit_int8(imm8 & 0x03);
6697 }
6698 
6699 void Assembler::vextractf64x2(XMMRegister dst, XMMRegister src, uint8_t imm8) {
6700   assert(VM_Version::supports_avx512dq(), "");
6701   assert(imm8 <= 0x03, "imm8: %u", imm8);
6702   InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6703   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6704   emit_int8(0x19);
6705   emit_int8((unsigned char)(0xC0 | encode));
6706   // 0x00 - extract from bits 127:0
6707   // 0x01 - extract from bits 255:128
6708   // 0x02 - extract from bits 383:256
6709   // 0x03 - extract from bits 511:384
6710   emit_int8(imm8 & 0x03);
6711 }
6712 
6713 void Assembler::vextractf64x4(XMMRegister dst, XMMRegister src, uint8_t imm8) {
6714   assert(VM_Version::supports_evex(), "");
6715   assert(imm8 <= 0x01, "imm8: %u", imm8);
6716   InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6717   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6718   emit_int8(0x1B);
6719   emit_int8((unsigned char)(0xC0 | encode));
6720   // 0x00 - extract from lower 256 bits
6721   // 0x01 - extract from upper 256 bits
6722   emit_int8(imm8 & 0x01);
6723 }
6724 
6725 void Assembler::vextractf64x4(Address dst, XMMRegister src, uint8_t imm8) {
6726   assert(VM_Version::supports_evex(), "");
6727   assert(src != xnoreg, "sanity");
6728   assert(imm8 <= 0x01, "imm8: %u", imm8);
6729   InstructionMark im(this);
6730   InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6731   attributes.set_address_attributes(/* tuple_type */ EVEX_T4,/* input_size_in_bits */  EVEX_64bit);
6732   attributes.reset_is_clear_context();
6733   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6734   emit_int8(0x1B);
6735   emit_operand(src, dst);
6736   // 0x00 - extract from lower 256 bits
6737   // 0x01 - extract from upper 256 bits
6738   emit_int8(imm8 & 0x01);
6739 }
6740 
6741 
6742 // legacy word/dword replicate
6743 void Assembler::vpbroadcastw(XMMRegister dst, XMMRegister src) {
6744   assert(VM_Version::supports_avx2(), "");
6745   InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
6746   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6747   emit_int8(0x79);
6748   emit_int8((unsigned char)(0xC0 | encode));
6749 }
6750 
6751 void Assembler::vpbroadcastd(XMMRegister dst, XMMRegister src) {
6752   assert(VM_Version::supports_avx2(), "");
6753   InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6754   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6755   emit_int8(0x58);
6756   emit_int8((unsigned char)(0xC0 | encode));
6757 }
6758 
6759 
6760 // xmm/mem sourced byte/word/dword/qword replicate
6761 
6762 // duplicate 1-byte integer data from src into programmed locations in dest : requires AVX512BW and AVX512VL
6763 void Assembler::evpbroadcastb(XMMRegister dst, XMMRegister src, int vector_len) {
6764   assert(VM_Version::supports_evex(), "");
6765   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
6766   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6767   emit_int8(0x78);
6768   emit_int8((unsigned char)(0xC0 | encode));
6769 }
6770 
6771 void Assembler::evpbroadcastb(XMMRegister dst, Address src, int vector_len) {
6772   assert(VM_Version::supports_evex(), "");
6773   assert(dst != xnoreg, "sanity");
6774   InstructionMark im(this);
6775   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
6776   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_8bit);
6777   // swap src<->dst for encoding
6778   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6779   emit_int8(0x78);
6780   emit_operand(dst, src);
6781 }
6782 
6783 // duplicate 2-byte integer data from src into programmed locations in dest : requires AVX512BW and AVX512VL
6784 void Assembler::evpbroadcastw(XMMRegister dst, XMMRegister src, int vector_len) {
6785   assert(VM_Version::supports_evex(), "");
6786   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
6787   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6788   emit_int8(0x79);
6789   emit_int8((unsigned char)(0xC0 | encode));
6790 }
6791 
6792 void Assembler::evpbroadcastw(XMMRegister dst, Address src, int vector_len) {
6793   assert(VM_Version::supports_evex(), "");
6794   assert(dst != xnoreg, "sanity");
6795   InstructionMark im(this);
6796   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
6797   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_16bit);
6798   // swap src<->dst for encoding
6799   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6800   emit_int8(0x79);
6801   emit_operand(dst, src);
6802 }
6803 
6804 // duplicate 4-byte integer data from src into programmed locations in dest : requires AVX512VL
6805 void Assembler::evpbroadcastd(XMMRegister dst, XMMRegister src, int vector_len) {
6806   assert(VM_Version::supports_evex(), "");
6807   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6808   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6809   emit_int8(0x58);
6810   emit_int8((unsigned char)(0xC0 | encode));
6811 }
6812 
6813 void Assembler::evpbroadcastd(XMMRegister dst, Address src, int vector_len) {
6814   assert(VM_Version::supports_evex(), "");
6815   assert(dst != xnoreg, "sanity");
6816   InstructionMark im(this);
6817   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6818   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
6819   // swap src<->dst for encoding
6820   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6821   emit_int8(0x58);
6822   emit_operand(dst, src);
6823 }
6824 
6825 // duplicate 8-byte integer data from src into programmed locations in dest : requires AVX512VL
6826 void Assembler::evpbroadcastq(XMMRegister dst, XMMRegister src, int vector_len) {
6827   assert(VM_Version::supports_evex(), "");
6828   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6829   attributes.set_rex_vex_w_reverted();
6830   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6831   emit_int8(0x59);
6832   emit_int8((unsigned char)(0xC0 | encode));
6833 }
6834 
6835 void Assembler::evpbroadcastq(XMMRegister dst, Address src, int vector_len) {
6836   assert(VM_Version::supports_evex(), "");
6837   assert(dst != xnoreg, "sanity");
6838   InstructionMark im(this);
6839   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6840   attributes.set_rex_vex_w_reverted();
6841   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
6842   // swap src<->dst for encoding
6843   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6844   emit_int8(0x59);
6845   emit_operand(dst, src);
6846 }
6847 void Assembler::evbroadcasti64x2(XMMRegister dst, XMMRegister src, int vector_len) {
6848   assert(vector_len != Assembler::AVX_128bit, "");
6849   assert(VM_Version::supports_avx512dq(), "");
6850   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6851   attributes.set_rex_vex_w_reverted();
6852   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6853   emit_int8(0x5A);
6854   emit_int8((unsigned char)(0xC0 | encode));
6855 }
6856 
6857 void Assembler::evbroadcasti64x2(XMMRegister dst, Address src, int vector_len) {
6858   assert(vector_len != Assembler::AVX_128bit, "");
6859   assert(VM_Version::supports_avx512dq(), "");
6860   assert(dst != xnoreg, "sanity");
6861   InstructionMark im(this);
6862   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6863   attributes.set_rex_vex_w_reverted();
6864   attributes.set_address_attributes(/* tuple_type */ EVEX_T2, /* input_size_in_bits */ EVEX_64bit);
6865   // swap src<->dst for encoding
6866   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6867   emit_int8(0x5A);
6868   emit_operand(dst, src);
6869 }
6870 
6871 // scalar single/double precision replicate
6872 
6873 // duplicate single precision data from src into programmed locations in dest : requires AVX512VL
6874 void Assembler::evpbroadcastss(XMMRegister dst, XMMRegister src, int vector_len) {
6875   assert(VM_Version::supports_evex(), "");
6876   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6877   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6878   emit_int8(0x18);
6879   emit_int8((unsigned char)(0xC0 | encode));
6880 }
6881 
6882 void Assembler::evpbroadcastss(XMMRegister dst, Address src, int vector_len) {
6883   assert(VM_Version::supports_evex(), "");
6884   assert(dst != xnoreg, "sanity");
6885   InstructionMark im(this);
6886   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6887   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
6888   // swap src<->dst for encoding
6889   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6890   emit_int8(0x18);
6891   emit_operand(dst, src);
6892 }
6893 
6894 // duplicate double precision data from src into programmed locations in dest : requires AVX512VL
6895 void Assembler::evpbroadcastsd(XMMRegister dst, XMMRegister src, int vector_len) {
6896   assert(VM_Version::supports_evex(), "");
6897   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6898   attributes.set_rex_vex_w_reverted();
6899   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6900   emit_int8(0x19);
6901   emit_int8((unsigned char)(0xC0 | encode));
6902 }
6903 
6904 void Assembler::evpbroadcastsd(XMMRegister dst, Address src, int vector_len) {
6905   assert(VM_Version::supports_evex(), "");
6906   assert(dst != xnoreg, "sanity");
6907   InstructionMark im(this);
6908   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6909   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
6910   attributes.set_rex_vex_w_reverted();
6911   // swap src<->dst for encoding
6912   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6913   emit_int8(0x19);
6914   emit_operand(dst, src);
6915 }
6916 
6917 
6918 // gpr source broadcast forms
6919 
6920 // duplicate 1-byte integer data from src into programmed locations in dest : requires AVX512BW and AVX512VL
6921 void Assembler::evpbroadcastb(XMMRegister dst, Register src, int vector_len) {
6922   assert(VM_Version::supports_evex(), "");
6923   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
6924   attributes.set_is_evex_instruction();
6925   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6926   emit_int8(0x7A);
6927   emit_int8((unsigned char)(0xC0 | encode));
6928 }
6929 
6930 // duplicate 2-byte integer data from src into programmed locations in dest : requires AVX512BW and AVX512VL
6931 void Assembler::evpbroadcastw(XMMRegister dst, Register src, int vector_len) {
6932   assert(VM_Version::supports_evex(), "");
6933   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
6934   attributes.set_is_evex_instruction();
6935   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6936   emit_int8(0x7B);
6937   emit_int8((unsigned char)(0xC0 | encode));
6938 }
6939 
6940 // duplicate 4-byte integer data from src into programmed locations in dest : requires AVX512VL
6941 void Assembler::evpbroadcastd(XMMRegister dst, Register src, int vector_len) {
6942   assert(VM_Version::supports_evex(), "");
6943   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6944   attributes.set_is_evex_instruction();
6945   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6946   emit_int8(0x7C);
6947   emit_int8((unsigned char)(0xC0 | encode));
6948 }
6949 
6950 // duplicate 8-byte integer data from src into programmed locations in dest : requires AVX512VL
6951 void Assembler::evpbroadcastq(XMMRegister dst, Register src, int vector_len) {
6952   assert(VM_Version::supports_evex(), "");
6953   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6954   attributes.set_is_evex_instruction();
6955   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6956   emit_int8(0x7C);
6957   emit_int8((unsigned char)(0xC0 | encode));
6958 }
6959 
6960 void Assembler::evpgatherdd(XMMRegister dst, KRegister mask, Address src, int vector_len) {
6961   assert(VM_Version::supports_evex(), "");
6962   assert(dst != xnoreg, "sanity");
6963   InstructionMark im(this);
6964   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6965   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
6966   attributes.reset_is_clear_context();
6967   attributes.set_embedded_opmask_register_specifier(mask);
6968   attributes.set_is_evex_instruction();
6969   // swap src<->dst for encoding
6970   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6971   emit_int8((unsigned char)0x90);
6972   emit_operand(dst, src);
6973 }
6974 
6975 // Carry-Less Multiplication Quadword
6976 void Assembler::pclmulqdq(XMMRegister dst, XMMRegister src, int mask) {
6977   assert(VM_Version::supports_clmul(), "");
6978   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
6979   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6980   emit_int8(0x44);
6981   emit_int8((unsigned char)(0xC0 | encode));
6982   emit_int8((unsigned char)mask);
6983 }
6984 
6985 // Carry-Less Multiplication Quadword
6986 void Assembler::vpclmulqdq(XMMRegister dst, XMMRegister nds, XMMRegister src, int mask) {
6987   assert(VM_Version::supports_avx() && VM_Version::supports_clmul(), "");
6988   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
6989   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6990   emit_int8(0x44);
6991   emit_int8((unsigned char)(0xC0 | encode));
6992   emit_int8((unsigned char)mask);
6993 }
6994 
6995 void Assembler::evpclmulqdq(XMMRegister dst, XMMRegister nds, XMMRegister src, int mask, int vector_len) {
6996   assert(VM_Version::supports_vpclmulqdq(), "Requires vector carryless multiplication support");
6997   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6998   attributes.set_is_evex_instruction();
6999   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
7000   emit_int8(0x44);
7001   emit_int8((unsigned char)(0xC0 | encode));
7002   emit_int8((unsigned char)mask);
7003 }
7004 
7005 void Assembler::vzeroupper() {
7006   if (VM_Version::supports_vzeroupper()) {
7007     InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
7008     (void)vex_prefix_and_encode(0, 0, 0, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
7009     emit_int8(0x77);
7010   }
7011 }
7012 
7013 #ifndef _LP64
7014 // 32bit only pieces of the assembler
7015 
7016 void Assembler::cmp_literal32(Register src1, int32_t imm32, RelocationHolder const& rspec) {
7017   // NO PREFIX AS NEVER 64BIT
7018   InstructionMark im(this);
7019   emit_int8((unsigned char)0x81);
7020   emit_int8((unsigned char)(0xF8 | src1->encoding()));
7021   emit_data(imm32, rspec, 0);
7022 }
7023 
7024 void Assembler::cmp_literal32(Address src1, int32_t imm32, RelocationHolder const& rspec) {
7025   // NO PREFIX AS NEVER 64BIT (not even 32bit versions of 64bit regs
7026   InstructionMark im(this);
7027   emit_int8((unsigned char)0x81);
7028   emit_operand(rdi, src1);
7029   emit_data(imm32, rspec, 0);
7030 }
7031 
7032 // The 64-bit (32bit platform) cmpxchg compares the value at adr with the contents of rdx:rax,
7033 // and stores rcx:rbx into adr if so; otherwise, the value at adr is loaded
7034 // into rdx:rax.  The ZF is set if the compared values were equal, and cleared otherwise.
7035 void Assembler::cmpxchg8(Address adr) {
7036   InstructionMark im(this);
7037   emit_int8(0x0F);
7038   emit_int8((unsigned char)0xC7);
7039   emit_operand(rcx, adr);
7040 }
7041 
7042 void Assembler::decl(Register dst) {
7043   // Don't use it directly. Use MacroAssembler::decrementl() instead.
7044  emit_int8(0x48 | dst->encoding());
7045 }
7046 
7047 #endif // _LP64
7048 
7049 // 64bit typically doesn't use the x87 but needs to for the trig funcs
7050 
7051 void Assembler::fabs() {
7052   emit_int8((unsigned char)0xD9);
7053   emit_int8((unsigned char)0xE1);
7054 }
7055 
7056 void Assembler::fadd(int i) {
7057   emit_farith(0xD8, 0xC0, i);
7058 }
7059 
7060 void Assembler::fadd_d(Address src) {
7061   InstructionMark im(this);
7062   emit_int8((unsigned char)0xDC);
7063   emit_operand32(rax, src);
7064 }
7065 
7066 void Assembler::fadd_s(Address src) {
7067   InstructionMark im(this);
7068   emit_int8((unsigned char)0xD8);
7069   emit_operand32(rax, src);
7070 }
7071 
7072 void Assembler::fadda(int i) {
7073   emit_farith(0xDC, 0xC0, i);
7074 }
7075 
7076 void Assembler::faddp(int i) {
7077   emit_farith(0xDE, 0xC0, i);
7078 }
7079 
7080 void Assembler::fchs() {
7081   emit_int8((unsigned char)0xD9);
7082   emit_int8((unsigned char)0xE0);
7083 }
7084 
7085 void Assembler::fcom(int i) {
7086   emit_farith(0xD8, 0xD0, i);
7087 }
7088 
7089 void Assembler::fcomp(int i) {
7090   emit_farith(0xD8, 0xD8, i);
7091 }
7092 
7093 void Assembler::fcomp_d(Address src) {
7094   InstructionMark im(this);
7095   emit_int8((unsigned char)0xDC);
7096   emit_operand32(rbx, src);
7097 }
7098 
7099 void Assembler::fcomp_s(Address src) {
7100   InstructionMark im(this);
7101   emit_int8((unsigned char)0xD8);
7102   emit_operand32(rbx, src);
7103 }
7104 
7105 void Assembler::fcompp() {
7106   emit_int8((unsigned char)0xDE);
7107   emit_int8((unsigned char)0xD9);
7108 }
7109 
7110 void Assembler::fcos() {
7111   emit_int8((unsigned char)0xD9);
7112   emit_int8((unsigned char)0xFF);
7113 }
7114 
7115 void Assembler::fdecstp() {
7116   emit_int8((unsigned char)0xD9);
7117   emit_int8((unsigned char)0xF6);
7118 }
7119 
7120 void Assembler::fdiv(int i) {
7121   emit_farith(0xD8, 0xF0, i);
7122 }
7123 
7124 void Assembler::fdiv_d(Address src) {
7125   InstructionMark im(this);
7126   emit_int8((unsigned char)0xDC);
7127   emit_operand32(rsi, src);
7128 }
7129 
7130 void Assembler::fdiv_s(Address src) {
7131   InstructionMark im(this);
7132   emit_int8((unsigned char)0xD8);
7133   emit_operand32(rsi, src);
7134 }
7135 
7136 void Assembler::fdiva(int i) {
7137   emit_farith(0xDC, 0xF8, i);
7138 }
7139 
7140 // Note: The Intel manual (Pentium Processor User's Manual, Vol.3, 1994)
7141 //       is erroneous for some of the floating-point instructions below.
7142 
7143 void Assembler::fdivp(int i) {
7144   emit_farith(0xDE, 0xF8, i);                    // ST(0) <- ST(0) / ST(1) and pop (Intel manual wrong)
7145 }
7146 
7147 void Assembler::fdivr(int i) {
7148   emit_farith(0xD8, 0xF8, i);
7149 }
7150 
7151 void Assembler::fdivr_d(Address src) {
7152   InstructionMark im(this);
7153   emit_int8((unsigned char)0xDC);
7154   emit_operand32(rdi, src);
7155 }
7156 
7157 void Assembler::fdivr_s(Address src) {
7158   InstructionMark im(this);
7159   emit_int8((unsigned char)0xD8);
7160   emit_operand32(rdi, src);
7161 }
7162 
7163 void Assembler::fdivra(int i) {
7164   emit_farith(0xDC, 0xF0, i);
7165 }
7166 
7167 void Assembler::fdivrp(int i) {
7168   emit_farith(0xDE, 0xF0, i);                    // ST(0) <- ST(1) / ST(0) and pop (Intel manual wrong)
7169 }
7170 
7171 void Assembler::ffree(int i) {
7172   emit_farith(0xDD, 0xC0, i);
7173 }
7174 
7175 void Assembler::fild_d(Address adr) {
7176   InstructionMark im(this);
7177   emit_int8((unsigned char)0xDF);
7178   emit_operand32(rbp, adr);
7179 }
7180 
7181 void Assembler::fild_s(Address adr) {
7182   InstructionMark im(this);
7183   emit_int8((unsigned char)0xDB);
7184   emit_operand32(rax, adr);
7185 }
7186 
7187 void Assembler::fincstp() {
7188   emit_int8((unsigned char)0xD9);
7189   emit_int8((unsigned char)0xF7);
7190 }
7191 
7192 void Assembler::finit() {
7193   emit_int8((unsigned char)0x9B);
7194   emit_int8((unsigned char)0xDB);
7195   emit_int8((unsigned char)0xE3);
7196 }
7197 
7198 void Assembler::fist_s(Address adr) {
7199   InstructionMark im(this);
7200   emit_int8((unsigned char)0xDB);
7201   emit_operand32(rdx, adr);
7202 }
7203 
7204 void Assembler::fistp_d(Address adr) {
7205   InstructionMark im(this);
7206   emit_int8((unsigned char)0xDF);
7207   emit_operand32(rdi, adr);
7208 }
7209 
7210 void Assembler::fistp_s(Address adr) {
7211   InstructionMark im(this);
7212   emit_int8((unsigned char)0xDB);
7213   emit_operand32(rbx, adr);
7214 }
7215 
7216 void Assembler::fld1() {
7217   emit_int8((unsigned char)0xD9);
7218   emit_int8((unsigned char)0xE8);
7219 }
7220 
7221 void Assembler::fld_d(Address adr) {
7222   InstructionMark im(this);
7223   emit_int8((unsigned char)0xDD);
7224   emit_operand32(rax, adr);
7225 }
7226 
7227 void Assembler::fld_s(Address adr) {
7228   InstructionMark im(this);
7229   emit_int8((unsigned char)0xD9);
7230   emit_operand32(rax, adr);
7231 }
7232 
7233 
7234 void Assembler::fld_s(int index) {
7235   emit_farith(0xD9, 0xC0, index);
7236 }
7237 
7238 void Assembler::fld_x(Address adr) {
7239   InstructionMark im(this);
7240   emit_int8((unsigned char)0xDB);
7241   emit_operand32(rbp, adr);
7242 }
7243 
7244 void Assembler::fldcw(Address src) {
7245   InstructionMark im(this);
7246   emit_int8((unsigned char)0xD9);
7247   emit_operand32(rbp, src);
7248 }
7249 
7250 void Assembler::fldenv(Address src) {
7251   InstructionMark im(this);
7252   emit_int8((unsigned char)0xD9);
7253   emit_operand32(rsp, src);
7254 }
7255 
7256 void Assembler::fldlg2() {
7257   emit_int8((unsigned char)0xD9);
7258   emit_int8((unsigned char)0xEC);
7259 }
7260 
7261 void Assembler::fldln2() {
7262   emit_int8((unsigned char)0xD9);
7263   emit_int8((unsigned char)0xED);
7264 }
7265 
7266 void Assembler::fldz() {
7267   emit_int8((unsigned char)0xD9);
7268   emit_int8((unsigned char)0xEE);
7269 }
7270 
7271 void Assembler::flog() {
7272   fldln2();
7273   fxch();
7274   fyl2x();
7275 }
7276 
7277 void Assembler::flog10() {
7278   fldlg2();
7279   fxch();
7280   fyl2x();
7281 }
7282 
7283 void Assembler::fmul(int i) {
7284   emit_farith(0xD8, 0xC8, i);
7285 }
7286 
7287 void Assembler::fmul_d(Address src) {
7288   InstructionMark im(this);
7289   emit_int8((unsigned char)0xDC);
7290   emit_operand32(rcx, src);
7291 }
7292 
7293 void Assembler::fmul_s(Address src) {
7294   InstructionMark im(this);
7295   emit_int8((unsigned char)0xD8);
7296   emit_operand32(rcx, src);
7297 }
7298 
7299 void Assembler::fmula(int i) {
7300   emit_farith(0xDC, 0xC8, i);
7301 }
7302 
7303 void Assembler::fmulp(int i) {
7304   emit_farith(0xDE, 0xC8, i);
7305 }
7306 
7307 void Assembler::fnsave(Address dst) {
7308   InstructionMark im(this);
7309   emit_int8((unsigned char)0xDD);
7310   emit_operand32(rsi, dst);
7311 }
7312 
7313 void Assembler::fnstcw(Address src) {
7314   InstructionMark im(this);
7315   emit_int8((unsigned char)0x9B);
7316   emit_int8((unsigned char)0xD9);
7317   emit_operand32(rdi, src);
7318 }
7319 
7320 void Assembler::fnstsw_ax() {
7321   emit_int8((unsigned char)0xDF);
7322   emit_int8((unsigned char)0xE0);
7323 }
7324 
7325 void Assembler::fprem() {
7326   emit_int8((unsigned char)0xD9);
7327   emit_int8((unsigned char)0xF8);
7328 }
7329 
7330 void Assembler::fprem1() {
7331   emit_int8((unsigned char)0xD9);
7332   emit_int8((unsigned char)0xF5);
7333 }
7334 
7335 void Assembler::frstor(Address src) {
7336   InstructionMark im(this);
7337   emit_int8((unsigned char)0xDD);
7338   emit_operand32(rsp, src);
7339 }
7340 
7341 void Assembler::fsin() {
7342   emit_int8((unsigned char)0xD9);
7343   emit_int8((unsigned char)0xFE);
7344 }
7345 
7346 void Assembler::fsqrt() {
7347   emit_int8((unsigned char)0xD9);
7348   emit_int8((unsigned char)0xFA);
7349 }
7350 
7351 void Assembler::fst_d(Address adr) {
7352   InstructionMark im(this);
7353   emit_int8((unsigned char)0xDD);
7354   emit_operand32(rdx, adr);
7355 }
7356 
7357 void Assembler::fst_s(Address adr) {
7358   InstructionMark im(this);
7359   emit_int8((unsigned char)0xD9);
7360   emit_operand32(rdx, adr);
7361 }
7362 
7363 void Assembler::fstp_d(Address adr) {
7364   InstructionMark im(this);
7365   emit_int8((unsigned char)0xDD);
7366   emit_operand32(rbx, adr);
7367 }
7368 
7369 void Assembler::fstp_d(int index) {
7370   emit_farith(0xDD, 0xD8, index);
7371 }
7372 
7373 void Assembler::fstp_s(Address adr) {
7374   InstructionMark im(this);
7375   emit_int8((unsigned char)0xD9);
7376   emit_operand32(rbx, adr);
7377 }
7378 
7379 void Assembler::fstp_x(Address adr) {
7380   InstructionMark im(this);
7381   emit_int8((unsigned char)0xDB);
7382   emit_operand32(rdi, adr);
7383 }
7384 
7385 void Assembler::fsub(int i) {
7386   emit_farith(0xD8, 0xE0, i);
7387 }
7388 
7389 void Assembler::fsub_d(Address src) {
7390   InstructionMark im(this);
7391   emit_int8((unsigned char)0xDC);
7392   emit_operand32(rsp, src);
7393 }
7394 
7395 void Assembler::fsub_s(Address src) {
7396   InstructionMark im(this);
7397   emit_int8((unsigned char)0xD8);
7398   emit_operand32(rsp, src);
7399 }
7400 
7401 void Assembler::fsuba(int i) {
7402   emit_farith(0xDC, 0xE8, i);
7403 }
7404 
7405 void Assembler::fsubp(int i) {
7406   emit_farith(0xDE, 0xE8, i);                    // ST(0) <- ST(0) - ST(1) and pop (Intel manual wrong)
7407 }
7408 
7409 void Assembler::fsubr(int i) {
7410   emit_farith(0xD8, 0xE8, i);
7411 }
7412 
7413 void Assembler::fsubr_d(Address src) {
7414   InstructionMark im(this);
7415   emit_int8((unsigned char)0xDC);
7416   emit_operand32(rbp, src);
7417 }
7418 
7419 void Assembler::fsubr_s(Address src) {
7420   InstructionMark im(this);
7421   emit_int8((unsigned char)0xD8);
7422   emit_operand32(rbp, src);
7423 }
7424 
7425 void Assembler::fsubra(int i) {
7426   emit_farith(0xDC, 0xE0, i);
7427 }
7428 
7429 void Assembler::fsubrp(int i) {
7430   emit_farith(0xDE, 0xE0, i);                    // ST(0) <- ST(1) - ST(0) and pop (Intel manual wrong)
7431 }
7432 
7433 void Assembler::ftan() {
7434   emit_int8((unsigned char)0xD9);
7435   emit_int8((unsigned char)0xF2);
7436   emit_int8((unsigned char)0xDD);
7437   emit_int8((unsigned char)0xD8);
7438 }
7439 
7440 void Assembler::ftst() {
7441   emit_int8((unsigned char)0xD9);
7442   emit_int8((unsigned char)0xE4);
7443 }
7444 
7445 void Assembler::fucomi(int i) {
7446   // make sure the instruction is supported (introduced for P6, together with cmov)
7447   guarantee(VM_Version::supports_cmov(), "illegal instruction");
7448   emit_farith(0xDB, 0xE8, i);
7449 }
7450 
7451 void Assembler::fucomip(int i) {
7452   // make sure the instruction is supported (introduced for P6, together with cmov)
7453   guarantee(VM_Version::supports_cmov(), "illegal instruction");
7454   emit_farith(0xDF, 0xE8, i);
7455 }
7456 
7457 void Assembler::fwait() {
7458   emit_int8((unsigned char)0x9B);
7459 }
7460 
7461 void Assembler::fxch(int i) {
7462   emit_farith(0xD9, 0xC8, i);
7463 }
7464 
7465 void Assembler::fyl2x() {
7466   emit_int8((unsigned char)0xD9);
7467   emit_int8((unsigned char)0xF1);
7468 }
7469 
7470 void Assembler::frndint() {
7471   emit_int8((unsigned char)0xD9);
7472   emit_int8((unsigned char)0xFC);
7473 }
7474 
7475 void Assembler::f2xm1() {
7476   emit_int8((unsigned char)0xD9);
7477   emit_int8((unsigned char)0xF0);
7478 }
7479 
7480 void Assembler::fldl2e() {
7481   emit_int8((unsigned char)0xD9);
7482   emit_int8((unsigned char)0xEA);
7483 }
7484 
7485 // SSE SIMD prefix byte values corresponding to VexSimdPrefix encoding.
7486 static int simd_pre[4] = { 0, 0x66, 0xF3, 0xF2 };
7487 // SSE opcode second byte values (first is 0x0F) corresponding to VexOpcode encoding.
7488 static int simd_opc[4] = { 0,    0, 0x38, 0x3A };
7489 
7490 // Generate SSE legacy REX prefix and SIMD opcode based on VEX encoding.
7491 void Assembler::rex_prefix(Address adr, XMMRegister xreg, VexSimdPrefix pre, VexOpcode opc, bool rex_w) {
7492   if (pre > 0) {
7493     emit_int8(simd_pre[pre]);
7494   }
7495   if (rex_w) {
7496     prefixq(adr, xreg);
7497   } else {
7498     prefix(adr, xreg);
7499   }
7500   if (opc > 0) {
7501     emit_int8(0x0F);
7502     int opc2 = simd_opc[opc];
7503     if (opc2 > 0) {
7504       emit_int8(opc2);
7505     }
7506   }
7507 }
7508 
7509 int Assembler::rex_prefix_and_encode(int dst_enc, int src_enc, VexSimdPrefix pre, VexOpcode opc, bool rex_w) {
7510   if (pre > 0) {
7511     emit_int8(simd_pre[pre]);
7512   }
7513   int encode = (rex_w) ? prefixq_and_encode(dst_enc, src_enc) : prefix_and_encode(dst_enc, src_enc);
7514   if (opc > 0) {
7515     emit_int8(0x0F);
7516     int opc2 = simd_opc[opc];
7517     if (opc2 > 0) {
7518       emit_int8(opc2);
7519     }
7520   }
7521   return encode;
7522 }
7523 
7524 
7525 void Assembler::vex_prefix(bool vex_r, bool vex_b, bool vex_x, int nds_enc, VexSimdPrefix pre, VexOpcode opc) {
7526   int vector_len = _attributes->get_vector_len();
7527   bool vex_w = _attributes->is_rex_vex_w();
7528   if (vex_b || vex_x || vex_w || (opc == VEX_OPCODE_0F_38) || (opc == VEX_OPCODE_0F_3A)) {
7529     prefix(VEX_3bytes);
7530 
7531     int byte1 = (vex_r ? VEX_R : 0) | (vex_x ? VEX_X : 0) | (vex_b ? VEX_B : 0);
7532     byte1 = (~byte1) & 0xE0;
7533     byte1 |= opc;
7534     emit_int8(byte1);
7535 
7536     int byte2 = ((~nds_enc) & 0xf) << 3;
7537     byte2 |= (vex_w ? VEX_W : 0) | ((vector_len > 0) ? 4 : 0) | pre;
7538     emit_int8(byte2);
7539   } else {
7540     prefix(VEX_2bytes);
7541 
7542     int byte1 = vex_r ? VEX_R : 0;
7543     byte1 = (~byte1) & 0x80;
7544     byte1 |= ((~nds_enc) & 0xf) << 3;
7545     byte1 |= ((vector_len > 0 ) ? 4 : 0) | pre;
7546     emit_int8(byte1);
7547   }
7548 }
7549 
7550 // This is a 4 byte encoding
7551 void Assembler::evex_prefix(bool vex_r, bool vex_b, bool vex_x, bool evex_r, bool evex_v, int nds_enc, VexSimdPrefix pre, VexOpcode opc){
7552   // EVEX 0x62 prefix
7553   prefix(EVEX_4bytes);
7554   bool vex_w = _attributes->is_rex_vex_w();
7555   int evex_encoding = (vex_w ? VEX_W : 0);
7556   // EVEX.b is not currently used for broadcast of single element or data rounding modes
7557   _attributes->set_evex_encoding(evex_encoding);
7558 
7559   // P0: byte 2, initialized to RXBR`00mm
7560   // instead of not'd
7561   int byte2 = (vex_r ? VEX_R : 0) | (vex_x ? VEX_X : 0) | (vex_b ? VEX_B : 0) | (evex_r ? EVEX_Rb : 0);
7562   byte2 = (~byte2) & 0xF0;
7563   // confine opc opcode extensions in mm bits to lower two bits
7564   // of form {0F, 0F_38, 0F_3A}
7565   byte2 |= opc;
7566   emit_int8(byte2);
7567 
7568   // P1: byte 3 as Wvvvv1pp
7569   int byte3 = ((~nds_enc) & 0xf) << 3;
7570   // p[10] is always 1
7571   byte3 |= EVEX_F;
7572   byte3 |= (vex_w & 1) << 7;
7573   // confine pre opcode extensions in pp bits to lower two bits
7574   // of form {66, F3, F2}
7575   byte3 |= pre;
7576   emit_int8(byte3);
7577 
7578   // P2: byte 4 as zL'Lbv'aaa
7579   // kregs are implemented in the low 3 bits as aaa (hard code k1, it will be initialized for now)
7580   int byte4 = (_attributes->is_no_reg_mask()) ?
7581               0 :
7582               _attributes->get_embedded_opmask_register_specifier();
7583   // EVEX.v` for extending EVEX.vvvv or VIDX
7584   byte4 |= (evex_v ? 0: EVEX_V);
7585   // third EXEC.b for broadcast actions
7586   byte4 |= (_attributes->is_extended_context() ? EVEX_Rb : 0);
7587   // fourth EVEX.L'L for vector length : 0 is 128, 1 is 256, 2 is 512, currently we do not support 1024
7588   byte4 |= ((_attributes->get_vector_len())& 0x3) << 5;
7589   // last is EVEX.z for zero/merge actions
7590   if (_attributes->is_no_reg_mask() == false) {
7591     byte4 |= (_attributes->is_clear_context() ? EVEX_Z : 0);
7592   }
7593   emit_int8(byte4);
7594 }
7595 
7596 void Assembler::vex_prefix(Address adr, int nds_enc, int xreg_enc, VexSimdPrefix pre, VexOpcode opc, InstructionAttr *attributes) {
7597   bool vex_r = ((xreg_enc & 8) == 8) ? 1 : 0;
7598   bool vex_b = adr.base_needs_rex();
7599   bool vex_x;
7600   if (adr.isxmmindex()) {
7601     vex_x = adr.xmmindex_needs_rex();
7602   } else {
7603     vex_x = adr.index_needs_rex();
7604   }
7605   set_attributes(attributes);
7606   attributes->set_current_assembler(this);
7607 
7608   // if vector length is turned off, revert to AVX for vectors smaller than 512-bit
7609   if (UseAVX > 2 && _legacy_mode_vl && attributes->uses_vl()) {
7610     switch (attributes->get_vector_len()) {
7611     case AVX_128bit:
7612     case AVX_256bit:
7613       attributes->set_is_legacy_mode();
7614       break;
7615     }
7616   }
7617 
7618   // For pure EVEX check and see if this instruction
7619   // is allowed in legacy mode and has resources which will
7620   // fit in it.  Pure EVEX instructions will use set_is_evex_instruction in their definition,
7621   // else that field is set when we encode to EVEX
7622   if (UseAVX > 2 && !attributes->is_legacy_mode() &&
7623       !_is_managed && !attributes->is_evex_instruction()) {
7624     if (!_legacy_mode_vl && attributes->get_vector_len() != AVX_512bit) {
7625       bool check_register_bank = NOT_IA32(true) IA32_ONLY(false);
7626       if (check_register_bank) {
7627         // check nds_enc and xreg_enc for upper bank usage
7628         if (nds_enc < 16 && xreg_enc < 16) {
7629           attributes->set_is_legacy_mode();
7630         }
7631       } else {
7632         attributes->set_is_legacy_mode();
7633       }
7634     }
7635   }
7636 
7637   _is_managed = false;
7638   if (UseAVX > 2 && !attributes->is_legacy_mode())
7639   {
7640     bool evex_r = (xreg_enc >= 16);
7641     bool evex_v;
7642     // EVEX.V' is set to true when VSIB is used as we may need to use higher order XMM registers (16-31)
7643     if (adr.isxmmindex())  {
7644       evex_v = ((adr._xmmindex->encoding() > 15) ? true : false);
7645     } else {
7646       evex_v = (nds_enc >= 16);
7647     }
7648     attributes->set_is_evex_instruction();
7649     evex_prefix(vex_r, vex_b, vex_x, evex_r, evex_v, nds_enc, pre, opc);
7650   } else {
7651     if (UseAVX > 2 && attributes->is_rex_vex_w_reverted()) {
7652       attributes->set_rex_vex_w(false);
7653     }
7654     vex_prefix(vex_r, vex_b, vex_x, nds_enc, pre, opc);
7655   }
7656 }
7657 
7658 int Assembler::vex_prefix_and_encode(int dst_enc, int nds_enc, int src_enc, VexSimdPrefix pre, VexOpcode opc, InstructionAttr *attributes) {
7659   bool vex_r = ((dst_enc & 8) == 8) ? 1 : 0;
7660   bool vex_b = ((src_enc & 8) == 8) ? 1 : 0;
7661   bool vex_x = false;
7662   set_attributes(attributes);
7663   attributes->set_current_assembler(this);
7664   bool check_register_bank = NOT_IA32(true) IA32_ONLY(false);
7665 
7666   // if vector length is turned off, revert to AVX for vectors smaller than 512-bit
7667   if (UseAVX > 2 && _legacy_mode_vl && attributes->uses_vl()) {
7668     switch (attributes->get_vector_len()) {
7669     case AVX_128bit:
7670     case AVX_256bit:
7671       if (check_register_bank) {
7672         if (dst_enc >= 16 || nds_enc >= 16 || src_enc >= 16) {
7673           // up propagate arithmetic instructions to meet RA requirements
7674           attributes->set_vector_len(AVX_512bit);
7675         } else {
7676           attributes->set_is_legacy_mode();
7677         }
7678       } else {
7679         attributes->set_is_legacy_mode();
7680       }
7681       break;
7682     }
7683   }
7684 
7685   // For pure EVEX check and see if this instruction
7686   // is allowed in legacy mode and has resources which will
7687   // fit in it.  Pure EVEX instructions will use set_is_evex_instruction in their definition,
7688   // else that field is set when we encode to EVEX
7689   if (UseAVX > 2 && !attributes->is_legacy_mode() &&
7690       !_is_managed && !attributes->is_evex_instruction()) {
7691     if (!_legacy_mode_vl && attributes->get_vector_len() != AVX_512bit) {
7692       if (check_register_bank) {
7693         // check dst_enc, nds_enc and src_enc for upper bank usage
7694         if (dst_enc < 16 && nds_enc < 16 && src_enc < 16) {
7695           attributes->set_is_legacy_mode();
7696         }
7697       } else {
7698         attributes->set_is_legacy_mode();
7699       }
7700     }
7701   }
7702 
7703   _is_managed = false;
7704   if (UseAVX > 2 && !attributes->is_legacy_mode())
7705   {
7706     bool evex_r = (dst_enc >= 16);
7707     bool evex_v = (nds_enc >= 16);
7708     // can use vex_x as bank extender on rm encoding
7709     vex_x = (src_enc >= 16);
7710     attributes->set_is_evex_instruction();
7711     evex_prefix(vex_r, vex_b, vex_x, evex_r, evex_v, nds_enc, pre, opc);
7712   } else {
7713     if (UseAVX > 2 && attributes->is_rex_vex_w_reverted()) {
7714       attributes->set_rex_vex_w(false);
7715     }
7716     vex_prefix(vex_r, vex_b, vex_x, nds_enc, pre, opc);
7717   }
7718 
7719   // return modrm byte components for operands
7720   return (((dst_enc & 7) << 3) | (src_enc & 7));
7721 }
7722 
7723 
7724 void Assembler::simd_prefix(XMMRegister xreg, XMMRegister nds, Address adr, VexSimdPrefix pre,
7725                             VexOpcode opc, InstructionAttr *attributes) {
7726   if (UseAVX > 0) {
7727     int xreg_enc = xreg->encoding();
7728     int nds_enc = nds->is_valid() ? nds->encoding() : 0;
7729     vex_prefix(adr, nds_enc, xreg_enc, pre, opc, attributes);
7730   } else {
7731     assert((nds == xreg) || (nds == xnoreg), "wrong sse encoding");
7732     rex_prefix(adr, xreg, pre, opc, attributes->is_rex_vex_w());
7733   }
7734 }
7735 
7736 int Assembler::simd_prefix_and_encode(XMMRegister dst, XMMRegister nds, XMMRegister src, VexSimdPrefix pre,
7737                                       VexOpcode opc, InstructionAttr *attributes) {
7738   int dst_enc = dst->encoding();
7739   int src_enc = src->encoding();
7740   if (UseAVX > 0) {
7741     int nds_enc = nds->is_valid() ? nds->encoding() : 0;
7742     return vex_prefix_and_encode(dst_enc, nds_enc, src_enc, pre, opc, attributes);
7743   } else {
7744     assert((nds == dst) || (nds == src) || (nds == xnoreg), "wrong sse encoding");
7745     return rex_prefix_and_encode(dst_enc, src_enc, pre, opc, attributes->is_rex_vex_w());
7746   }
7747 }
7748 
7749 void Assembler::cmppd(XMMRegister dst, XMMRegister nds, XMMRegister src, int cop, int vector_len) {
7750   assert(VM_Version::supports_avx(), "");
7751   assert(!VM_Version::supports_evex(), "");
7752   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
7753   int encode = simd_prefix_and_encode(dst, nds, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7754   emit_int8((unsigned char)0xC2);
7755   emit_int8((unsigned char)(0xC0 | encode));
7756   emit_int8((unsigned char)(0xF & cop));
7757 }
7758 
7759 void Assembler::blendvpd(XMMRegister dst, XMMRegister nds, XMMRegister src1, XMMRegister src2, int vector_len) {
7760   assert(VM_Version::supports_avx(), "");
7761   assert(!VM_Version::supports_evex(), "");
7762   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
7763   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src1->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
7764   emit_int8((unsigned char)0x4B);
7765   emit_int8((unsigned char)(0xC0 | encode));
7766   int src2_enc = src2->encoding();
7767   emit_int8((unsigned char)(0xF0 & src2_enc<<4));
7768 }
7769 
7770 void Assembler::cmpps(XMMRegister dst, XMMRegister nds, XMMRegister src, int cop, int vector_len) {
7771   assert(VM_Version::supports_avx(), "");
7772   assert(!VM_Version::supports_evex(), "");
7773   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
7774   int encode = simd_prefix_and_encode(dst, nds, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
7775   emit_int8((unsigned char)0xC2);
7776   emit_int8((unsigned char)(0xC0 | encode));
7777   emit_int8((unsigned char)(0xF & cop));
7778 }
7779 
7780 void Assembler::blendvps(XMMRegister dst, XMMRegister nds, XMMRegister src1, XMMRegister src2, int vector_len) {
7781   assert(VM_Version::supports_avx(), "");
7782   assert(!VM_Version::supports_evex(), "");
7783   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
7784   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src1->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
7785   emit_int8((unsigned char)0x4A);
7786   emit_int8((unsigned char)(0xC0 | encode));
7787   int src2_enc = src2->encoding();
7788   emit_int8((unsigned char)(0xF0 & src2_enc<<4));
7789 }
7790 
7791 void Assembler::vpblendd(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8, int vector_len) {
7792   assert(VM_Version::supports_avx2(), "");
7793   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
7794   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
7795   emit_int8((unsigned char)0x02);
7796   emit_int8((unsigned char)(0xC0 | encode));
7797   emit_int8((unsigned char)imm8);
7798 }
7799 
7800 void Assembler::shlxl(Register dst, Register src1, Register src2) {
7801   assert(VM_Version::supports_bmi2(), "");
7802   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
7803   int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src1->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
7804   emit_int8((unsigned char)0xF7);
7805   emit_int8((unsigned char)(0xC0 | encode));
7806 }
7807 
7808 void Assembler::shlxq(Register dst, Register src1, Register src2) {
7809   assert(VM_Version::supports_bmi2(), "");
7810   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
7811   int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src1->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
7812   emit_int8((unsigned char)0xF7);
7813   emit_int8((unsigned char)(0xC0 | encode));
7814 }
7815 
7816 #ifndef _LP64
7817 
7818 void Assembler::incl(Register dst) {
7819   // Don't use it directly. Use MacroAssembler::incrementl() instead.
7820   emit_int8(0x40 | dst->encoding());
7821 }
7822 
7823 void Assembler::lea(Register dst, Address src) {
7824   leal(dst, src);
7825 }
7826 
7827 void Assembler::mov_literal32(Address dst, int32_t imm32, RelocationHolder const& rspec) {
7828   InstructionMark im(this);
7829   emit_int8((unsigned char)0xC7);
7830   emit_operand(rax, dst);
7831   emit_data((int)imm32, rspec, 0);
7832 }
7833 
7834 void Assembler::mov_literal32(Register dst, int32_t imm32, RelocationHolder const& rspec) {
7835   InstructionMark im(this);
7836   int encode = prefix_and_encode(dst->encoding());
7837   emit_int8((unsigned char)(0xB8 | encode));
7838   emit_data((int)imm32, rspec, 0);
7839 }
7840 
7841 void Assembler::popa() { // 32bit
7842   emit_int8(0x61);
7843 }
7844 
7845 void Assembler::push_literal32(int32_t imm32, RelocationHolder const& rspec) {
7846   InstructionMark im(this);
7847   emit_int8(0x68);
7848   emit_data(imm32, rspec, 0);
7849 }
7850 
7851 void Assembler::pusha() { // 32bit
7852   emit_int8(0x60);
7853 }
7854 
7855 void Assembler::set_byte_if_not_zero(Register dst) {
7856   emit_int8(0x0F);
7857   emit_int8((unsigned char)0x95);
7858   emit_int8((unsigned char)(0xE0 | dst->encoding()));
7859 }
7860 
7861 void Assembler::shldl(Register dst, Register src) {
7862   emit_int8(0x0F);
7863   emit_int8((unsigned char)0xA5);
7864   emit_int8((unsigned char)(0xC0 | src->encoding() << 3 | dst->encoding()));
7865 }
7866 
7867 // 0F A4 / r ib
7868 void Assembler::shldl(Register dst, Register src, int8_t imm8) {
7869   emit_int8(0x0F);
7870   emit_int8((unsigned char)0xA4);
7871   emit_int8((unsigned char)(0xC0 | src->encoding() << 3 | dst->encoding()));
7872   emit_int8(imm8);
7873 }
7874 
7875 void Assembler::shrdl(Register dst, Register src) {
7876   emit_int8(0x0F);
7877   emit_int8((unsigned char)0xAD);
7878   emit_int8((unsigned char)(0xC0 | src->encoding() << 3 | dst->encoding()));
7879 }
7880 
7881 #else // LP64
7882 
7883 void Assembler::set_byte_if_not_zero(Register dst) {
7884   int enc = prefix_and_encode(dst->encoding(), true);
7885   emit_int8(0x0F);
7886   emit_int8((unsigned char)0x95);
7887   emit_int8((unsigned char)(0xE0 | enc));
7888 }
7889 
7890 // 64bit only pieces of the assembler
7891 // This should only be used by 64bit instructions that can use rip-relative
7892 // it cannot be used by instructions that want an immediate value.
7893 
7894 bool Assembler::reachable(AddressLiteral adr) {
7895   int64_t disp;
7896   // None will force a 64bit literal to the code stream. Likely a placeholder
7897   // for something that will be patched later and we need to certain it will
7898   // always be reachable.
7899   if (adr.reloc() == relocInfo::none) {
7900     return false;
7901   }
7902   if (adr.reloc() == relocInfo::internal_word_type) {
7903     // This should be rip relative and easily reachable.
7904     return true;
7905   }
7906   if (adr.reloc() == relocInfo::virtual_call_type ||
7907       adr.reloc() == relocInfo::opt_virtual_call_type ||
7908       adr.reloc() == relocInfo::static_call_type ||
7909       adr.reloc() == relocInfo::static_stub_type ) {
7910     // This should be rip relative within the code cache and easily
7911     // reachable until we get huge code caches. (At which point
7912     // ic code is going to have issues).
7913     return true;
7914   }
7915   if (adr.reloc() != relocInfo::external_word_type &&
7916       adr.reloc() != relocInfo::poll_return_type &&  // these are really external_word but need special
7917       adr.reloc() != relocInfo::poll_type &&         // relocs to identify them
7918       adr.reloc() != relocInfo::runtime_call_type ) {
7919     return false;
7920   }
7921 
7922   // Stress the correction code
7923   if (ForceUnreachable) {
7924     // Must be runtimecall reloc, see if it is in the codecache
7925     // Flipping stuff in the codecache to be unreachable causes issues
7926     // with things like inline caches where the additional instructions
7927     // are not handled.
7928     if (CodeCache::find_blob(adr._target) == NULL) {
7929       return false;
7930     }
7931   }
7932   // For external_word_type/runtime_call_type if it is reachable from where we
7933   // are now (possibly a temp buffer) and where we might end up
7934   // anywhere in the codeCache then we are always reachable.
7935   // This would have to change if we ever save/restore shared code
7936   // to be more pessimistic.
7937   disp = (int64_t)adr._target - ((int64_t)CodeCache::low_bound() + sizeof(int));
7938   if (!is_simm32(disp)) return false;
7939   disp = (int64_t)adr._target - ((int64_t)CodeCache::high_bound() + sizeof(int));
7940   if (!is_simm32(disp)) return false;
7941 
7942   disp = (int64_t)adr._target - ((int64_t)pc() + sizeof(int));
7943 
7944   // Because rip relative is a disp + address_of_next_instruction and we
7945   // don't know the value of address_of_next_instruction we apply a fudge factor
7946   // to make sure we will be ok no matter the size of the instruction we get placed into.
7947   // We don't have to fudge the checks above here because they are already worst case.
7948 
7949   // 12 == override/rex byte, opcode byte, rm byte, sib byte, a 4-byte disp , 4-byte literal
7950   // + 4 because better safe than sorry.
7951   const int fudge = 12 + 4;
7952   if (disp < 0) {
7953     disp -= fudge;
7954   } else {
7955     disp += fudge;
7956   }
7957   return is_simm32(disp);
7958 }
7959 
7960 // Check if the polling page is not reachable from the code cache using rip-relative
7961 // addressing.
7962 bool Assembler::is_polling_page_far() {
7963   intptr_t addr = (intptr_t)os::get_polling_page();
7964   return ForceUnreachable ||
7965          !is_simm32(addr - (intptr_t)CodeCache::low_bound()) ||
7966          !is_simm32(addr - (intptr_t)CodeCache::high_bound());
7967 }
7968 
7969 void Assembler::emit_data64(jlong data,
7970                             relocInfo::relocType rtype,
7971                             int format) {
7972   if (rtype == relocInfo::none) {
7973     emit_int64(data);
7974   } else {
7975     emit_data64(data, Relocation::spec_simple(rtype), format);
7976   }
7977 }
7978 
7979 void Assembler::emit_data64(jlong data,
7980                             RelocationHolder const& rspec,
7981                             int format) {
7982   assert(imm_operand == 0, "default format must be immediate in this file");
7983   assert(imm_operand == format, "must be immediate");
7984   assert(inst_mark() != NULL, "must be inside InstructionMark");
7985   // Do not use AbstractAssembler::relocate, which is not intended for
7986   // embedded words.  Instead, relocate to the enclosing instruction.
7987   code_section()->relocate(inst_mark(), rspec, format);
7988 #ifdef ASSERT
7989   check_relocation(rspec, format);
7990 #endif
7991   emit_int64(data);
7992 }
7993 
7994 int Assembler::prefix_and_encode(int reg_enc, bool byteinst) {
7995   if (reg_enc >= 8) {
7996     prefix(REX_B);
7997     reg_enc -= 8;
7998   } else if (byteinst && reg_enc >= 4) {
7999     prefix(REX);
8000   }
8001   return reg_enc;
8002 }
8003 
8004 int Assembler::prefixq_and_encode(int reg_enc) {
8005   if (reg_enc < 8) {
8006     prefix(REX_W);
8007   } else {
8008     prefix(REX_WB);
8009     reg_enc -= 8;
8010   }
8011   return reg_enc;
8012 }
8013 
8014 int Assembler::prefix_and_encode(int dst_enc, bool dst_is_byte, int src_enc, bool src_is_byte) {
8015   if (dst_enc < 8) {
8016     if (src_enc >= 8) {
8017       prefix(REX_B);
8018       src_enc -= 8;
8019     } else if ((src_is_byte && src_enc >= 4) || (dst_is_byte && dst_enc >= 4)) {
8020       prefix(REX);
8021     }
8022   } else {
8023     if (src_enc < 8) {
8024       prefix(REX_R);
8025     } else {
8026       prefix(REX_RB);
8027       src_enc -= 8;
8028     }
8029     dst_enc -= 8;
8030   }
8031   return dst_enc << 3 | src_enc;
8032 }
8033 
8034 int Assembler::prefixq_and_encode(int dst_enc, int src_enc) {
8035   if (dst_enc < 8) {
8036     if (src_enc < 8) {
8037       prefix(REX_W);
8038     } else {
8039       prefix(REX_WB);
8040       src_enc -= 8;
8041     }
8042   } else {
8043     if (src_enc < 8) {
8044       prefix(REX_WR);
8045     } else {
8046       prefix(REX_WRB);
8047       src_enc -= 8;
8048     }
8049     dst_enc -= 8;
8050   }
8051   return dst_enc << 3 | src_enc;
8052 }
8053 
8054 void Assembler::prefix(Register reg) {
8055   if (reg->encoding() >= 8) {
8056     prefix(REX_B);
8057   }
8058 }
8059 
8060 void Assembler::prefix(Register dst, Register src, Prefix p) {
8061   if (src->encoding() >= 8) {
8062     p = (Prefix)(p | REX_B);
8063   }
8064   if (dst->encoding() >= 8) {
8065     p = (Prefix)( p | REX_R);
8066   }
8067   if (p != Prefix_EMPTY) {
8068     // do not generate an empty prefix
8069     prefix(p);
8070   }
8071 }
8072 
8073 void Assembler::prefix(Register dst, Address adr, Prefix p) {
8074   if (adr.base_needs_rex()) {
8075     if (adr.index_needs_rex()) {
8076       assert(false, "prefix(Register dst, Address adr, Prefix p) does not support handling of an X");
8077     } else {
8078       prefix(REX_B);
8079     }
8080   } else {
8081     if (adr.index_needs_rex()) {
8082       assert(false, "prefix(Register dst, Address adr, Prefix p) does not support handling of an X");
8083     }
8084   }
8085   if (dst->encoding() >= 8) {
8086     p = (Prefix)(p | REX_R);
8087   }
8088   if (p != Prefix_EMPTY) {
8089     // do not generate an empty prefix
8090     prefix(p);
8091   }
8092 }
8093 
8094 void Assembler::prefix(Address adr) {
8095   if (adr.base_needs_rex()) {
8096     if (adr.index_needs_rex()) {
8097       prefix(REX_XB);
8098     } else {
8099       prefix(REX_B);
8100     }
8101   } else {
8102     if (adr.index_needs_rex()) {
8103       prefix(REX_X);
8104     }
8105   }
8106 }
8107 
8108 void Assembler::prefixq(Address adr) {
8109   if (adr.base_needs_rex()) {
8110     if (adr.index_needs_rex()) {
8111       prefix(REX_WXB);
8112     } else {
8113       prefix(REX_WB);
8114     }
8115   } else {
8116     if (adr.index_needs_rex()) {
8117       prefix(REX_WX);
8118     } else {
8119       prefix(REX_W);
8120     }
8121   }
8122 }
8123 
8124 
8125 void Assembler::prefix(Address adr, Register reg, bool byteinst) {
8126   if (reg->encoding() < 8) {
8127     if (adr.base_needs_rex()) {
8128       if (adr.index_needs_rex()) {
8129         prefix(REX_XB);
8130       } else {
8131         prefix(REX_B);
8132       }
8133     } else {
8134       if (adr.index_needs_rex()) {
8135         prefix(REX_X);
8136       } else if (byteinst && reg->encoding() >= 4 ) {
8137         prefix(REX);
8138       }
8139     }
8140   } else {
8141     if (adr.base_needs_rex()) {
8142       if (adr.index_needs_rex()) {
8143         prefix(REX_RXB);
8144       } else {
8145         prefix(REX_RB);
8146       }
8147     } else {
8148       if (adr.index_needs_rex()) {
8149         prefix(REX_RX);
8150       } else {
8151         prefix(REX_R);
8152       }
8153     }
8154   }
8155 }
8156 
8157 void Assembler::prefixq(Address adr, Register src) {
8158   if (src->encoding() < 8) {
8159     if (adr.base_needs_rex()) {
8160       if (adr.index_needs_rex()) {
8161         prefix(REX_WXB);
8162       } else {
8163         prefix(REX_WB);
8164       }
8165     } else {
8166       if (adr.index_needs_rex()) {
8167         prefix(REX_WX);
8168       } else {
8169         prefix(REX_W);
8170       }
8171     }
8172   } else {
8173     if (adr.base_needs_rex()) {
8174       if (adr.index_needs_rex()) {
8175         prefix(REX_WRXB);
8176       } else {
8177         prefix(REX_WRB);
8178       }
8179     } else {
8180       if (adr.index_needs_rex()) {
8181         prefix(REX_WRX);
8182       } else {
8183         prefix(REX_WR);
8184       }
8185     }
8186   }
8187 }
8188 
8189 void Assembler::prefix(Address adr, XMMRegister reg) {
8190   if (reg->encoding() < 8) {
8191     if (adr.base_needs_rex()) {
8192       if (adr.index_needs_rex()) {
8193         prefix(REX_XB);
8194       } else {
8195         prefix(REX_B);
8196       }
8197     } else {
8198       if (adr.index_needs_rex()) {
8199         prefix(REX_X);
8200       }
8201     }
8202   } else {
8203     if (adr.base_needs_rex()) {
8204       if (adr.index_needs_rex()) {
8205         prefix(REX_RXB);
8206       } else {
8207         prefix(REX_RB);
8208       }
8209     } else {
8210       if (adr.index_needs_rex()) {
8211         prefix(REX_RX);
8212       } else {
8213         prefix(REX_R);
8214       }
8215     }
8216   }
8217 }
8218 
8219 void Assembler::prefixq(Address adr, XMMRegister src) {
8220   if (src->encoding() < 8) {
8221     if (adr.base_needs_rex()) {
8222       if (adr.index_needs_rex()) {
8223         prefix(REX_WXB);
8224       } else {
8225         prefix(REX_WB);
8226       }
8227     } else {
8228       if (adr.index_needs_rex()) {
8229         prefix(REX_WX);
8230       } else {
8231         prefix(REX_W);
8232       }
8233     }
8234   } else {
8235     if (adr.base_needs_rex()) {
8236       if (adr.index_needs_rex()) {
8237         prefix(REX_WRXB);
8238       } else {
8239         prefix(REX_WRB);
8240       }
8241     } else {
8242       if (adr.index_needs_rex()) {
8243         prefix(REX_WRX);
8244       } else {
8245         prefix(REX_WR);
8246       }
8247     }
8248   }
8249 }
8250 
8251 void Assembler::adcq(Register dst, int32_t imm32) {
8252   (void) prefixq_and_encode(dst->encoding());
8253   emit_arith(0x81, 0xD0, dst, imm32);
8254 }
8255 
8256 void Assembler::adcq(Register dst, Address src) {
8257   InstructionMark im(this);
8258   prefixq(src, dst);
8259   emit_int8(0x13);
8260   emit_operand(dst, src);
8261 }
8262 
8263 void Assembler::adcq(Register dst, Register src) {
8264   (void) prefixq_and_encode(dst->encoding(), src->encoding());
8265   emit_arith(0x13, 0xC0, dst, src);
8266 }
8267 
8268 void Assembler::addq(Address dst, int32_t imm32) {
8269   InstructionMark im(this);
8270   prefixq(dst);
8271   emit_arith_operand(0x81, rax, dst,imm32);
8272 }
8273 
8274 void Assembler::addq(Address dst, Register src) {
8275   InstructionMark im(this);
8276   prefixq(dst, src);
8277   emit_int8(0x01);
8278   emit_operand(src, dst);
8279 }
8280 
8281 void Assembler::addq(Register dst, int32_t imm32) {
8282   (void) prefixq_and_encode(dst->encoding());
8283   emit_arith(0x81, 0xC0, dst, imm32);
8284 }
8285 
8286 void Assembler::addq(Register dst, Address src) {
8287   InstructionMark im(this);
8288   prefixq(src, dst);
8289   emit_int8(0x03);
8290   emit_operand(dst, src);
8291 }
8292 
8293 void Assembler::addq(Register dst, Register src) {
8294   (void) prefixq_and_encode(dst->encoding(), src->encoding());
8295   emit_arith(0x03, 0xC0, dst, src);
8296 }
8297 
8298 void Assembler::adcxq(Register dst, Register src) {
8299   //assert(VM_Version::supports_adx(), "adx instructions not supported");
8300   emit_int8((unsigned char)0x66);
8301   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
8302   emit_int8(0x0F);
8303   emit_int8(0x38);
8304   emit_int8((unsigned char)0xF6);
8305   emit_int8((unsigned char)(0xC0 | encode));
8306 }
8307 
8308 void Assembler::adoxq(Register dst, Register src) {
8309   //assert(VM_Version::supports_adx(), "adx instructions not supported");
8310   emit_int8((unsigned char)0xF3);
8311   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
8312   emit_int8(0x0F);
8313   emit_int8(0x38);
8314   emit_int8((unsigned char)0xF6);
8315   emit_int8((unsigned char)(0xC0 | encode));
8316 }
8317 
8318 void Assembler::andq(Address dst, int32_t imm32) {
8319   InstructionMark im(this);
8320   prefixq(dst);
8321   emit_int8((unsigned char)0x81);
8322   emit_operand(rsp, dst, 4);
8323   emit_int32(imm32);
8324 }
8325 
8326 void Assembler::andq(Register dst, int32_t imm32) {
8327   (void) prefixq_and_encode(dst->encoding());
8328   emit_arith(0x81, 0xE0, dst, imm32);
8329 }
8330 
8331 void Assembler::andq(Register dst, Address src) {
8332   InstructionMark im(this);
8333   prefixq(src, dst);
8334   emit_int8(0x23);
8335   emit_operand(dst, src);
8336 }
8337 
8338 void Assembler::andq(Register dst, Register src) {
8339   (void) prefixq_and_encode(dst->encoding(), src->encoding());
8340   emit_arith(0x23, 0xC0, dst, src);
8341 }
8342 
8343 void Assembler::andnq(Register dst, Register src1, Register src2) {
8344   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
8345   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
8346   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
8347   emit_int8((unsigned char)0xF2);
8348   emit_int8((unsigned char)(0xC0 | encode));
8349 }
8350 
8351 void Assembler::andnq(Register dst, Register src1, Address src2) {
8352   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
8353   InstructionMark im(this);
8354   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
8355   vex_prefix(src2, src1->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
8356   emit_int8((unsigned char)0xF2);
8357   emit_operand(dst, src2);
8358 }
8359 
8360 void Assembler::bsfq(Register dst, Register src) {
8361   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
8362   emit_int8(0x0F);
8363   emit_int8((unsigned char)0xBC);
8364   emit_int8((unsigned char)(0xC0 | encode));
8365 }
8366 
8367 void Assembler::bsrq(Register dst, Register src) {
8368   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
8369   emit_int8(0x0F);
8370   emit_int8((unsigned char)0xBD);
8371   emit_int8((unsigned char)(0xC0 | encode));
8372 }
8373 
8374 void Assembler::bswapq(Register reg) {
8375   int encode = prefixq_and_encode(reg->encoding());
8376   emit_int8(0x0F);
8377   emit_int8((unsigned char)(0xC8 | encode));
8378 }
8379 
8380 void Assembler::blsiq(Register dst, Register src) {
8381   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
8382   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
8383   int encode = vex_prefix_and_encode(rbx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
8384   emit_int8((unsigned char)0xF3);
8385   emit_int8((unsigned char)(0xC0 | encode));
8386 }
8387 
8388 void Assembler::blsiq(Register dst, Address src) {
8389   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
8390   InstructionMark im(this);
8391   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
8392   vex_prefix(src, dst->encoding(), rbx->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
8393   emit_int8((unsigned char)0xF3);
8394   emit_operand(rbx, src);
8395 }
8396 
8397 void Assembler::blsmskq(Register dst, Register src) {
8398   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
8399   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
8400   int encode = vex_prefix_and_encode(rdx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
8401   emit_int8((unsigned char)0xF3);
8402   emit_int8((unsigned char)(0xC0 | encode));
8403 }
8404 
8405 void Assembler::blsmskq(Register dst, Address src) {
8406   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
8407   InstructionMark im(this);
8408   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
8409   vex_prefix(src, dst->encoding(), rdx->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
8410   emit_int8((unsigned char)0xF3);
8411   emit_operand(rdx, src);
8412 }
8413 
8414 void Assembler::blsrq(Register dst, Register src) {
8415   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
8416   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
8417   int encode = vex_prefix_and_encode(rcx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
8418   emit_int8((unsigned char)0xF3);
8419   emit_int8((unsigned char)(0xC0 | encode));
8420 }
8421 
8422 void Assembler::blsrq(Register dst, Address src) {
8423   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
8424   InstructionMark im(this);
8425   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
8426   vex_prefix(src, dst->encoding(), rcx->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
8427   emit_int8((unsigned char)0xF3);
8428   emit_operand(rcx, src);
8429 }
8430 
8431 void Assembler::cdqq() {
8432   prefix(REX_W);
8433   emit_int8((unsigned char)0x99);
8434 }
8435 
8436 void Assembler::clflush(Address adr) {
8437   assert(VM_Version::supports_clflush(), "should do");
8438   prefix(adr);
8439   emit_int8(0x0F);
8440   emit_int8((unsigned char)0xAE);
8441   emit_operand(rdi, adr);
8442 }
8443 
8444 #ifdef _LP64
8445 void Assembler::clflushopt(Address adr) {
8446   assert(VM_Version::supports_clflushopt(), "should do!");
8447   // adr should be base reg only with no index or offset
8448   assert(adr.index() == noreg, "index should be noreg");
8449   assert(adr.scale() == Address::no_scale, "scale should be no_scale");
8450   assert(adr.disp() == 0, "displacement should be 0");
8451   // instruction prefix is 0x66
8452   emit_int8(0x66);
8453   prefix(adr);
8454   // opcode family is 0x0f 0xAE
8455   emit_int8(0x0F);
8456   emit_int8((unsigned char)0xAE);
8457   // extended opcode byte is 7 == rdi
8458   emit_operand(rdi, adr);
8459 }
8460 
8461 void Assembler::clwb(Address adr) {
8462   assert(VM_Version::supports_clwb(), "should do!");
8463   // adr should be base reg only with no index or offset
8464   assert(adr.index() == noreg, "index should be noreg");
8465   assert(adr.scale() == Address::no_scale, "scale should be no_scale");
8466   assert(adr.disp() == 0, "displacement should be 0");
8467   // instruction prefix is 0x66
8468   emit_int8(0x66);
8469   prefix(adr);
8470   // opcode family is 0x0f 0xAE
8471   emit_int8(0x0F);
8472   emit_int8((unsigned char)0xAE);
8473   // extended opcode byte is 6 == rsi
8474   emit_operand(rsi, adr);
8475 }
8476 #endif
8477 
8478 void Assembler::cmovq(Condition cc, Register dst, Register src) {
8479   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
8480   emit_int8(0x0F);
8481   emit_int8(0x40 | cc);
8482   emit_int8((unsigned char)(0xC0 | encode));
8483 }
8484 
8485 void Assembler::cmovq(Condition cc, Register dst, Address src) {
8486   InstructionMark im(this);
8487   prefixq(src, dst);
8488   emit_int8(0x0F);
8489   emit_int8(0x40 | cc);
8490   emit_operand(dst, src);
8491 }
8492 
8493 void Assembler::cmpq(Address dst, int32_t imm32) {
8494   InstructionMark im(this);
8495   prefixq(dst);
8496   emit_int8((unsigned char)0x81);
8497   emit_operand(rdi, dst, 4);
8498   emit_int32(imm32);
8499 }
8500 
8501 void Assembler::cmpq(Register dst, int32_t imm32) {
8502   (void) prefixq_and_encode(dst->encoding());
8503   emit_arith(0x81, 0xF8, dst, imm32);
8504 }
8505 
8506 void Assembler::cmpq(Address dst, Register src) {
8507   InstructionMark im(this);
8508   prefixq(dst, src);
8509   emit_int8(0x3B);
8510   emit_operand(src, dst);
8511 }
8512 
8513 void Assembler::cmpq(Register dst, Register src) {
8514   (void) prefixq_and_encode(dst->encoding(), src->encoding());
8515   emit_arith(0x3B, 0xC0, dst, src);
8516 }
8517 
8518 void Assembler::cmpq(Register dst, Address  src) {
8519   InstructionMark im(this);
8520   prefixq(src, dst);
8521   emit_int8(0x3B);
8522   emit_operand(dst, src);
8523 }
8524 
8525 void Assembler::cmpxchgq(Register reg, Address adr) {
8526   InstructionMark im(this);
8527   prefixq(adr, reg);
8528   emit_int8(0x0F);
8529   emit_int8((unsigned char)0xB1);
8530   emit_operand(reg, adr);
8531 }
8532 
8533 void Assembler::cvtsi2sdq(XMMRegister dst, Register src) {
8534   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
8535   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
8536   int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
8537   emit_int8(0x2A);
8538   emit_int8((unsigned char)(0xC0 | encode));
8539 }
8540 
8541 void Assembler::cvtsi2sdq(XMMRegister dst, Address src) {
8542   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
8543   InstructionMark im(this);
8544   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
8545   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
8546   simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
8547   emit_int8(0x2A);
8548   emit_operand(dst, src);
8549 }
8550 
8551 void Assembler::cvtsi2ssq(XMMRegister dst, Address src) {
8552   NOT_LP64(assert(VM_Version::supports_sse(), ""));
8553   InstructionMark im(this);
8554   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
8555   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
8556   simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
8557   emit_int8(0x2A);
8558   emit_operand(dst, src);
8559 }
8560 
8561 void Assembler::cvttsd2siq(Register dst, XMMRegister src) {
8562   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
8563   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
8564   int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
8565   emit_int8(0x2C);
8566   emit_int8((unsigned char)(0xC0 | encode));
8567 }
8568 
8569 void Assembler::cvttss2siq(Register dst, XMMRegister src) {
8570   NOT_LP64(assert(VM_Version::supports_sse(), ""));
8571   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
8572   int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
8573   emit_int8(0x2C);
8574   emit_int8((unsigned char)(0xC0 | encode));
8575 }
8576 
8577 void Assembler::decl(Register dst) {
8578   // Don't use it directly. Use MacroAssembler::decrementl() instead.
8579   // Use two-byte form (one-byte form is a REX prefix in 64-bit mode)
8580   int encode = prefix_and_encode(dst->encoding());
8581   emit_int8((unsigned char)0xFF);
8582   emit_int8((unsigned char)(0xC8 | encode));
8583 }
8584 
8585 void Assembler::decq(Register dst) {
8586   // Don't use it directly. Use MacroAssembler::decrementq() instead.
8587   // Use two-byte form (one-byte from is a REX prefix in 64-bit mode)
8588   int encode = prefixq_and_encode(dst->encoding());
8589   emit_int8((unsigned char)0xFF);
8590   emit_int8(0xC8 | encode);
8591 }
8592 
8593 void Assembler::decq(Address dst) {
8594   // Don't use it directly. Use MacroAssembler::decrementq() instead.
8595   InstructionMark im(this);
8596   prefixq(dst);
8597   emit_int8((unsigned char)0xFF);
8598   emit_operand(rcx, dst);
8599 }
8600 
8601 void Assembler::fxrstor(Address src) {
8602   prefixq(src);
8603   emit_int8(0x0F);
8604   emit_int8((unsigned char)0xAE);
8605   emit_operand(as_Register(1), src);
8606 }
8607 
8608 void Assembler::xrstor(Address src) {
8609   prefixq(src);
8610   emit_int8(0x0F);
8611   emit_int8((unsigned char)0xAE);
8612   emit_operand(as_Register(5), src);
8613 }
8614 
8615 void Assembler::fxsave(Address dst) {
8616   prefixq(dst);
8617   emit_int8(0x0F);
8618   emit_int8((unsigned char)0xAE);
8619   emit_operand(as_Register(0), dst);
8620 }
8621 
8622 void Assembler::xsave(Address dst) {
8623   prefixq(dst);
8624   emit_int8(0x0F);
8625   emit_int8((unsigned char)0xAE);
8626   emit_operand(as_Register(4), dst);
8627 }
8628 
8629 void Assembler::idivq(Register src) {
8630   int encode = prefixq_and_encode(src->encoding());
8631   emit_int8((unsigned char)0xF7);
8632   emit_int8((unsigned char)(0xF8 | encode));
8633 }
8634 
8635 void Assembler::imulq(Register dst, Register src) {
8636   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
8637   emit_int8(0x0F);
8638   emit_int8((unsigned char)0xAF);
8639   emit_int8((unsigned char)(0xC0 | encode));
8640 }
8641 
8642 void Assembler::imulq(Register dst, Register src, int value) {
8643   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
8644   if (is8bit(value)) {
8645     emit_int8(0x6B);
8646     emit_int8((unsigned char)(0xC0 | encode));
8647     emit_int8(value & 0xFF);
8648   } else {
8649     emit_int8(0x69);
8650     emit_int8((unsigned char)(0xC0 | encode));
8651     emit_int32(value);
8652   }
8653 }
8654 
8655 void Assembler::imulq(Register dst, Address src) {
8656   InstructionMark im(this);
8657   prefixq(src, dst);
8658   emit_int8(0x0F);
8659   emit_int8((unsigned char) 0xAF);
8660   emit_operand(dst, src);
8661 }
8662 
8663 void Assembler::incl(Register dst) {
8664   // Don't use it directly. Use MacroAssembler::incrementl() instead.
8665   // Use two-byte form (one-byte from is a REX prefix in 64-bit mode)
8666   int encode = prefix_and_encode(dst->encoding());
8667   emit_int8((unsigned char)0xFF);
8668   emit_int8((unsigned char)(0xC0 | encode));
8669 }
8670 
8671 void Assembler::incq(Register dst) {
8672   // Don't use it directly. Use MacroAssembler::incrementq() instead.
8673   // Use two-byte form (one-byte from is a REX prefix in 64-bit mode)
8674   int encode = prefixq_and_encode(dst->encoding());
8675   emit_int8((unsigned char)0xFF);
8676   emit_int8((unsigned char)(0xC0 | encode));
8677 }
8678 
8679 void Assembler::incq(Address dst) {
8680   // Don't use it directly. Use MacroAssembler::incrementq() instead.
8681   InstructionMark im(this);
8682   prefixq(dst);
8683   emit_int8((unsigned char)0xFF);
8684   emit_operand(rax, dst);
8685 }
8686 
8687 void Assembler::lea(Register dst, Address src) {
8688   leaq(dst, src);
8689 }
8690 
8691 void Assembler::leaq(Register dst, Address src) {
8692   InstructionMark im(this);
8693   prefixq(src, dst);
8694   emit_int8((unsigned char)0x8D);
8695   emit_operand(dst, src);
8696 }
8697 
8698 void Assembler::mov64(Register dst, int64_t imm64) {
8699   InstructionMark im(this);
8700   int encode = prefixq_and_encode(dst->encoding());
8701   emit_int8((unsigned char)(0xB8 | encode));
8702   emit_int64(imm64);
8703 }
8704 
8705 void Assembler::mov_literal64(Register dst, intptr_t imm64, RelocationHolder const& rspec) {
8706   InstructionMark im(this);
8707   int encode = prefixq_and_encode(dst->encoding());
8708   emit_int8(0xB8 | encode);
8709   emit_data64(imm64, rspec);
8710 }
8711 
8712 void Assembler::mov_narrow_oop(Register dst, int32_t imm32, RelocationHolder const& rspec) {
8713   InstructionMark im(this);
8714   int encode = prefix_and_encode(dst->encoding());
8715   emit_int8((unsigned char)(0xB8 | encode));
8716   emit_data((int)imm32, rspec, narrow_oop_operand);
8717 }
8718 
8719 void Assembler::mov_narrow_oop(Address dst, int32_t imm32,  RelocationHolder const& rspec) {
8720   InstructionMark im(this);
8721   prefix(dst);
8722   emit_int8((unsigned char)0xC7);
8723   emit_operand(rax, dst, 4);
8724   emit_data((int)imm32, rspec, narrow_oop_operand);
8725 }
8726 
8727 void Assembler::cmp_narrow_oop(Register src1, int32_t imm32, RelocationHolder const& rspec) {
8728   InstructionMark im(this);
8729   int encode = prefix_and_encode(src1->encoding());
8730   emit_int8((unsigned char)0x81);
8731   emit_int8((unsigned char)(0xF8 | encode));
8732   emit_data((int)imm32, rspec, narrow_oop_operand);
8733 }
8734 
8735 void Assembler::cmp_narrow_oop(Address src1, int32_t imm32, RelocationHolder const& rspec) {
8736   InstructionMark im(this);
8737   prefix(src1);
8738   emit_int8((unsigned char)0x81);
8739   emit_operand(rax, src1, 4);
8740   emit_data((int)imm32, rspec, narrow_oop_operand);
8741 }
8742 
8743 void Assembler::lzcntq(Register dst, Register src) {
8744   assert(VM_Version::supports_lzcnt(), "encoding is treated as BSR");
8745   emit_int8((unsigned char)0xF3);
8746   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
8747   emit_int8(0x0F);
8748   emit_int8((unsigned char)0xBD);
8749   emit_int8((unsigned char)(0xC0 | encode));
8750 }
8751 
8752 void Assembler::movdq(XMMRegister dst, Register src) {
8753   // table D-1 says MMX/SSE2
8754   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
8755   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
8756   int encode = simd_prefix_and_encode(dst, xnoreg, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
8757   emit_int8(0x6E);
8758   emit_int8((unsigned char)(0xC0 | encode));
8759 }
8760 
8761 void Assembler::movdq(Register dst, XMMRegister src) {
8762   // table D-1 says MMX/SSE2
8763   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
8764   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
8765   // swap src/dst to get correct prefix
8766   int encode = simd_prefix_and_encode(src, xnoreg, as_XMMRegister(dst->encoding()), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
8767   emit_int8(0x7E);
8768   emit_int8((unsigned char)(0xC0 | encode));
8769 }
8770 
8771 void Assembler::movq(Register dst, Register src) {
8772   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
8773   emit_int8((unsigned char)0x8B);
8774   emit_int8((unsigned char)(0xC0 | encode));
8775 }
8776 
8777 void Assembler::movq(Register dst, Address src) {
8778   InstructionMark im(this);
8779   prefixq(src, dst);
8780   emit_int8((unsigned char)0x8B);
8781   emit_operand(dst, src);
8782 }
8783 
8784 void Assembler::movq(Address dst, Register src) {
8785   InstructionMark im(this);
8786   prefixq(dst, src);
8787   emit_int8((unsigned char)0x89);
8788   emit_operand(src, dst);
8789 }
8790 
8791 void Assembler::movsbq(Register dst, Address src) {
8792   InstructionMark im(this);
8793   prefixq(src, dst);
8794   emit_int8(0x0F);
8795   emit_int8((unsigned char)0xBE);
8796   emit_operand(dst, src);
8797 }
8798 
8799 void Assembler::movsbq(Register dst, Register src) {
8800   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
8801   emit_int8(0x0F);
8802   emit_int8((unsigned char)0xBE);
8803   emit_int8((unsigned char)(0xC0 | encode));
8804 }
8805 
8806 void Assembler::movslq(Register dst, int32_t imm32) {
8807   // dbx shows movslq(rcx, 3) as movq     $0x0000000049000000,(%rbx)
8808   // and movslq(r8, 3); as movl     $0x0000000048000000,(%rbx)
8809   // as a result we shouldn't use until tested at runtime...
8810   ShouldNotReachHere();
8811   InstructionMark im(this);
8812   int encode = prefixq_and_encode(dst->encoding());
8813   emit_int8((unsigned char)(0xC7 | encode));
8814   emit_int32(imm32);
8815 }
8816 
8817 void Assembler::movslq(Address dst, int32_t imm32) {
8818   assert(is_simm32(imm32), "lost bits");
8819   InstructionMark im(this);
8820   prefixq(dst);
8821   emit_int8((unsigned char)0xC7);
8822   emit_operand(rax, dst, 4);
8823   emit_int32(imm32);
8824 }
8825 
8826 void Assembler::movslq(Register dst, Address src) {
8827   InstructionMark im(this);
8828   prefixq(src, dst);
8829   emit_int8(0x63);
8830   emit_operand(dst, src);
8831 }
8832 
8833 void Assembler::movslq(Register dst, Register src) {
8834   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
8835   emit_int8(0x63);
8836   emit_int8((unsigned char)(0xC0 | encode));
8837 }
8838 
8839 void Assembler::movswq(Register dst, Address src) {
8840   InstructionMark im(this);
8841   prefixq(src, dst);
8842   emit_int8(0x0F);
8843   emit_int8((unsigned char)0xBF);
8844   emit_operand(dst, src);
8845 }
8846 
8847 void Assembler::movswq(Register dst, Register src) {
8848   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
8849   emit_int8((unsigned char)0x0F);
8850   emit_int8((unsigned char)0xBF);
8851   emit_int8((unsigned char)(0xC0 | encode));
8852 }
8853 
8854 void Assembler::movzbq(Register dst, Address src) {
8855   InstructionMark im(this);
8856   prefixq(src, dst);
8857   emit_int8((unsigned char)0x0F);
8858   emit_int8((unsigned char)0xB6);
8859   emit_operand(dst, src);
8860 }
8861 
8862 void Assembler::movzbq(Register dst, Register src) {
8863   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
8864   emit_int8(0x0F);
8865   emit_int8((unsigned char)0xB6);
8866   emit_int8(0xC0 | encode);
8867 }
8868 
8869 void Assembler::movzwq(Register dst, Address src) {
8870   InstructionMark im(this);
8871   prefixq(src, dst);
8872   emit_int8((unsigned char)0x0F);
8873   emit_int8((unsigned char)0xB7);
8874   emit_operand(dst, src);
8875 }
8876 
8877 void Assembler::movzwq(Register dst, Register src) {
8878   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
8879   emit_int8((unsigned char)0x0F);
8880   emit_int8((unsigned char)0xB7);
8881   emit_int8((unsigned char)(0xC0 | encode));
8882 }
8883 
8884 void Assembler::mulq(Address src) {
8885   InstructionMark im(this);
8886   prefixq(src);
8887   emit_int8((unsigned char)0xF7);
8888   emit_operand(rsp, src);
8889 }
8890 
8891 void Assembler::mulq(Register src) {
8892   int encode = prefixq_and_encode(src->encoding());
8893   emit_int8((unsigned char)0xF7);
8894   emit_int8((unsigned char)(0xE0 | encode));
8895 }
8896 
8897 void Assembler::mulxq(Register dst1, Register dst2, Register src) {
8898   assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
8899   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
8900   int encode = vex_prefix_and_encode(dst1->encoding(), dst2->encoding(), src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_38, &attributes);
8901   emit_int8((unsigned char)0xF6);
8902   emit_int8((unsigned char)(0xC0 | encode));
8903 }
8904 
8905 void Assembler::negq(Register dst) {
8906   int encode = prefixq_and_encode(dst->encoding());
8907   emit_int8((unsigned char)0xF7);
8908   emit_int8((unsigned char)(0xD8 | encode));
8909 }
8910 
8911 void Assembler::notq(Register dst) {
8912   int encode = prefixq_and_encode(dst->encoding());
8913   emit_int8((unsigned char)0xF7);
8914   emit_int8((unsigned char)(0xD0 | encode));
8915 }
8916 
8917 void Assembler::orq(Address dst, int32_t imm32) {
8918   InstructionMark im(this);
8919   prefixq(dst);
8920   emit_int8((unsigned char)0x81);
8921   emit_operand(rcx, dst, 4);
8922   emit_int32(imm32);
8923 }
8924 
8925 void Assembler::orq(Register dst, int32_t imm32) {
8926   (void) prefixq_and_encode(dst->encoding());
8927   emit_arith(0x81, 0xC8, dst, imm32);
8928 }
8929 
8930 void Assembler::orq(Register dst, Address src) {
8931   InstructionMark im(this);
8932   prefixq(src, dst);
8933   emit_int8(0x0B);
8934   emit_operand(dst, src);
8935 }
8936 
8937 void Assembler::orq(Register dst, Register src) {
8938   (void) prefixq_and_encode(dst->encoding(), src->encoding());
8939   emit_arith(0x0B, 0xC0, dst, src);
8940 }
8941 
8942 void Assembler::popa() { // 64bit
8943   movq(r15, Address(rsp, 0));
8944   movq(r14, Address(rsp, wordSize));
8945   movq(r13, Address(rsp, 2 * wordSize));
8946   movq(r12, Address(rsp, 3 * wordSize));
8947   movq(r11, Address(rsp, 4 * wordSize));
8948   movq(r10, Address(rsp, 5 * wordSize));
8949   movq(r9,  Address(rsp, 6 * wordSize));
8950   movq(r8,  Address(rsp, 7 * wordSize));
8951   movq(rdi, Address(rsp, 8 * wordSize));
8952   movq(rsi, Address(rsp, 9 * wordSize));
8953   movq(rbp, Address(rsp, 10 * wordSize));
8954   // skip rsp
8955   movq(rbx, Address(rsp, 12 * wordSize));
8956   movq(rdx, Address(rsp, 13 * wordSize));
8957   movq(rcx, Address(rsp, 14 * wordSize));
8958   movq(rax, Address(rsp, 15 * wordSize));
8959 
8960   addq(rsp, 16 * wordSize);
8961 }
8962 
8963 void Assembler::popcntq(Register dst, Address src) {
8964   assert(VM_Version::supports_popcnt(), "must support");
8965   InstructionMark im(this);
8966   emit_int8((unsigned char)0xF3);
8967   prefixq(src, dst);
8968   emit_int8((unsigned char)0x0F);
8969   emit_int8((unsigned char)0xB8);
8970   emit_operand(dst, src);
8971 }
8972 
8973 void Assembler::popcntq(Register dst, Register src) {
8974   assert(VM_Version::supports_popcnt(), "must support");
8975   emit_int8((unsigned char)0xF3);
8976   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
8977   emit_int8((unsigned char)0x0F);
8978   emit_int8((unsigned char)0xB8);
8979   emit_int8((unsigned char)(0xC0 | encode));
8980 }
8981 
8982 void Assembler::popq(Address dst) {
8983   InstructionMark im(this);
8984   prefixq(dst);
8985   emit_int8((unsigned char)0x8F);
8986   emit_operand(rax, dst);
8987 }
8988 
8989 void Assembler::pusha() { // 64bit
8990   // we have to store original rsp.  ABI says that 128 bytes
8991   // below rsp are local scratch.
8992   movq(Address(rsp, -5 * wordSize), rsp);
8993 
8994   subq(rsp, 16 * wordSize);
8995 
8996   movq(Address(rsp, 15 * wordSize), rax);
8997   movq(Address(rsp, 14 * wordSize), rcx);
8998   movq(Address(rsp, 13 * wordSize), rdx);
8999   movq(Address(rsp, 12 * wordSize), rbx);
9000   // skip rsp
9001   movq(Address(rsp, 10 * wordSize), rbp);
9002   movq(Address(rsp, 9 * wordSize), rsi);
9003   movq(Address(rsp, 8 * wordSize), rdi);
9004   movq(Address(rsp, 7 * wordSize), r8);
9005   movq(Address(rsp, 6 * wordSize), r9);
9006   movq(Address(rsp, 5 * wordSize), r10);
9007   movq(Address(rsp, 4 * wordSize), r11);
9008   movq(Address(rsp, 3 * wordSize), r12);
9009   movq(Address(rsp, 2 * wordSize), r13);
9010   movq(Address(rsp, wordSize), r14);
9011   movq(Address(rsp, 0), r15);
9012 }
9013 
9014 void Assembler::pushq(Address src) {
9015   InstructionMark im(this);
9016   prefixq(src);
9017   emit_int8((unsigned char)0xFF);
9018   emit_operand(rsi, src);
9019 }
9020 
9021 void Assembler::rclq(Register dst, int imm8) {
9022   assert(isShiftCount(imm8 >> 1), "illegal shift count");
9023   int encode = prefixq_and_encode(dst->encoding());
9024   if (imm8 == 1) {
9025     emit_int8((unsigned char)0xD1);
9026     emit_int8((unsigned char)(0xD0 | encode));
9027   } else {
9028     emit_int8((unsigned char)0xC1);
9029     emit_int8((unsigned char)(0xD0 | encode));
9030     emit_int8(imm8);
9031   }
9032 }
9033 
9034 void Assembler::rcrq(Register dst, int imm8) {
9035   assert(isShiftCount(imm8 >> 1), "illegal shift count");
9036   int encode = prefixq_and_encode(dst->encoding());
9037   if (imm8 == 1) {
9038     emit_int8((unsigned char)0xD1);
9039     emit_int8((unsigned char)(0xD8 | encode));
9040   } else {
9041     emit_int8((unsigned char)0xC1);
9042     emit_int8((unsigned char)(0xD8 | encode));
9043     emit_int8(imm8);
9044   }
9045 }
9046 
9047 void Assembler::rorq(Register dst, int imm8) {
9048   assert(isShiftCount(imm8 >> 1), "illegal shift count");
9049   int encode = prefixq_and_encode(dst->encoding());
9050   if (imm8 == 1) {
9051     emit_int8((unsigned char)0xD1);
9052     emit_int8((unsigned char)(0xC8 | encode));
9053   } else {
9054     emit_int8((unsigned char)0xC1);
9055     emit_int8((unsigned char)(0xc8 | encode));
9056     emit_int8(imm8);
9057   }
9058 }
9059 
9060 void Assembler::rorxq(Register dst, Register src, int imm8) {
9061   assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
9062   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
9063   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_3A, &attributes);
9064   emit_int8((unsigned char)0xF0);
9065   emit_int8((unsigned char)(0xC0 | encode));
9066   emit_int8(imm8);
9067 }
9068 
9069 void Assembler::rorxd(Register dst, Register src, int imm8) {
9070   assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
9071   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
9072   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_3A, &attributes);
9073   emit_int8((unsigned char)0xF0);
9074   emit_int8((unsigned char)(0xC0 | encode));
9075   emit_int8(imm8);
9076 }
9077 
9078 void Assembler::sarq(Register dst, int imm8) {
9079   assert(isShiftCount(imm8 >> 1), "illegal shift count");
9080   int encode = prefixq_and_encode(dst->encoding());
9081   if (imm8 == 1) {
9082     emit_int8((unsigned char)0xD1);
9083     emit_int8((unsigned char)(0xF8 | encode));
9084   } else {
9085     emit_int8((unsigned char)0xC1);
9086     emit_int8((unsigned char)(0xF8 | encode));
9087     emit_int8(imm8);
9088   }
9089 }
9090 
9091 void Assembler::sarq(Register dst) {
9092   int encode = prefixq_and_encode(dst->encoding());
9093   emit_int8((unsigned char)0xD3);
9094   emit_int8((unsigned char)(0xF8 | encode));
9095 }
9096 
9097 void Assembler::sbbq(Address dst, int32_t imm32) {
9098   InstructionMark im(this);
9099   prefixq(dst);
9100   emit_arith_operand(0x81, rbx, dst, imm32);
9101 }
9102 
9103 void Assembler::sbbq(Register dst, int32_t imm32) {
9104   (void) prefixq_and_encode(dst->encoding());
9105   emit_arith(0x81, 0xD8, dst, imm32);
9106 }
9107 
9108 void Assembler::sbbq(Register dst, Address src) {
9109   InstructionMark im(this);
9110   prefixq(src, dst);
9111   emit_int8(0x1B);
9112   emit_operand(dst, src);
9113 }
9114 
9115 void Assembler::sbbq(Register dst, Register src) {
9116   (void) prefixq_and_encode(dst->encoding(), src->encoding());
9117   emit_arith(0x1B, 0xC0, dst, src);
9118 }
9119 
9120 void Assembler::shlq(Register dst, int imm8) {
9121   assert(isShiftCount(imm8 >> 1), "illegal shift count");
9122   int encode = prefixq_and_encode(dst->encoding());
9123   if (imm8 == 1) {
9124     emit_int8((unsigned char)0xD1);
9125     emit_int8((unsigned char)(0xE0 | encode));
9126   } else {
9127     emit_int8((unsigned char)0xC1);
9128     emit_int8((unsigned char)(0xE0 | encode));
9129     emit_int8(imm8);
9130   }
9131 }
9132 
9133 void Assembler::shlq(Register dst) {
9134   int encode = prefixq_and_encode(dst->encoding());
9135   emit_int8((unsigned char)0xD3);
9136   emit_int8((unsigned char)(0xE0 | encode));
9137 }
9138 
9139 void Assembler::shrq(Register dst, int imm8) {
9140   assert(isShiftCount(imm8 >> 1), "illegal shift count");
9141   int encode = prefixq_and_encode(dst->encoding());
9142   emit_int8((unsigned char)0xC1);
9143   emit_int8((unsigned char)(0xE8 | encode));
9144   emit_int8(imm8);
9145 }
9146 
9147 void Assembler::shrq(Register dst) {
9148   int encode = prefixq_and_encode(dst->encoding());
9149   emit_int8((unsigned char)0xD3);
9150   emit_int8(0xE8 | encode);
9151 }
9152 
9153 void Assembler::subq(Address dst, int32_t imm32) {
9154   InstructionMark im(this);
9155   prefixq(dst);
9156   emit_arith_operand(0x81, rbp, dst, imm32);
9157 }
9158 
9159 void Assembler::subq(Address dst, Register src) {
9160   InstructionMark im(this);
9161   prefixq(dst, src);
9162   emit_int8(0x29);
9163   emit_operand(src, dst);
9164 }
9165 
9166 void Assembler::subq(Register dst, int32_t imm32) {
9167   (void) prefixq_and_encode(dst->encoding());
9168   emit_arith(0x81, 0xE8, dst, imm32);
9169 }
9170 
9171 // Force generation of a 4 byte immediate value even if it fits into 8bit
9172 void Assembler::subq_imm32(Register dst, int32_t imm32) {
9173   (void) prefixq_and_encode(dst->encoding());
9174   emit_arith_imm32(0x81, 0xE8, dst, imm32);
9175 }
9176 
9177 void Assembler::subq(Register dst, Address src) {
9178   InstructionMark im(this);
9179   prefixq(src, dst);
9180   emit_int8(0x2B);
9181   emit_operand(dst, src);
9182 }
9183 
9184 void Assembler::subq(Register dst, Register src) {
9185   (void) prefixq_and_encode(dst->encoding(), src->encoding());
9186   emit_arith(0x2B, 0xC0, dst, src);
9187 }
9188 
9189 void Assembler::testq(Register dst, int32_t imm32) {
9190   // not using emit_arith because test
9191   // doesn't support sign-extension of
9192   // 8bit operands
9193   int encode = dst->encoding();
9194   if (encode == 0) {
9195     prefix(REX_W);
9196     emit_int8((unsigned char)0xA9);
9197   } else {
9198     encode = prefixq_and_encode(encode);
9199     emit_int8((unsigned char)0xF7);
9200     emit_int8((unsigned char)(0xC0 | encode));
9201   }
9202   emit_int32(imm32);
9203 }
9204 
9205 void Assembler::testq(Register dst, Register src) {
9206   (void) prefixq_and_encode(dst->encoding(), src->encoding());
9207   emit_arith(0x85, 0xC0, dst, src);
9208 }
9209 
9210 void Assembler::testq(Register dst, Address src) {
9211   InstructionMark im(this);
9212   prefixq(src, dst);
9213   emit_int8((unsigned char)0x85);
9214   emit_operand(dst, src);
9215 }
9216 
9217 void Assembler::xaddq(Address dst, Register src) {
9218   InstructionMark im(this);
9219   prefixq(dst, src);
9220   emit_int8(0x0F);
9221   emit_int8((unsigned char)0xC1);
9222   emit_operand(src, dst);
9223 }
9224 
9225 void Assembler::xchgq(Register dst, Address src) {
9226   InstructionMark im(this);
9227   prefixq(src, dst);
9228   emit_int8((unsigned char)0x87);
9229   emit_operand(dst, src);
9230 }
9231 
9232 void Assembler::xchgq(Register dst, Register src) {
9233   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
9234   emit_int8((unsigned char)0x87);
9235   emit_int8((unsigned char)(0xc0 | encode));
9236 }
9237 
9238 void Assembler::xorq(Register dst, Register src) {
9239   (void) prefixq_and_encode(dst->encoding(), src->encoding());
9240   emit_arith(0x33, 0xC0, dst, src);
9241 }
9242 
9243 void Assembler::xorq(Register dst, Address src) {
9244   InstructionMark im(this);
9245   prefixq(src, dst);
9246   emit_int8(0x33);
9247   emit_operand(dst, src);
9248 }
9249 
9250 #endif // !LP64