1 /*
   2  * Copyright (c) 1997, 2018, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #include "precompiled.hpp"
  26 #include "asm/assembler.hpp"
  27 #include "asm/assembler.inline.hpp"
  28 #include "gc/shared/cardTableBarrierSet.hpp"
  29 #include "gc/shared/collectedHeap.inline.hpp"
  30 #include "interpreter/interpreter.hpp"
  31 #include "memory/resourceArea.hpp"
  32 #include "prims/methodHandles.hpp"
  33 #include "runtime/biasedLocking.hpp"
  34 #include "runtime/objectMonitor.hpp"
  35 #include "runtime/os.hpp"
  36 #include "runtime/sharedRuntime.hpp"
  37 #include "runtime/stubRoutines.hpp"
  38 #include "utilities/macros.hpp"
  39 
  40 #ifdef PRODUCT
  41 #define BLOCK_COMMENT(str) /* nothing */
  42 #define STOP(error) stop(error)
  43 #else
  44 #define BLOCK_COMMENT(str) block_comment(str)
  45 #define STOP(error) block_comment(error); stop(error)
  46 #endif
  47 
  48 #define BIND(label) bind(label); BLOCK_COMMENT(#label ":")
  49 // Implementation of AddressLiteral
  50 
  51 // A 2-D table for managing compressed displacement(disp8) on EVEX enabled platforms.
  52 unsigned char tuple_table[Assembler::EVEX_ETUP + 1][Assembler::AVX_512bit + 1] = {
  53   // -----------------Table 4.5 -------------------- //
  54   16, 32, 64,  // EVEX_FV(0)
  55   4,  4,  4,   // EVEX_FV(1) - with Evex.b
  56   16, 32, 64,  // EVEX_FV(2) - with Evex.w
  57   8,  8,  8,   // EVEX_FV(3) - with Evex.w and Evex.b
  58   8,  16, 32,  // EVEX_HV(0)
  59   4,  4,  4,   // EVEX_HV(1) - with Evex.b
  60   // -----------------Table 4.6 -------------------- //
  61   16, 32, 64,  // EVEX_FVM(0)
  62   1,  1,  1,   // EVEX_T1S(0)
  63   2,  2,  2,   // EVEX_T1S(1)
  64   4,  4,  4,   // EVEX_T1S(2)
  65   8,  8,  8,   // EVEX_T1S(3)
  66   4,  4,  4,   // EVEX_T1F(0)
  67   8,  8,  8,   // EVEX_T1F(1)
  68   8,  8,  8,   // EVEX_T2(0)
  69   0,  16, 16,  // EVEX_T2(1)
  70   0,  16, 16,  // EVEX_T4(0)
  71   0,  0,  32,  // EVEX_T4(1)
  72   0,  0,  32,  // EVEX_T8(0)
  73   8,  16, 32,  // EVEX_HVM(0)
  74   4,  8,  16,  // EVEX_QVM(0)
  75   2,  4,  8,   // EVEX_OVM(0)
  76   16, 16, 16,  // EVEX_M128(0)
  77   8,  32, 64,  // EVEX_DUP(0)
  78   0,  0,  0    // EVEX_NTUP
  79 };
  80 
  81 AddressLiteral::AddressLiteral(address target, relocInfo::relocType rtype) {
  82   _is_lval = false;
  83   _target = target;
  84   switch (rtype) {
  85   case relocInfo::oop_type:
  86   case relocInfo::metadata_type:
  87     // Oops are a special case. Normally they would be their own section
  88     // but in cases like icBuffer they are literals in the code stream that
  89     // we don't have a section for. We use none so that we get a literal address
  90     // which is always patchable.
  91     break;
  92   case relocInfo::external_word_type:
  93     _rspec = external_word_Relocation::spec(target);
  94     break;
  95   case relocInfo::internal_word_type:
  96     _rspec = internal_word_Relocation::spec(target);
  97     break;
  98   case relocInfo::opt_virtual_call_type:
  99     _rspec = opt_virtual_call_Relocation::spec();
 100     break;
 101   case relocInfo::static_call_type:
 102     _rspec = static_call_Relocation::spec();
 103     break;
 104   case relocInfo::runtime_call_type:
 105     _rspec = runtime_call_Relocation::spec();
 106     break;
 107   case relocInfo::poll_type:
 108   case relocInfo::poll_return_type:
 109     _rspec = Relocation::spec_simple(rtype);
 110     break;
 111   case relocInfo::none:
 112     break;
 113   default:
 114     ShouldNotReachHere();
 115     break;
 116   }
 117 }
 118 
 119 // Implementation of Address
 120 
 121 #ifdef _LP64
 122 
 123 Address Address::make_array(ArrayAddress adr) {
 124   // Not implementable on 64bit machines
 125   // Should have been handled higher up the call chain.
 126   ShouldNotReachHere();
 127   return Address();
 128 }
 129 
 130 // exceedingly dangerous constructor
 131 Address::Address(int disp, address loc, relocInfo::relocType rtype) {
 132   _base  = noreg;
 133   _index = noreg;
 134   _scale = no_scale;
 135   _disp  = disp;
 136   _xmmindex = xnoreg;
 137   _isxmmindex = false;
 138   switch (rtype) {
 139     case relocInfo::external_word_type:
 140       _rspec = external_word_Relocation::spec(loc);
 141       break;
 142     case relocInfo::internal_word_type:
 143       _rspec = internal_word_Relocation::spec(loc);
 144       break;
 145     case relocInfo::runtime_call_type:
 146       // HMM
 147       _rspec = runtime_call_Relocation::spec();
 148       break;
 149     case relocInfo::poll_type:
 150     case relocInfo::poll_return_type:
 151       _rspec = Relocation::spec_simple(rtype);
 152       break;
 153     case relocInfo::none:
 154       break;
 155     default:
 156       ShouldNotReachHere();
 157   }
 158 }
 159 #else // LP64
 160 
 161 Address Address::make_array(ArrayAddress adr) {
 162   AddressLiteral base = adr.base();
 163   Address index = adr.index();
 164   assert(index._disp == 0, "must not have disp"); // maybe it can?
 165   Address array(index._base, index._index, index._scale, (intptr_t) base.target());
 166   array._rspec = base._rspec;
 167   return array;
 168 }
 169 
 170 // exceedingly dangerous constructor
 171 Address::Address(address loc, RelocationHolder spec) {
 172   _base  = noreg;
 173   _index = noreg;
 174   _scale = no_scale;
 175   _disp  = (intptr_t) loc;
 176   _rspec = spec;
 177   _xmmindex = xnoreg;
 178   _isxmmindex = false;
 179 }
 180 
 181 #endif // _LP64
 182 
 183 
 184 
 185 // Convert the raw encoding form into the form expected by the constructor for
 186 // Address.  An index of 4 (rsp) corresponds to having no index, so convert
 187 // that to noreg for the Address constructor.
 188 Address Address::make_raw(int base, int index, int scale, int disp, relocInfo::relocType disp_reloc) {
 189   RelocationHolder rspec;
 190   if (disp_reloc != relocInfo::none) {
 191     rspec = Relocation::spec_simple(disp_reloc);
 192   }
 193   bool valid_index = index != rsp->encoding();
 194   if (valid_index) {
 195     Address madr(as_Register(base), as_Register(index), (Address::ScaleFactor)scale, in_ByteSize(disp));
 196     madr._rspec = rspec;
 197     return madr;
 198   } else {
 199     Address madr(as_Register(base), noreg, Address::no_scale, in_ByteSize(disp));
 200     madr._rspec = rspec;
 201     return madr;
 202   }
 203 }
 204 
 205 // Implementation of Assembler
 206 
 207 int AbstractAssembler::code_fill_byte() {
 208   return (u_char)'\xF4'; // hlt
 209 }
 210 
 211 // make this go away someday
 212 void Assembler::emit_data(jint data, relocInfo::relocType rtype, int format) {
 213   if (rtype == relocInfo::none)
 214     emit_int32(data);
 215   else
 216     emit_data(data, Relocation::spec_simple(rtype), format);
 217 }
 218 
 219 void Assembler::emit_data(jint data, RelocationHolder const& rspec, int format) {
 220   assert(imm_operand == 0, "default format must be immediate in this file");
 221   assert(inst_mark() != NULL, "must be inside InstructionMark");
 222   if (rspec.type() !=  relocInfo::none) {
 223     #ifdef ASSERT
 224       check_relocation(rspec, format);
 225     #endif
 226     // Do not use AbstractAssembler::relocate, which is not intended for
 227     // embedded words.  Instead, relocate to the enclosing instruction.
 228 
 229     // hack. call32 is too wide for mask so use disp32
 230     if (format == call32_operand)
 231       code_section()->relocate(inst_mark(), rspec, disp32_operand);
 232     else
 233       code_section()->relocate(inst_mark(), rspec, format);
 234   }
 235   emit_int32(data);
 236 }
 237 
 238 static int encode(Register r) {
 239   int enc = r->encoding();
 240   if (enc >= 8) {
 241     enc -= 8;
 242   }
 243   return enc;
 244 }
 245 
 246 void Assembler::emit_arith_b(int op1, int op2, Register dst, int imm8) {
 247   assert(dst->has_byte_register(), "must have byte register");
 248   assert(isByte(op1) && isByte(op2), "wrong opcode");
 249   assert(isByte(imm8), "not a byte");
 250   assert((op1 & 0x01) == 0, "should be 8bit operation");
 251   emit_int8(op1);
 252   emit_int8(op2 | encode(dst));
 253   emit_int8(imm8);
 254 }
 255 
 256 
 257 void Assembler::emit_arith(int op1, int op2, Register dst, int32_t imm32) {
 258   assert(isByte(op1) && isByte(op2), "wrong opcode");
 259   assert((op1 & 0x01) == 1, "should be 32bit operation");
 260   assert((op1 & 0x02) == 0, "sign-extension bit should not be set");
 261   if (is8bit(imm32)) {
 262     emit_int8(op1 | 0x02); // set sign bit
 263     emit_int8(op2 | encode(dst));
 264     emit_int8(imm32 & 0xFF);
 265   } else {
 266     emit_int8(op1);
 267     emit_int8(op2 | encode(dst));
 268     emit_int32(imm32);
 269   }
 270 }
 271 
 272 // Force generation of a 4 byte immediate value even if it fits into 8bit
 273 void Assembler::emit_arith_imm32(int op1, int op2, Register dst, int32_t imm32) {
 274   assert(isByte(op1) && isByte(op2), "wrong opcode");
 275   assert((op1 & 0x01) == 1, "should be 32bit operation");
 276   assert((op1 & 0x02) == 0, "sign-extension bit should not be set");
 277   emit_int8(op1);
 278   emit_int8(op2 | encode(dst));
 279   emit_int32(imm32);
 280 }
 281 
 282 // immediate-to-memory forms
 283 void Assembler::emit_arith_operand(int op1, Register rm, Address adr, int32_t imm32) {
 284   assert((op1 & 0x01) == 1, "should be 32bit operation");
 285   assert((op1 & 0x02) == 0, "sign-extension bit should not be set");
 286   if (is8bit(imm32)) {
 287     emit_int8(op1 | 0x02); // set sign bit
 288     emit_operand(rm, adr, 1);
 289     emit_int8(imm32 & 0xFF);
 290   } else {
 291     emit_int8(op1);
 292     emit_operand(rm, adr, 4);
 293     emit_int32(imm32);
 294   }
 295 }
 296 
 297 
 298 void Assembler::emit_arith(int op1, int op2, Register dst, Register src) {
 299   assert(isByte(op1) && isByte(op2), "wrong opcode");
 300   emit_int8(op1);
 301   emit_int8(op2 | encode(dst) << 3 | encode(src));
 302 }
 303 
 304 
 305 bool Assembler::query_compressed_disp_byte(int disp, bool is_evex_inst, int vector_len,
 306                                            int cur_tuple_type, int in_size_in_bits, int cur_encoding) {
 307   int mod_idx = 0;
 308   // We will test if the displacement fits the compressed format and if so
 309   // apply the compression to the displacment iff the result is8bit.
 310   if (VM_Version::supports_evex() && is_evex_inst) {
 311     switch (cur_tuple_type) {
 312     case EVEX_FV:
 313       if ((cur_encoding & VEX_W) == VEX_W) {
 314         mod_idx = ((cur_encoding & EVEX_Rb) == EVEX_Rb) ? 3 : 2;
 315       } else {
 316         mod_idx = ((cur_encoding & EVEX_Rb) == EVEX_Rb) ? 1 : 0;
 317       }
 318       break;
 319 
 320     case EVEX_HV:
 321       mod_idx = ((cur_encoding & EVEX_Rb) == EVEX_Rb) ? 1 : 0;
 322       break;
 323 
 324     case EVEX_FVM:
 325       break;
 326 
 327     case EVEX_T1S:
 328       switch (in_size_in_bits) {
 329       case EVEX_8bit:
 330         break;
 331 
 332       case EVEX_16bit:
 333         mod_idx = 1;
 334         break;
 335 
 336       case EVEX_32bit:
 337         mod_idx = 2;
 338         break;
 339 
 340       case EVEX_64bit:
 341         mod_idx = 3;
 342         break;
 343       }
 344       break;
 345 
 346     case EVEX_T1F:
 347     case EVEX_T2:
 348     case EVEX_T4:
 349       mod_idx = (in_size_in_bits == EVEX_64bit) ? 1 : 0;
 350       break;
 351 
 352     case EVEX_T8:
 353       break;
 354 
 355     case EVEX_HVM:
 356       break;
 357 
 358     case EVEX_QVM:
 359       break;
 360 
 361     case EVEX_OVM:
 362       break;
 363 
 364     case EVEX_M128:
 365       break;
 366 
 367     case EVEX_DUP:
 368       break;
 369 
 370     default:
 371       assert(0, "no valid evex tuple_table entry");
 372       break;
 373     }
 374 
 375     if (vector_len >= AVX_128bit && vector_len <= AVX_512bit) {
 376       int disp_factor = tuple_table[cur_tuple_type + mod_idx][vector_len];
 377       if ((disp % disp_factor) == 0) {
 378         int new_disp = disp / disp_factor;
 379         if ((-0x80 <= new_disp && new_disp < 0x80)) {
 380           disp = new_disp;
 381         }
 382       } else {
 383         return false;
 384       }
 385     }
 386   }
 387   return (-0x80 <= disp && disp < 0x80);
 388 }
 389 
 390 
 391 bool Assembler::emit_compressed_disp_byte(int &disp) {
 392   int mod_idx = 0;
 393   // We will test if the displacement fits the compressed format and if so
 394   // apply the compression to the displacment iff the result is8bit.
 395   if (VM_Version::supports_evex() && _attributes && _attributes->is_evex_instruction()) {
 396     int evex_encoding = _attributes->get_evex_encoding();
 397     int tuple_type = _attributes->get_tuple_type();
 398     switch (tuple_type) {
 399     case EVEX_FV:
 400       if ((evex_encoding & VEX_W) == VEX_W) {
 401         mod_idx = ((evex_encoding & EVEX_Rb) == EVEX_Rb) ? 3 : 2;
 402       } else {
 403         mod_idx = ((evex_encoding & EVEX_Rb) == EVEX_Rb) ? 1 : 0;
 404       }
 405       break;
 406 
 407     case EVEX_HV:
 408       mod_idx = ((evex_encoding & EVEX_Rb) == EVEX_Rb) ? 1 : 0;
 409       break;
 410 
 411     case EVEX_FVM:
 412       break;
 413 
 414     case EVEX_T1S:
 415       switch (_attributes->get_input_size()) {
 416       case EVEX_8bit:
 417         break;
 418 
 419       case EVEX_16bit:
 420         mod_idx = 1;
 421         break;
 422 
 423       case EVEX_32bit:
 424         mod_idx = 2;
 425         break;
 426 
 427       case EVEX_64bit:
 428         mod_idx = 3;
 429         break;
 430       }
 431       break;
 432 
 433     case EVEX_T1F:
 434     case EVEX_T2:
 435     case EVEX_T4:
 436       mod_idx = (_attributes->get_input_size() == EVEX_64bit) ? 1 : 0;
 437       break;
 438 
 439     case EVEX_T8:
 440       break;
 441 
 442     case EVEX_HVM:
 443       break;
 444 
 445     case EVEX_QVM:
 446       break;
 447 
 448     case EVEX_OVM:
 449       break;
 450 
 451     case EVEX_M128:
 452       break;
 453 
 454     case EVEX_DUP:
 455       break;
 456 
 457     default:
 458       assert(0, "no valid evex tuple_table entry");
 459       break;
 460     }
 461 
 462     int vector_len = _attributes->get_vector_len();
 463     if (vector_len >= AVX_128bit && vector_len <= AVX_512bit) {
 464       int disp_factor = tuple_table[tuple_type + mod_idx][vector_len];
 465       if ((disp % disp_factor) == 0) {
 466         int new_disp = disp / disp_factor;
 467         if (is8bit(new_disp)) {
 468           disp = new_disp;
 469         }
 470       } else {
 471         return false;
 472       }
 473     }
 474   }
 475   return is8bit(disp);
 476 }
 477 
 478 
 479 void Assembler::emit_operand(Register reg, Register base, Register index,
 480                              Address::ScaleFactor scale, int disp,
 481                              RelocationHolder const& rspec,
 482                              int rip_relative_correction) {
 483   relocInfo::relocType rtype = (relocInfo::relocType) rspec.type();
 484 
 485   // Encode the registers as needed in the fields they are used in
 486 
 487   int regenc = encode(reg) << 3;
 488   int indexenc = index->is_valid() ? encode(index) << 3 : 0;
 489   int baseenc = base->is_valid() ? encode(base) : 0;
 490 
 491   if (base->is_valid()) {
 492     if (index->is_valid()) {
 493       assert(scale != Address::no_scale, "inconsistent address");
 494       // [base + index*scale + disp]
 495       if (disp == 0 && rtype == relocInfo::none  &&
 496           base != rbp LP64_ONLY(&& base != r13)) {
 497         // [base + index*scale]
 498         // [00 reg 100][ss index base]
 499         assert(index != rsp, "illegal addressing mode");
 500         emit_int8(0x04 | regenc);
 501         emit_int8(scale << 6 | indexenc | baseenc);
 502       } else if (emit_compressed_disp_byte(disp) && rtype == relocInfo::none) {
 503         // [base + index*scale + imm8]
 504         // [01 reg 100][ss index base] imm8
 505         assert(index != rsp, "illegal addressing mode");
 506         emit_int8(0x44 | regenc);
 507         emit_int8(scale << 6 | indexenc | baseenc);
 508         emit_int8(disp & 0xFF);
 509       } else {
 510         // [base + index*scale + disp32]
 511         // [10 reg 100][ss index base] disp32
 512         assert(index != rsp, "illegal addressing mode");
 513         emit_int8(0x84 | regenc);
 514         emit_int8(scale << 6 | indexenc | baseenc);
 515         emit_data(disp, rspec, disp32_operand);
 516       }
 517     } else if (base == rsp LP64_ONLY(|| base == r12)) {
 518       // [rsp + disp]
 519       if (disp == 0 && rtype == relocInfo::none) {
 520         // [rsp]
 521         // [00 reg 100][00 100 100]
 522         emit_int8(0x04 | regenc);
 523         emit_int8(0x24);
 524       } else if (emit_compressed_disp_byte(disp) && rtype == relocInfo::none) {
 525         // [rsp + imm8]
 526         // [01 reg 100][00 100 100] disp8
 527         emit_int8(0x44 | regenc);
 528         emit_int8(0x24);
 529         emit_int8(disp & 0xFF);
 530       } else {
 531         // [rsp + imm32]
 532         // [10 reg 100][00 100 100] disp32
 533         emit_int8(0x84 | regenc);
 534         emit_int8(0x24);
 535         emit_data(disp, rspec, disp32_operand);
 536       }
 537     } else {
 538       // [base + disp]
 539       assert(base != rsp LP64_ONLY(&& base != r12), "illegal addressing mode");
 540       if (disp == 0 && rtype == relocInfo::none &&
 541           base != rbp LP64_ONLY(&& base != r13)) {
 542         // [base]
 543         // [00 reg base]
 544         emit_int8(0x00 | regenc | baseenc);
 545       } else if (emit_compressed_disp_byte(disp) && rtype == relocInfo::none) {
 546         // [base + disp8]
 547         // [01 reg base] disp8
 548         emit_int8(0x40 | regenc | baseenc);
 549         emit_int8(disp & 0xFF);
 550       } else {
 551         // [base + disp32]
 552         // [10 reg base] disp32
 553         emit_int8(0x80 | regenc | baseenc);
 554         emit_data(disp, rspec, disp32_operand);
 555       }
 556     }
 557   } else {
 558     if (index->is_valid()) {
 559       assert(scale != Address::no_scale, "inconsistent address");
 560       // [index*scale + disp]
 561       // [00 reg 100][ss index 101] disp32
 562       assert(index != rsp, "illegal addressing mode");
 563       emit_int8(0x04 | regenc);
 564       emit_int8(scale << 6 | indexenc | 0x05);
 565       emit_data(disp, rspec, disp32_operand);
 566     } else if (rtype != relocInfo::none ) {
 567       // [disp] (64bit) RIP-RELATIVE (32bit) abs
 568       // [00 000 101] disp32
 569 
 570       emit_int8(0x05 | regenc);
 571       // Note that the RIP-rel. correction applies to the generated
 572       // disp field, but _not_ to the target address in the rspec.
 573 
 574       // disp was created by converting the target address minus the pc
 575       // at the start of the instruction. That needs more correction here.
 576       // intptr_t disp = target - next_ip;
 577       assert(inst_mark() != NULL, "must be inside InstructionMark");
 578       address next_ip = pc() + sizeof(int32_t) + rip_relative_correction;
 579       int64_t adjusted = disp;
 580       // Do rip-rel adjustment for 64bit
 581       LP64_ONLY(adjusted -=  (next_ip - inst_mark()));
 582       assert(is_simm32(adjusted),
 583              "must be 32bit offset (RIP relative address)");
 584       emit_data((int32_t) adjusted, rspec, disp32_operand);
 585 
 586     } else {
 587       // 32bit never did this, did everything as the rip-rel/disp code above
 588       // [disp] ABSOLUTE
 589       // [00 reg 100][00 100 101] disp32
 590       emit_int8(0x04 | regenc);
 591       emit_int8(0x25);
 592       emit_data(disp, rspec, disp32_operand);
 593     }
 594   }
 595 }
 596 
 597 void Assembler::emit_operand(XMMRegister reg, Register base, Register index,
 598                              Address::ScaleFactor scale, int disp,
 599                              RelocationHolder const& rspec) {
 600   if (UseAVX > 2) {
 601     int xreg_enc = reg->encoding();
 602     if (xreg_enc > 15) {
 603       XMMRegister new_reg = as_XMMRegister(xreg_enc & 0xf);
 604       emit_operand((Register)new_reg, base, index, scale, disp, rspec);
 605       return;
 606     }
 607   }
 608   emit_operand((Register)reg, base, index, scale, disp, rspec);
 609 }
 610 
 611 void Assembler::emit_operand(XMMRegister reg, Register base, XMMRegister index,
 612                              Address::ScaleFactor scale, int disp,
 613                              RelocationHolder const& rspec) {
 614   if (UseAVX > 2) {
 615     int xreg_enc = reg->encoding();
 616     int xmmindex_enc = index->encoding();
 617     XMMRegister new_reg = as_XMMRegister(xreg_enc & 0xf);
 618     XMMRegister new_index = as_XMMRegister(xmmindex_enc & 0xf);
 619     emit_operand((Register)new_reg, base, (Register)new_index, scale, disp, rspec);
 620   } else {
 621     emit_operand((Register)reg, base, (Register)index, scale, disp, rspec);
 622   }
 623 }
 624 
 625 
 626 // Secret local extension to Assembler::WhichOperand:
 627 #define end_pc_operand (_WhichOperand_limit)
 628 
 629 address Assembler::locate_operand(address inst, WhichOperand which) {
 630   // Decode the given instruction, and return the address of
 631   // an embedded 32-bit operand word.
 632 
 633   // If "which" is disp32_operand, selects the displacement portion
 634   // of an effective address specifier.
 635   // If "which" is imm64_operand, selects the trailing immediate constant.
 636   // If "which" is call32_operand, selects the displacement of a call or jump.
 637   // Caller is responsible for ensuring that there is such an operand,
 638   // and that it is 32/64 bits wide.
 639 
 640   // If "which" is end_pc_operand, find the end of the instruction.
 641 
 642   address ip = inst;
 643   bool is_64bit = false;
 644 
 645   debug_only(bool has_disp32 = false);
 646   int tail_size = 0; // other random bytes (#32, #16, etc.) at end of insn
 647 
 648   again_after_prefix:
 649   switch (0xFF & *ip++) {
 650 
 651   // These convenience macros generate groups of "case" labels for the switch.
 652 #define REP4(x) (x)+0: case (x)+1: case (x)+2: case (x)+3
 653 #define REP8(x) (x)+0: case (x)+1: case (x)+2: case (x)+3: \
 654              case (x)+4: case (x)+5: case (x)+6: case (x)+7
 655 #define REP16(x) REP8((x)+0): \
 656               case REP8((x)+8)
 657 
 658   case CS_segment:
 659   case SS_segment:
 660   case DS_segment:
 661   case ES_segment:
 662   case FS_segment:
 663   case GS_segment:
 664     // Seems dubious
 665     LP64_ONLY(assert(false, "shouldn't have that prefix"));
 666     assert(ip == inst+1, "only one prefix allowed");
 667     goto again_after_prefix;
 668 
 669   case 0x67:
 670   case REX:
 671   case REX_B:
 672   case REX_X:
 673   case REX_XB:
 674   case REX_R:
 675   case REX_RB:
 676   case REX_RX:
 677   case REX_RXB:
 678     NOT_LP64(assert(false, "64bit prefixes"));
 679     goto again_after_prefix;
 680 
 681   case REX_W:
 682   case REX_WB:
 683   case REX_WX:
 684   case REX_WXB:
 685   case REX_WR:
 686   case REX_WRB:
 687   case REX_WRX:
 688   case REX_WRXB:
 689     NOT_LP64(assert(false, "64bit prefixes"));
 690     is_64bit = true;
 691     goto again_after_prefix;
 692 
 693   case 0xFF: // pushq a; decl a; incl a; call a; jmp a
 694   case 0x88: // movb a, r
 695   case 0x89: // movl a, r
 696   case 0x8A: // movb r, a
 697   case 0x8B: // movl r, a
 698   case 0x8F: // popl a
 699     debug_only(has_disp32 = true);
 700     break;
 701 
 702   case 0x68: // pushq #32
 703     if (which == end_pc_operand) {
 704       return ip + 4;
 705     }
 706     assert(which == imm_operand && !is_64bit, "pushl has no disp32 or 64bit immediate");
 707     return ip;                  // not produced by emit_operand
 708 
 709   case 0x66: // movw ... (size prefix)
 710     again_after_size_prefix2:
 711     switch (0xFF & *ip++) {
 712     case REX:
 713     case REX_B:
 714     case REX_X:
 715     case REX_XB:
 716     case REX_R:
 717     case REX_RB:
 718     case REX_RX:
 719     case REX_RXB:
 720     case REX_W:
 721     case REX_WB:
 722     case REX_WX:
 723     case REX_WXB:
 724     case REX_WR:
 725     case REX_WRB:
 726     case REX_WRX:
 727     case REX_WRXB:
 728       NOT_LP64(assert(false, "64bit prefix found"));
 729       goto again_after_size_prefix2;
 730     case 0x8B: // movw r, a
 731     case 0x89: // movw a, r
 732       debug_only(has_disp32 = true);
 733       break;
 734     case 0xC7: // movw a, #16
 735       debug_only(has_disp32 = true);
 736       tail_size = 2;  // the imm16
 737       break;
 738     case 0x0F: // several SSE/SSE2 variants
 739       ip--;    // reparse the 0x0F
 740       goto again_after_prefix;
 741     default:
 742       ShouldNotReachHere();
 743     }
 744     break;
 745 
 746   case REP8(0xB8): // movl/q r, #32/#64(oop?)
 747     if (which == end_pc_operand)  return ip + (is_64bit ? 8 : 4);
 748     // these asserts are somewhat nonsensical
 749 #ifndef _LP64
 750     assert(which == imm_operand || which == disp32_operand,
 751            "which %d is_64_bit %d ip " INTPTR_FORMAT, which, is_64bit, p2i(ip));
 752 #else
 753     assert((which == call32_operand || which == imm_operand) && is_64bit ||
 754            which == narrow_oop_operand && !is_64bit,
 755            "which %d is_64_bit %d ip " INTPTR_FORMAT, which, is_64bit, p2i(ip));
 756 #endif // _LP64
 757     return ip;
 758 
 759   case 0x69: // imul r, a, #32
 760   case 0xC7: // movl a, #32(oop?)
 761     tail_size = 4;
 762     debug_only(has_disp32 = true); // has both kinds of operands!
 763     break;
 764 
 765   case 0x0F: // movx..., etc.
 766     switch (0xFF & *ip++) {
 767     case 0x3A: // pcmpestri
 768       tail_size = 1;
 769     case 0x38: // ptest, pmovzxbw
 770       ip++; // skip opcode
 771       debug_only(has_disp32 = true); // has both kinds of operands!
 772       break;
 773 
 774     case 0x70: // pshufd r, r/a, #8
 775       debug_only(has_disp32 = true); // has both kinds of operands!
 776     case 0x73: // psrldq r, #8
 777       tail_size = 1;
 778       break;
 779 
 780     case 0x12: // movlps
 781     case 0x28: // movaps
 782     case 0x2E: // ucomiss
 783     case 0x2F: // comiss
 784     case 0x54: // andps
 785     case 0x55: // andnps
 786     case 0x56: // orps
 787     case 0x57: // xorps
 788     case 0x58: // addpd
 789     case 0x59: // mulpd
 790     case 0x6E: // movd
 791     case 0x7E: // movd
 792     case 0xAE: // ldmxcsr, stmxcsr, fxrstor, fxsave, clflush
 793     case 0xFE: // paddd
 794       debug_only(has_disp32 = true);
 795       break;
 796 
 797     case 0xAD: // shrd r, a, %cl
 798     case 0xAF: // imul r, a
 799     case 0xBE: // movsbl r, a (movsxb)
 800     case 0xBF: // movswl r, a (movsxw)
 801     case 0xB6: // movzbl r, a (movzxb)
 802     case 0xB7: // movzwl r, a (movzxw)
 803     case REP16(0x40): // cmovl cc, r, a
 804     case 0xB0: // cmpxchgb
 805     case 0xB1: // cmpxchg
 806     case 0xC1: // xaddl
 807     case 0xC7: // cmpxchg8
 808     case REP16(0x90): // setcc a
 809       debug_only(has_disp32 = true);
 810       // fall out of the switch to decode the address
 811       break;
 812 
 813     case 0xC4: // pinsrw r, a, #8
 814       debug_only(has_disp32 = true);
 815     case 0xC5: // pextrw r, r, #8
 816       tail_size = 1;  // the imm8
 817       break;
 818 
 819     case 0xAC: // shrd r, a, #8
 820       debug_only(has_disp32 = true);
 821       tail_size = 1;  // the imm8
 822       break;
 823 
 824     case REP16(0x80): // jcc rdisp32
 825       if (which == end_pc_operand)  return ip + 4;
 826       assert(which == call32_operand, "jcc has no disp32 or imm");
 827       return ip;
 828     default:
 829       ShouldNotReachHere();
 830     }
 831     break;
 832 
 833   case 0x81: // addl a, #32; addl r, #32
 834     // also: orl, adcl, sbbl, andl, subl, xorl, cmpl
 835     // on 32bit in the case of cmpl, the imm might be an oop
 836     tail_size = 4;
 837     debug_only(has_disp32 = true); // has both kinds of operands!
 838     break;
 839 
 840   case 0x83: // addl a, #8; addl r, #8
 841     // also: orl, adcl, sbbl, andl, subl, xorl, cmpl
 842     debug_only(has_disp32 = true); // has both kinds of operands!
 843     tail_size = 1;
 844     break;
 845 
 846   case 0x9B:
 847     switch (0xFF & *ip++) {
 848     case 0xD9: // fnstcw a
 849       debug_only(has_disp32 = true);
 850       break;
 851     default:
 852       ShouldNotReachHere();
 853     }
 854     break;
 855 
 856   case REP4(0x00): // addb a, r; addl a, r; addb r, a; addl r, a
 857   case REP4(0x10): // adc...
 858   case REP4(0x20): // and...
 859   case REP4(0x30): // xor...
 860   case REP4(0x08): // or...
 861   case REP4(0x18): // sbb...
 862   case REP4(0x28): // sub...
 863   case 0xF7: // mull a
 864   case 0x8D: // lea r, a
 865   case 0x87: // xchg r, a
 866   case REP4(0x38): // cmp...
 867   case 0x85: // test r, a
 868     debug_only(has_disp32 = true); // has both kinds of operands!
 869     break;
 870 
 871   case 0xC1: // sal a, #8; sar a, #8; shl a, #8; shr a, #8
 872   case 0xC6: // movb a, #8
 873   case 0x80: // cmpb a, #8
 874   case 0x6B: // imul r, a, #8
 875     debug_only(has_disp32 = true); // has both kinds of operands!
 876     tail_size = 1; // the imm8
 877     break;
 878 
 879   case 0xC4: // VEX_3bytes
 880   case 0xC5: // VEX_2bytes
 881     assert((UseAVX > 0), "shouldn't have VEX prefix");
 882     assert(ip == inst+1, "no prefixes allowed");
 883     // C4 and C5 are also used as opcodes for PINSRW and PEXTRW instructions
 884     // but they have prefix 0x0F and processed when 0x0F processed above.
 885     //
 886     // In 32-bit mode the VEX first byte C4 and C5 alias onto LDS and LES
 887     // instructions (these instructions are not supported in 64-bit mode).
 888     // To distinguish them bits [7:6] are set in the VEX second byte since
 889     // ModRM byte can not be of the form 11xxxxxx in 32-bit mode. To set
 890     // those VEX bits REX and vvvv bits are inverted.
 891     //
 892     // Fortunately C2 doesn't generate these instructions so we don't need
 893     // to check for them in product version.
 894 
 895     // Check second byte
 896     NOT_LP64(assert((0xC0 & *ip) == 0xC0, "shouldn't have LDS and LES instructions"));
 897 
 898     int vex_opcode;
 899     // First byte
 900     if ((0xFF & *inst) == VEX_3bytes) {
 901       vex_opcode = VEX_OPCODE_MASK & *ip;
 902       ip++; // third byte
 903       is_64bit = ((VEX_W & *ip) == VEX_W);
 904     } else {
 905       vex_opcode = VEX_OPCODE_0F;
 906     }
 907     ip++; // opcode
 908     // To find the end of instruction (which == end_pc_operand).
 909     switch (vex_opcode) {
 910       case VEX_OPCODE_0F:
 911         switch (0xFF & *ip) {
 912         case 0x70: // pshufd r, r/a, #8
 913         case 0x71: // ps[rl|ra|ll]w r, #8
 914         case 0x72: // ps[rl|ra|ll]d r, #8
 915         case 0x73: // ps[rl|ra|ll]q r, #8
 916         case 0xC2: // cmp[ps|pd|ss|sd] r, r, r/a, #8
 917         case 0xC4: // pinsrw r, r, r/a, #8
 918         case 0xC5: // pextrw r/a, r, #8
 919         case 0xC6: // shufp[s|d] r, r, r/a, #8
 920           tail_size = 1;  // the imm8
 921           break;
 922         }
 923         break;
 924       case VEX_OPCODE_0F_3A:
 925         tail_size = 1;
 926         break;
 927     }
 928     ip++; // skip opcode
 929     debug_only(has_disp32 = true); // has both kinds of operands!
 930     break;
 931 
 932   case 0x62: // EVEX_4bytes
 933     assert(VM_Version::supports_evex(), "shouldn't have EVEX prefix");
 934     assert(ip == inst+1, "no prefixes allowed");
 935     // no EVEX collisions, all instructions that have 0x62 opcodes
 936     // have EVEX versions and are subopcodes of 0x66
 937     ip++; // skip P0 and exmaine W in P1
 938     is_64bit = ((VEX_W & *ip) == VEX_W);
 939     ip++; // move to P2
 940     ip++; // skip P2, move to opcode
 941     // To find the end of instruction (which == end_pc_operand).
 942     switch (0xFF & *ip) {
 943     case 0x22: // pinsrd r, r/a, #8
 944     case 0x61: // pcmpestri r, r/a, #8
 945     case 0x70: // pshufd r, r/a, #8
 946     case 0x73: // psrldq r, #8
 947       tail_size = 1;  // the imm8
 948       break;
 949     default:
 950       break;
 951     }
 952     ip++; // skip opcode
 953     debug_only(has_disp32 = true); // has both kinds of operands!
 954     break;
 955 
 956   case 0xD1: // sal a, 1; sar a, 1; shl a, 1; shr a, 1
 957   case 0xD3: // sal a, %cl; sar a, %cl; shl a, %cl; shr a, %cl
 958   case 0xD9: // fld_s a; fst_s a; fstp_s a; fldcw a
 959   case 0xDD: // fld_d a; fst_d a; fstp_d a
 960   case 0xDB: // fild_s a; fistp_s a; fld_x a; fstp_x a
 961   case 0xDF: // fild_d a; fistp_d a
 962   case 0xD8: // fadd_s a; fsubr_s a; fmul_s a; fdivr_s a; fcomp_s a
 963   case 0xDC: // fadd_d a; fsubr_d a; fmul_d a; fdivr_d a; fcomp_d a
 964   case 0xDE: // faddp_d a; fsubrp_d a; fmulp_d a; fdivrp_d a; fcompp_d a
 965     debug_only(has_disp32 = true);
 966     break;
 967 
 968   case 0xE8: // call rdisp32
 969   case 0xE9: // jmp  rdisp32
 970     if (which == end_pc_operand)  return ip + 4;
 971     assert(which == call32_operand, "call has no disp32 or imm");
 972     return ip;
 973 
 974   case 0xF0:                    // Lock
 975     goto again_after_prefix;
 976 
 977   case 0xF3:                    // For SSE
 978   case 0xF2:                    // For SSE2
 979     switch (0xFF & *ip++) {
 980     case REX:
 981     case REX_B:
 982     case REX_X:
 983     case REX_XB:
 984     case REX_R:
 985     case REX_RB:
 986     case REX_RX:
 987     case REX_RXB:
 988     case REX_W:
 989     case REX_WB:
 990     case REX_WX:
 991     case REX_WXB:
 992     case REX_WR:
 993     case REX_WRB:
 994     case REX_WRX:
 995     case REX_WRXB:
 996       NOT_LP64(assert(false, "found 64bit prefix"));
 997       ip++;
 998     default:
 999       ip++;
1000     }
1001     debug_only(has_disp32 = true); // has both kinds of operands!
1002     break;
1003 
1004   default:
1005     ShouldNotReachHere();
1006 
1007 #undef REP8
1008 #undef REP16
1009   }
1010 
1011   assert(which != call32_operand, "instruction is not a call, jmp, or jcc");
1012 #ifdef _LP64
1013   assert(which != imm_operand, "instruction is not a movq reg, imm64");
1014 #else
1015   // assert(which != imm_operand || has_imm32, "instruction has no imm32 field");
1016   assert(which != imm_operand || has_disp32, "instruction has no imm32 field");
1017 #endif // LP64
1018   assert(which != disp32_operand || has_disp32, "instruction has no disp32 field");
1019 
1020   // parse the output of emit_operand
1021   int op2 = 0xFF & *ip++;
1022   int base = op2 & 0x07;
1023   int op3 = -1;
1024   const int b100 = 4;
1025   const int b101 = 5;
1026   if (base == b100 && (op2 >> 6) != 3) {
1027     op3 = 0xFF & *ip++;
1028     base = op3 & 0x07;   // refetch the base
1029   }
1030   // now ip points at the disp (if any)
1031 
1032   switch (op2 >> 6) {
1033   case 0:
1034     // [00 reg  100][ss index base]
1035     // [00 reg  100][00   100  esp]
1036     // [00 reg base]
1037     // [00 reg  100][ss index  101][disp32]
1038     // [00 reg  101]               [disp32]
1039 
1040     if (base == b101) {
1041       if (which == disp32_operand)
1042         return ip;              // caller wants the disp32
1043       ip += 4;                  // skip the disp32
1044     }
1045     break;
1046 
1047   case 1:
1048     // [01 reg  100][ss index base][disp8]
1049     // [01 reg  100][00   100  esp][disp8]
1050     // [01 reg base]               [disp8]
1051     ip += 1;                    // skip the disp8
1052     break;
1053 
1054   case 2:
1055     // [10 reg  100][ss index base][disp32]
1056     // [10 reg  100][00   100  esp][disp32]
1057     // [10 reg base]               [disp32]
1058     if (which == disp32_operand)
1059       return ip;                // caller wants the disp32
1060     ip += 4;                    // skip the disp32
1061     break;
1062 
1063   case 3:
1064     // [11 reg base]  (not a memory addressing mode)
1065     break;
1066   }
1067 
1068   if (which == end_pc_operand) {
1069     return ip + tail_size;
1070   }
1071 
1072 #ifdef _LP64
1073   assert(which == narrow_oop_operand && !is_64bit, "instruction is not a movl adr, imm32");
1074 #else
1075   assert(which == imm_operand, "instruction has only an imm field");
1076 #endif // LP64
1077   return ip;
1078 }
1079 
1080 address Assembler::locate_next_instruction(address inst) {
1081   // Secretly share code with locate_operand:
1082   return locate_operand(inst, end_pc_operand);
1083 }
1084 
1085 
1086 #ifdef ASSERT
1087 void Assembler::check_relocation(RelocationHolder const& rspec, int format) {
1088   address inst = inst_mark();
1089   assert(inst != NULL && inst < pc(), "must point to beginning of instruction");
1090   address opnd;
1091 
1092   Relocation* r = rspec.reloc();
1093   if (r->type() == relocInfo::none) {
1094     return;
1095   } else if (r->is_call() || format == call32_operand) {
1096     // assert(format == imm32_operand, "cannot specify a nonzero format");
1097     opnd = locate_operand(inst, call32_operand);
1098   } else if (r->is_data()) {
1099     assert(format == imm_operand || format == disp32_operand
1100            LP64_ONLY(|| format == narrow_oop_operand), "format ok");
1101     opnd = locate_operand(inst, (WhichOperand)format);
1102   } else {
1103     assert(format == imm_operand, "cannot specify a format");
1104     return;
1105   }
1106   assert(opnd == pc(), "must put operand where relocs can find it");
1107 }
1108 #endif // ASSERT
1109 
1110 void Assembler::emit_operand32(Register reg, Address adr) {
1111   assert(reg->encoding() < 8, "no extended registers");
1112   assert(!adr.base_needs_rex() && !adr.index_needs_rex(), "no extended registers");
1113   emit_operand(reg, adr._base, adr._index, adr._scale, adr._disp,
1114                adr._rspec);
1115 }
1116 
1117 void Assembler::emit_operand(Register reg, Address adr,
1118                              int rip_relative_correction) {
1119   emit_operand(reg, adr._base, adr._index, adr._scale, adr._disp,
1120                adr._rspec,
1121                rip_relative_correction);
1122 }
1123 
1124 void Assembler::emit_operand(XMMRegister reg, Address adr) {
1125     if (adr.isxmmindex()) {
1126        emit_operand(reg, adr._base, adr._xmmindex, adr._scale, adr._disp, adr._rspec);
1127     } else {
1128        emit_operand(reg, adr._base, adr._index, adr._scale, adr._disp,
1129        adr._rspec);
1130     }
1131 }
1132 
1133 // MMX operations
1134 void Assembler::emit_operand(MMXRegister reg, Address adr) {
1135   assert(!adr.base_needs_rex() && !adr.index_needs_rex(), "no extended registers");
1136   emit_operand((Register)reg, adr._base, adr._index, adr._scale, adr._disp, adr._rspec);
1137 }
1138 
1139 // work around gcc (3.2.1-7a) bug
1140 void Assembler::emit_operand(Address adr, MMXRegister reg) {
1141   assert(!adr.base_needs_rex() && !adr.index_needs_rex(), "no extended registers");
1142   emit_operand((Register)reg, adr._base, adr._index, adr._scale, adr._disp, adr._rspec);
1143 }
1144 
1145 
1146 void Assembler::emit_farith(int b1, int b2, int i) {
1147   assert(isByte(b1) && isByte(b2), "wrong opcode");
1148   assert(0 <= i &&  i < 8, "illegal stack offset");
1149   emit_int8(b1);
1150   emit_int8(b2 + i);
1151 }
1152 
1153 
1154 // Now the Assembler instructions (identical for 32/64 bits)
1155 
1156 void Assembler::adcl(Address dst, int32_t imm32) {
1157   InstructionMark im(this);
1158   prefix(dst);
1159   emit_arith_operand(0x81, rdx, dst, imm32);
1160 }
1161 
1162 void Assembler::adcl(Address dst, Register src) {
1163   InstructionMark im(this);
1164   prefix(dst, src);
1165   emit_int8(0x11);
1166   emit_operand(src, dst);
1167 }
1168 
1169 void Assembler::adcl(Register dst, int32_t imm32) {
1170   prefix(dst);
1171   emit_arith(0x81, 0xD0, dst, imm32);
1172 }
1173 
1174 void Assembler::adcl(Register dst, Address src) {
1175   InstructionMark im(this);
1176   prefix(src, dst);
1177   emit_int8(0x13);
1178   emit_operand(dst, src);
1179 }
1180 
1181 void Assembler::adcl(Register dst, Register src) {
1182   (void) prefix_and_encode(dst->encoding(), src->encoding());
1183   emit_arith(0x13, 0xC0, dst, src);
1184 }
1185 
1186 void Assembler::addl(Address dst, int32_t imm32) {
1187   InstructionMark im(this);
1188   prefix(dst);
1189   emit_arith_operand(0x81, rax, dst, imm32);
1190 }
1191 
1192 void Assembler::addb(Address dst, int imm8) {
1193   InstructionMark im(this);
1194   prefix(dst);
1195   emit_int8((unsigned char)0x80);
1196   emit_operand(rax, dst, 1);
1197   emit_int8(imm8);
1198 }
1199 
1200 void Assembler::addw(Address dst, int imm16) {
1201   InstructionMark im(this);
1202   emit_int8(0x66);
1203   prefix(dst);
1204   emit_int8((unsigned char)0x81);
1205   emit_operand(rax, dst, 2);
1206   emit_int16(imm16);
1207 }
1208 
1209 void Assembler::addl(Address dst, Register src) {
1210   InstructionMark im(this);
1211   prefix(dst, src);
1212   emit_int8(0x01);
1213   emit_operand(src, dst);
1214 }
1215 
1216 void Assembler::addl(Register dst, int32_t imm32) {
1217   prefix(dst);
1218   emit_arith(0x81, 0xC0, dst, imm32);
1219 }
1220 
1221 void Assembler::addl(Register dst, Address src) {
1222   InstructionMark im(this);
1223   prefix(src, dst);
1224   emit_int8(0x03);
1225   emit_operand(dst, src);
1226 }
1227 
1228 void Assembler::addl(Register dst, Register src) {
1229   (void) prefix_and_encode(dst->encoding(), src->encoding());
1230   emit_arith(0x03, 0xC0, dst, src);
1231 }
1232 
1233 void Assembler::addr_nop_4() {
1234   assert(UseAddressNop, "no CPU support");
1235   // 4 bytes: NOP DWORD PTR [EAX+0]
1236   emit_int8(0x0F);
1237   emit_int8(0x1F);
1238   emit_int8(0x40); // emit_rm(cbuf, 0x1, EAX_enc, EAX_enc);
1239   emit_int8(0);    // 8-bits offset (1 byte)
1240 }
1241 
1242 void Assembler::addr_nop_5() {
1243   assert(UseAddressNop, "no CPU support");
1244   // 5 bytes: NOP DWORD PTR [EAX+EAX*0+0] 8-bits offset
1245   emit_int8(0x0F);
1246   emit_int8(0x1F);
1247   emit_int8(0x44); // emit_rm(cbuf, 0x1, EAX_enc, 0x4);
1248   emit_int8(0x00); // emit_rm(cbuf, 0x0, EAX_enc, EAX_enc);
1249   emit_int8(0);    // 8-bits offset (1 byte)
1250 }
1251 
1252 void Assembler::addr_nop_7() {
1253   assert(UseAddressNop, "no CPU support");
1254   // 7 bytes: NOP DWORD PTR [EAX+0] 32-bits offset
1255   emit_int8(0x0F);
1256   emit_int8(0x1F);
1257   emit_int8((unsigned char)0x80);
1258                    // emit_rm(cbuf, 0x2, EAX_enc, EAX_enc);
1259   emit_int32(0);   // 32-bits offset (4 bytes)
1260 }
1261 
1262 void Assembler::addr_nop_8() {
1263   assert(UseAddressNop, "no CPU support");
1264   // 8 bytes: NOP DWORD PTR [EAX+EAX*0+0] 32-bits offset
1265   emit_int8(0x0F);
1266   emit_int8(0x1F);
1267   emit_int8((unsigned char)0x84);
1268                    // emit_rm(cbuf, 0x2, EAX_enc, 0x4);
1269   emit_int8(0x00); // emit_rm(cbuf, 0x0, EAX_enc, EAX_enc);
1270   emit_int32(0);   // 32-bits offset (4 bytes)
1271 }
1272 
1273 void Assembler::addsd(XMMRegister dst, XMMRegister src) {
1274   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1275   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1276   attributes.set_rex_vex_w_reverted();
1277   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
1278   emit_int8(0x58);
1279   emit_int8((unsigned char)(0xC0 | encode));
1280 }
1281 
1282 void Assembler::addsd(XMMRegister dst, Address src) {
1283   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1284   InstructionMark im(this);
1285   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1286   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
1287   attributes.set_rex_vex_w_reverted();
1288   simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
1289   emit_int8(0x58);
1290   emit_operand(dst, src);
1291 }
1292 
1293 void Assembler::addss(XMMRegister dst, XMMRegister src) {
1294   NOT_LP64(assert(VM_Version::supports_sse(), ""));
1295   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1296   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
1297   emit_int8(0x58);
1298   emit_int8((unsigned char)(0xC0 | encode));
1299 }
1300 
1301 void Assembler::addss(XMMRegister dst, Address src) {
1302   NOT_LP64(assert(VM_Version::supports_sse(), ""));
1303   InstructionMark im(this);
1304   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1305   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
1306   simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
1307   emit_int8(0x58);
1308   emit_operand(dst, src);
1309 }
1310 
1311 void Assembler::aesdec(XMMRegister dst, Address src) {
1312   assert(VM_Version::supports_aes(), "");
1313   InstructionMark im(this);
1314   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
1315   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
1316   emit_int8((unsigned char)0xDE);
1317   emit_operand(dst, src);
1318 }
1319 
1320 void Assembler::aesdec(XMMRegister dst, XMMRegister src) {
1321   assert(VM_Version::supports_aes(), "");
1322   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
1323   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
1324   emit_int8((unsigned char)0xDE);
1325   emit_int8(0xC0 | encode);
1326 }
1327 
1328 void Assembler::vaesdec(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
1329   assert(VM_Version::supports_vaes(), "");
1330   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
1331   attributes.set_is_evex_instruction();
1332   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
1333   emit_int8((unsigned char)0xDE);
1334   emit_int8((unsigned char)(0xC0 | encode));
1335 }
1336 
1337 
1338 void Assembler::aesdeclast(XMMRegister dst, Address src) {
1339   assert(VM_Version::supports_aes(), "");
1340   InstructionMark im(this);
1341   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
1342   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
1343   emit_int8((unsigned char)0xDF);
1344   emit_operand(dst, src);
1345 }
1346 
1347 void Assembler::aesdeclast(XMMRegister dst, XMMRegister src) {
1348   assert(VM_Version::supports_aes(), "");
1349   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
1350   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
1351   emit_int8((unsigned char)0xDF);
1352   emit_int8((unsigned char)(0xC0 | encode));
1353 }
1354 
1355 void Assembler::vaesdeclast(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
1356   assert(VM_Version::supports_vaes(), "");
1357   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
1358   attributes.set_is_evex_instruction();
1359   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
1360   emit_int8((unsigned char)0xDF);
1361   emit_int8((unsigned char)(0xC0 | encode));
1362 }
1363 
1364 void Assembler::aesenc(XMMRegister dst, Address src) {
1365   assert(VM_Version::supports_aes(), "");
1366   InstructionMark im(this);
1367   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
1368   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
1369   emit_int8((unsigned char)0xDC);
1370   emit_operand(dst, src);
1371 }
1372 
1373 void Assembler::aesenc(XMMRegister dst, XMMRegister src) {
1374   assert(VM_Version::supports_aes(), "");
1375   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
1376   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
1377   emit_int8((unsigned char)0xDC);
1378   emit_int8(0xC0 | encode);
1379 }
1380 
1381 void Assembler::aesenclast(XMMRegister dst, Address src) {
1382   assert(VM_Version::supports_aes(), "");
1383   InstructionMark im(this);
1384   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
1385   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
1386   emit_int8((unsigned char)0xDD);
1387   emit_operand(dst, src);
1388 }
1389 
1390 void Assembler::aesenclast(XMMRegister dst, XMMRegister src) {
1391   assert(VM_Version::supports_aes(), "");
1392   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
1393   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
1394   emit_int8((unsigned char)0xDD);
1395   emit_int8((unsigned char)(0xC0 | encode));
1396 }
1397 
1398 void Assembler::andl(Address dst, int32_t imm32) {
1399   InstructionMark im(this);
1400   prefix(dst);
1401   emit_int8((unsigned char)0x81);
1402   emit_operand(rsp, dst, 4);
1403   emit_int32(imm32);
1404 }
1405 
1406 void Assembler::andl(Register dst, int32_t imm32) {
1407   prefix(dst);
1408   emit_arith(0x81, 0xE0, dst, imm32);
1409 }
1410 
1411 void Assembler::andl(Register dst, Address src) {
1412   InstructionMark im(this);
1413   prefix(src, dst);
1414   emit_int8(0x23);
1415   emit_operand(dst, src);
1416 }
1417 
1418 void Assembler::andl(Register dst, Register src) {
1419   (void) prefix_and_encode(dst->encoding(), src->encoding());
1420   emit_arith(0x23, 0xC0, dst, src);
1421 }
1422 
1423 void Assembler::andnl(Register dst, Register src1, Register src2) {
1424   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1425   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
1426   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
1427   emit_int8((unsigned char)0xF2);
1428   emit_int8((unsigned char)(0xC0 | encode));
1429 }
1430 
1431 void Assembler::andnl(Register dst, Register src1, Address src2) {
1432   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1433   InstructionMark im(this);
1434   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
1435   vex_prefix(src2, src1->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
1436   emit_int8((unsigned char)0xF2);
1437   emit_operand(dst, src2);
1438 }
1439 
1440 void Assembler::bsfl(Register dst, Register src) {
1441   int encode = prefix_and_encode(dst->encoding(), src->encoding());
1442   emit_int8(0x0F);
1443   emit_int8((unsigned char)0xBC);
1444   emit_int8((unsigned char)(0xC0 | encode));
1445 }
1446 
1447 void Assembler::bsrl(Register dst, Register src) {
1448   int encode = prefix_and_encode(dst->encoding(), src->encoding());
1449   emit_int8(0x0F);
1450   emit_int8((unsigned char)0xBD);
1451   emit_int8((unsigned char)(0xC0 | encode));
1452 }
1453 
1454 void Assembler::bswapl(Register reg) { // bswap
1455   int encode = prefix_and_encode(reg->encoding());
1456   emit_int8(0x0F);
1457   emit_int8((unsigned char)(0xC8 | encode));
1458 }
1459 
1460 void Assembler::blsil(Register dst, Register src) {
1461   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1462   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
1463   int encode = vex_prefix_and_encode(rbx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
1464   emit_int8((unsigned char)0xF3);
1465   emit_int8((unsigned char)(0xC0 | encode));
1466 }
1467 
1468 void Assembler::blsil(Register dst, Address src) {
1469   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1470   InstructionMark im(this);
1471   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
1472   vex_prefix(src, dst->encoding(), rbx->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
1473   emit_int8((unsigned char)0xF3);
1474   emit_operand(rbx, src);
1475 }
1476 
1477 void Assembler::blsmskl(Register dst, Register src) {
1478   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1479   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
1480   int encode = vex_prefix_and_encode(rdx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
1481   emit_int8((unsigned char)0xF3);
1482   emit_int8((unsigned char)(0xC0 | encode));
1483 }
1484 
1485 void Assembler::blsmskl(Register dst, Address src) {
1486   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1487   InstructionMark im(this);
1488   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
1489   vex_prefix(src, dst->encoding(), rdx->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
1490   emit_int8((unsigned char)0xF3);
1491   emit_operand(rdx, src);
1492 }
1493 
1494 void Assembler::blsrl(Register dst, Register src) {
1495   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1496   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
1497   int encode = vex_prefix_and_encode(rcx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
1498   emit_int8((unsigned char)0xF3);
1499   emit_int8((unsigned char)(0xC0 | encode));
1500 }
1501 
1502 void Assembler::blsrl(Register dst, Address src) {
1503   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1504   InstructionMark im(this);
1505   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
1506   vex_prefix(src, dst->encoding(), rcx->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
1507   emit_int8((unsigned char)0xF3);
1508   emit_operand(rcx, src);
1509 }
1510 
1511 void Assembler::call(Label& L, relocInfo::relocType rtype) {
1512   // suspect disp32 is always good
1513   int operand = LP64_ONLY(disp32_operand) NOT_LP64(imm_operand);
1514 
1515   if (L.is_bound()) {
1516     const int long_size = 5;
1517     int offs = (int)( target(L) - pc() );
1518     assert(offs <= 0, "assembler error");
1519     InstructionMark im(this);
1520     // 1110 1000 #32-bit disp
1521     emit_int8((unsigned char)0xE8);
1522     emit_data(offs - long_size, rtype, operand);
1523   } else {
1524     InstructionMark im(this);
1525     // 1110 1000 #32-bit disp
1526     L.add_patch_at(code(), locator());
1527 
1528     emit_int8((unsigned char)0xE8);
1529     emit_data(int(0), rtype, operand);
1530   }
1531 }
1532 
1533 void Assembler::call(Register dst) {
1534   int encode = prefix_and_encode(dst->encoding());
1535   emit_int8((unsigned char)0xFF);
1536   emit_int8((unsigned char)(0xD0 | encode));
1537 }
1538 
1539 
1540 void Assembler::call(Address adr) {
1541   InstructionMark im(this);
1542   prefix(adr);
1543   emit_int8((unsigned char)0xFF);
1544   emit_operand(rdx, adr);
1545 }
1546 
1547 void Assembler::call_literal(address entry, RelocationHolder const& rspec) {
1548   InstructionMark im(this);
1549   emit_int8((unsigned char)0xE8);
1550   intptr_t disp = entry - (pc() + sizeof(int32_t));
1551   // Entry is NULL in case of a scratch emit.
1552   assert(entry == NULL || is_simm32(disp), "disp=" INTPTR_FORMAT " must be 32bit offset (call2)", disp);
1553   // Technically, should use call32_operand, but this format is
1554   // implied by the fact that we're emitting a call instruction.
1555 
1556   int operand = LP64_ONLY(disp32_operand) NOT_LP64(call32_operand);
1557   emit_data((int) disp, rspec, operand);
1558 }
1559 
1560 void Assembler::cdql() {
1561   emit_int8((unsigned char)0x99);
1562 }
1563 
1564 void Assembler::cld() {
1565   emit_int8((unsigned char)0xFC);
1566 }
1567 
1568 void Assembler::cmovl(Condition cc, Register dst, Register src) {
1569   NOT_LP64(guarantee(VM_Version::supports_cmov(), "illegal instruction"));
1570   int encode = prefix_and_encode(dst->encoding(), src->encoding());
1571   emit_int8(0x0F);
1572   emit_int8(0x40 | cc);
1573   emit_int8((unsigned char)(0xC0 | encode));
1574 }
1575 
1576 
1577 void Assembler::cmovl(Condition cc, Register dst, Address src) {
1578   NOT_LP64(guarantee(VM_Version::supports_cmov(), "illegal instruction"));
1579   prefix(src, dst);
1580   emit_int8(0x0F);
1581   emit_int8(0x40 | cc);
1582   emit_operand(dst, src);
1583 }
1584 
1585 void Assembler::cmpb(Address dst, int imm8) {
1586   InstructionMark im(this);
1587   prefix(dst);
1588   emit_int8((unsigned char)0x80);
1589   emit_operand(rdi, dst, 1);
1590   emit_int8(imm8);
1591 }
1592 
1593 void Assembler::cmpl(Address dst, int32_t imm32) {
1594   InstructionMark im(this);
1595   prefix(dst);
1596   emit_int8((unsigned char)0x81);
1597   emit_operand(rdi, dst, 4);
1598   emit_int32(imm32);
1599 }
1600 
1601 void Assembler::cmpl(Register dst, int32_t imm32) {
1602   prefix(dst);
1603   emit_arith(0x81, 0xF8, dst, imm32);
1604 }
1605 
1606 void Assembler::cmpl(Register dst, Register src) {
1607   (void) prefix_and_encode(dst->encoding(), src->encoding());
1608   emit_arith(0x3B, 0xC0, dst, src);
1609 }
1610 
1611 void Assembler::cmpl(Register dst, Address  src) {
1612   InstructionMark im(this);
1613   prefix(src, dst);
1614   emit_int8((unsigned char)0x3B);
1615   emit_operand(dst, src);
1616 }
1617 
1618 void Assembler::cmpw(Address dst, int imm16) {
1619   InstructionMark im(this);
1620   assert(!dst.base_needs_rex() && !dst.index_needs_rex(), "no extended registers");
1621   emit_int8(0x66);
1622   emit_int8((unsigned char)0x81);
1623   emit_operand(rdi, dst, 2);
1624   emit_int16(imm16);
1625 }
1626 
1627 // The 32-bit cmpxchg compares the value at adr with the contents of rax,
1628 // and stores reg into adr if so; otherwise, the value at adr is loaded into rax,.
1629 // The ZF is set if the compared values were equal, and cleared otherwise.
1630 void Assembler::cmpxchgl(Register reg, Address adr) { // cmpxchg
1631   InstructionMark im(this);
1632   prefix(adr, reg);
1633   emit_int8(0x0F);
1634   emit_int8((unsigned char)0xB1);
1635   emit_operand(reg, adr);
1636 }
1637 
1638 // The 8-bit cmpxchg compares the value at adr with the contents of rax,
1639 // and stores reg into adr if so; otherwise, the value at adr is loaded into rax,.
1640 // The ZF is set if the compared values were equal, and cleared otherwise.
1641 void Assembler::cmpxchgb(Register reg, Address adr) { // cmpxchg
1642   InstructionMark im(this);
1643   prefix(adr, reg, true);
1644   emit_int8(0x0F);
1645   emit_int8((unsigned char)0xB0);
1646   emit_operand(reg, adr);
1647 }
1648 
1649 void Assembler::comisd(XMMRegister dst, Address src) {
1650   // NOTE: dbx seems to decode this as comiss even though the
1651   // 0x66 is there. Strangly ucomisd comes out correct
1652   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1653   InstructionMark im(this);
1654   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);;
1655   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
1656   attributes.set_rex_vex_w_reverted();
1657   simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
1658   emit_int8(0x2F);
1659   emit_operand(dst, src);
1660 }
1661 
1662 void Assembler::comisd(XMMRegister dst, XMMRegister src) {
1663   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1664   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1665   attributes.set_rex_vex_w_reverted();
1666   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
1667   emit_int8(0x2F);
1668   emit_int8((unsigned char)(0xC0 | encode));
1669 }
1670 
1671 void Assembler::comiss(XMMRegister dst, Address src) {
1672   NOT_LP64(assert(VM_Version::supports_sse(), ""));
1673   InstructionMark im(this);
1674   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1675   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
1676   simd_prefix(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
1677   emit_int8(0x2F);
1678   emit_operand(dst, src);
1679 }
1680 
1681 void Assembler::comiss(XMMRegister dst, XMMRegister src) {
1682   NOT_LP64(assert(VM_Version::supports_sse(), ""));
1683   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1684   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
1685   emit_int8(0x2F);
1686   emit_int8((unsigned char)(0xC0 | encode));
1687 }
1688 
1689 void Assembler::cpuid() {
1690   emit_int8(0x0F);
1691   emit_int8((unsigned char)0xA2);
1692 }
1693 
1694 // Opcode / Instruction                      Op /  En  64 - Bit Mode     Compat / Leg Mode Description                  Implemented
1695 // F2 0F 38 F0 / r       CRC32 r32, r / m8   RM        Valid             Valid             Accumulate CRC32 on r / m8.  v
1696 // F2 REX 0F 38 F0 / r   CRC32 r32, r / m8*  RM        Valid             N.E.              Accumulate CRC32 on r / m8.  -
1697 // F2 REX.W 0F 38 F0 / r CRC32 r64, r / m8   RM        Valid             N.E.              Accumulate CRC32 on r / m8.  -
1698 //
1699 // F2 0F 38 F1 / r       CRC32 r32, r / m16  RM        Valid             Valid             Accumulate CRC32 on r / m16. v
1700 //
1701 // F2 0F 38 F1 / r       CRC32 r32, r / m32  RM        Valid             Valid             Accumulate CRC32 on r / m32. v
1702 //
1703 // F2 REX.W 0F 38 F1 / r CRC32 r64, r / m64  RM        Valid             N.E.              Accumulate CRC32 on r / m64. v
1704 void Assembler::crc32(Register crc, Register v, int8_t sizeInBytes) {
1705   assert(VM_Version::supports_sse4_2(), "");
1706   int8_t w = 0x01;
1707   Prefix p = Prefix_EMPTY;
1708 
1709   emit_int8((int8_t)0xF2);
1710   switch (sizeInBytes) {
1711   case 1:
1712     w = 0;
1713     break;
1714   case 2:
1715   case 4:
1716     break;
1717   LP64_ONLY(case 8:)
1718     // This instruction is not valid in 32 bits
1719     // Note:
1720     // http://www.intel.com/content/dam/www/public/us/en/documents/manuals/64-ia-32-architectures-software-developer-instruction-set-reference-manual-325383.pdf
1721     //
1722     // Page B - 72   Vol. 2C says
1723     // qwreg2 to qwreg            1111 0010 : 0100 1R0B : 0000 1111 : 0011 1000 : 1111 0000 : 11 qwreg1 qwreg2
1724     // mem64 to qwreg             1111 0010 : 0100 1R0B : 0000 1111 : 0011 1000 : 1111 0000 : mod qwreg r / m
1725     //                                                                            F0!!!
1726     // while 3 - 208 Vol. 2A
1727     // F2 REX.W 0F 38 F1 / r       CRC32 r64, r / m64             RM         Valid      N.E.Accumulate CRC32 on r / m64.
1728     //
1729     // the 0 on a last bit is reserved for a different flavor of this instruction :
1730     // F2 REX.W 0F 38 F0 / r       CRC32 r64, r / m8              RM         Valid      N.E.Accumulate CRC32 on r / m8.
1731     p = REX_W;
1732     break;
1733   default:
1734     assert(0, "Unsupported value for a sizeInBytes argument");
1735     break;
1736   }
1737   LP64_ONLY(prefix(crc, v, p);)
1738   emit_int8((int8_t)0x0F);
1739   emit_int8(0x38);
1740   emit_int8((int8_t)(0xF0 | w));
1741   emit_int8(0xC0 | ((crc->encoding() & 0x7) << 3) | (v->encoding() & 7));
1742 }
1743 
1744 void Assembler::crc32(Register crc, Address adr, int8_t sizeInBytes) {
1745   assert(VM_Version::supports_sse4_2(), "");
1746   InstructionMark im(this);
1747   int8_t w = 0x01;
1748   Prefix p = Prefix_EMPTY;
1749 
1750   emit_int8((int8_t)0xF2);
1751   switch (sizeInBytes) {
1752   case 1:
1753     w = 0;
1754     break;
1755   case 2:
1756   case 4:
1757     break;
1758   LP64_ONLY(case 8:)
1759     // This instruction is not valid in 32 bits
1760     p = REX_W;
1761     break;
1762   default:
1763     assert(0, "Unsupported value for a sizeInBytes argument");
1764     break;
1765   }
1766   LP64_ONLY(prefix(crc, adr, p);)
1767   emit_int8((int8_t)0x0F);
1768   emit_int8(0x38);
1769   emit_int8((int8_t)(0xF0 | w));
1770   emit_operand(crc, adr);
1771 }
1772 
1773 void Assembler::cvtdq2pd(XMMRegister dst, XMMRegister src) {
1774   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1775   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
1776   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
1777   emit_int8((unsigned char)0xE6);
1778   emit_int8((unsigned char)(0xC0 | encode));
1779 }
1780 
1781 void Assembler::cvtdq2ps(XMMRegister dst, XMMRegister src) {
1782   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1783   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
1784   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
1785   emit_int8(0x5B);
1786   emit_int8((unsigned char)(0xC0 | encode));
1787 }
1788 
1789 void Assembler::cvtsd2ss(XMMRegister dst, XMMRegister src) {
1790   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1791   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1792   attributes.set_rex_vex_w_reverted();
1793   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
1794   emit_int8(0x5A);
1795   emit_int8((unsigned char)(0xC0 | encode));
1796 }
1797 
1798 void Assembler::cvtsd2ss(XMMRegister dst, Address src) {
1799   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1800   InstructionMark im(this);
1801   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1802   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
1803   attributes.set_rex_vex_w_reverted();
1804   simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
1805   emit_int8(0x5A);
1806   emit_operand(dst, src);
1807 }
1808 
1809 void Assembler::cvtsi2sdl(XMMRegister dst, Register src) {
1810   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1811   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1812   int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
1813   emit_int8(0x2A);
1814   emit_int8((unsigned char)(0xC0 | encode));
1815 }
1816 
1817 void Assembler::cvtsi2sdl(XMMRegister dst, Address src) {
1818   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1819   InstructionMark im(this);
1820   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1821   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
1822   simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
1823   emit_int8(0x2A);
1824   emit_operand(dst, src);
1825 }
1826 
1827 void Assembler::cvtsi2ssl(XMMRegister dst, Register src) {
1828   NOT_LP64(assert(VM_Version::supports_sse(), ""));
1829   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1830   int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
1831   emit_int8(0x2A);
1832   emit_int8((unsigned char)(0xC0 | encode));
1833 }
1834 
1835 void Assembler::cvtsi2ssl(XMMRegister dst, Address src) {
1836   NOT_LP64(assert(VM_Version::supports_sse(), ""));
1837   InstructionMark im(this);
1838   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1839   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
1840   simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
1841   emit_int8(0x2A);
1842   emit_operand(dst, src);
1843 }
1844 
1845 void Assembler::cvtsi2ssq(XMMRegister dst, Register src) {
1846   NOT_LP64(assert(VM_Version::supports_sse(), ""));
1847   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1848   int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
1849   emit_int8(0x2A);
1850   emit_int8((unsigned char)(0xC0 | encode));
1851 }
1852 
1853 void Assembler::cvtss2sd(XMMRegister dst, XMMRegister src) {
1854   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1855   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1856   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
1857   emit_int8(0x5A);
1858   emit_int8((unsigned char)(0xC0 | encode));
1859 }
1860 
1861 void Assembler::cvtss2sd(XMMRegister dst, Address src) {
1862   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1863   InstructionMark im(this);
1864   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1865   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
1866   simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
1867   emit_int8(0x5A);
1868   emit_operand(dst, src);
1869 }
1870 
1871 
1872 void Assembler::cvttsd2sil(Register dst, XMMRegister src) {
1873   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1874   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1875   int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
1876   emit_int8(0x2C);
1877   emit_int8((unsigned char)(0xC0 | encode));
1878 }
1879 
1880 void Assembler::cvttss2sil(Register dst, XMMRegister src) {
1881   NOT_LP64(assert(VM_Version::supports_sse(), ""));
1882   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1883   int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
1884   emit_int8(0x2C);
1885   emit_int8((unsigned char)(0xC0 | encode));
1886 }
1887 
1888 void Assembler::cvttpd2dq(XMMRegister dst, XMMRegister src) {
1889   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1890   int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_128bit;
1891   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
1892   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
1893   emit_int8((unsigned char)0xE6);
1894   emit_int8((unsigned char)(0xC0 | encode));
1895 }
1896 
1897 void Assembler::decl(Address dst) {
1898   // Don't use it directly. Use MacroAssembler::decrement() instead.
1899   InstructionMark im(this);
1900   prefix(dst);
1901   emit_int8((unsigned char)0xFF);
1902   emit_operand(rcx, dst);
1903 }
1904 
1905 void Assembler::divsd(XMMRegister dst, Address src) {
1906   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1907   InstructionMark im(this);
1908   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1909   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
1910   attributes.set_rex_vex_w_reverted();
1911   simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
1912   emit_int8(0x5E);
1913   emit_operand(dst, src);
1914 }
1915 
1916 void Assembler::divsd(XMMRegister dst, XMMRegister src) {
1917   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1918   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1919   attributes.set_rex_vex_w_reverted();
1920   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
1921   emit_int8(0x5E);
1922   emit_int8((unsigned char)(0xC0 | encode));
1923 }
1924 
1925 void Assembler::divss(XMMRegister dst, Address src) {
1926   NOT_LP64(assert(VM_Version::supports_sse(), ""));
1927   InstructionMark im(this);
1928   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1929   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
1930   simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
1931   emit_int8(0x5E);
1932   emit_operand(dst, src);
1933 }
1934 
1935 void Assembler::divss(XMMRegister dst, XMMRegister src) {
1936   NOT_LP64(assert(VM_Version::supports_sse(), ""));
1937   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1938   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
1939   emit_int8(0x5E);
1940   emit_int8((unsigned char)(0xC0 | encode));
1941 }
1942 
1943 void Assembler::emms() {
1944   NOT_LP64(assert(VM_Version::supports_mmx(), ""));
1945   emit_int8(0x0F);
1946   emit_int8(0x77);
1947 }
1948 
1949 void Assembler::hlt() {
1950   emit_int8((unsigned char)0xF4);
1951 }
1952 
1953 void Assembler::idivl(Register src) {
1954   int encode = prefix_and_encode(src->encoding());
1955   emit_int8((unsigned char)0xF7);
1956   emit_int8((unsigned char)(0xF8 | encode));
1957 }
1958 
1959 void Assembler::divl(Register src) { // Unsigned
1960   int encode = prefix_and_encode(src->encoding());
1961   emit_int8((unsigned char)0xF7);
1962   emit_int8((unsigned char)(0xF0 | encode));
1963 }
1964 
1965 void Assembler::imull(Register src) {
1966   int encode = prefix_and_encode(src->encoding());
1967   emit_int8((unsigned char)0xF7);
1968   emit_int8((unsigned char)(0xE8 | encode));
1969 }
1970 
1971 void Assembler::imull(Register dst, Register src) {
1972   int encode = prefix_and_encode(dst->encoding(), src->encoding());
1973   emit_int8(0x0F);
1974   emit_int8((unsigned char)0xAF);
1975   emit_int8((unsigned char)(0xC0 | encode));
1976 }
1977 
1978 
1979 void Assembler::imull(Register dst, Register src, int value) {
1980   int encode = prefix_and_encode(dst->encoding(), src->encoding());
1981   if (is8bit(value)) {
1982     emit_int8(0x6B);
1983     emit_int8((unsigned char)(0xC0 | encode));
1984     emit_int8(value & 0xFF);
1985   } else {
1986     emit_int8(0x69);
1987     emit_int8((unsigned char)(0xC0 | encode));
1988     emit_int32(value);
1989   }
1990 }
1991 
1992 void Assembler::imull(Register dst, Address src) {
1993   InstructionMark im(this);
1994   prefix(src, dst);
1995   emit_int8(0x0F);
1996   emit_int8((unsigned char) 0xAF);
1997   emit_operand(dst, src);
1998 }
1999 
2000 
2001 void Assembler::incl(Address dst) {
2002   // Don't use it directly. Use MacroAssembler::increment() instead.
2003   InstructionMark im(this);
2004   prefix(dst);
2005   emit_int8((unsigned char)0xFF);
2006   emit_operand(rax, dst);
2007 }
2008 
2009 void Assembler::jcc(Condition cc, Label& L, bool maybe_short) {
2010   InstructionMark im(this);
2011   assert((0 <= cc) && (cc < 16), "illegal cc");
2012   if (L.is_bound()) {
2013     address dst = target(L);
2014     assert(dst != NULL, "jcc most probably wrong");
2015 
2016     const int short_size = 2;
2017     const int long_size = 6;
2018     intptr_t offs = (intptr_t)dst - (intptr_t)pc();
2019     if (maybe_short && is8bit(offs - short_size)) {
2020       // 0111 tttn #8-bit disp
2021       emit_int8(0x70 | cc);
2022       emit_int8((offs - short_size) & 0xFF);
2023     } else {
2024       // 0000 1111 1000 tttn #32-bit disp
2025       assert(is_simm32(offs - long_size),
2026              "must be 32bit offset (call4)");
2027       emit_int8(0x0F);
2028       emit_int8((unsigned char)(0x80 | cc));
2029       emit_int32(offs - long_size);
2030     }
2031   } else {
2032     // Note: could eliminate cond. jumps to this jump if condition
2033     //       is the same however, seems to be rather unlikely case.
2034     // Note: use jccb() if label to be bound is very close to get
2035     //       an 8-bit displacement
2036     L.add_patch_at(code(), locator());
2037     emit_int8(0x0F);
2038     emit_int8((unsigned char)(0x80 | cc));
2039     emit_int32(0);
2040   }
2041 }
2042 
2043 void Assembler::jccb_0(Condition cc, Label& L, const char* file, int line) {
2044   if (L.is_bound()) {
2045     const int short_size = 2;
2046     address entry = target(L);
2047 #ifdef ASSERT
2048     intptr_t dist = (intptr_t)entry - ((intptr_t)pc() + short_size);
2049     intptr_t delta = short_branch_delta();
2050     if (delta != 0) {
2051       dist += (dist < 0 ? (-delta) :delta);
2052     }
2053     assert(is8bit(dist), "Dispacement too large for a short jmp at %s:%d", file, line);
2054 #endif
2055     intptr_t offs = (intptr_t)entry - (intptr_t)pc();
2056     // 0111 tttn #8-bit disp
2057     emit_int8(0x70 | cc);
2058     emit_int8((offs - short_size) & 0xFF);
2059   } else {
2060     InstructionMark im(this);
2061     L.add_patch_at(code(), locator(), file, line);
2062     emit_int8(0x70 | cc);
2063     emit_int8(0);
2064   }
2065 }
2066 
2067 void Assembler::jmp(Address adr) {
2068   InstructionMark im(this);
2069   prefix(adr);
2070   emit_int8((unsigned char)0xFF);
2071   emit_operand(rsp, adr);
2072 }
2073 
2074 void Assembler::jmp(Label& L, bool maybe_short) {
2075   if (L.is_bound()) {
2076     address entry = target(L);
2077     assert(entry != NULL, "jmp most probably wrong");
2078     InstructionMark im(this);
2079     const int short_size = 2;
2080     const int long_size = 5;
2081     intptr_t offs = entry - pc();
2082     if (maybe_short && is8bit(offs - short_size)) {
2083       emit_int8((unsigned char)0xEB);
2084       emit_int8((offs - short_size) & 0xFF);
2085     } else {
2086       emit_int8((unsigned char)0xE9);
2087       emit_int32(offs - long_size);
2088     }
2089   } else {
2090     // By default, forward jumps are always 32-bit displacements, since
2091     // we can't yet know where the label will be bound.  If you're sure that
2092     // the forward jump will not run beyond 256 bytes, use jmpb to
2093     // force an 8-bit displacement.
2094     InstructionMark im(this);
2095     L.add_patch_at(code(), locator());
2096     emit_int8((unsigned char)0xE9);
2097     emit_int32(0);
2098   }
2099 }
2100 
2101 void Assembler::jmp(Register entry) {
2102   int encode = prefix_and_encode(entry->encoding());
2103   emit_int8((unsigned char)0xFF);
2104   emit_int8((unsigned char)(0xE0 | encode));
2105 }
2106 
2107 void Assembler::jmp_literal(address dest, RelocationHolder const& rspec) {
2108   InstructionMark im(this);
2109   emit_int8((unsigned char)0xE9);
2110   assert(dest != NULL, "must have a target");
2111   intptr_t disp = dest - (pc() + sizeof(int32_t));
2112   assert(is_simm32(disp), "must be 32bit offset (jmp)");
2113   emit_data(disp, rspec.reloc(), call32_operand);
2114 }
2115 
2116 void Assembler::jmpb_0(Label& L, const char* file, int line) {
2117   if (L.is_bound()) {
2118     const int short_size = 2;
2119     address entry = target(L);
2120     assert(entry != NULL, "jmp most probably wrong");
2121 #ifdef ASSERT
2122     intptr_t dist = (intptr_t)entry - ((intptr_t)pc() + short_size);
2123     intptr_t delta = short_branch_delta();
2124     if (delta != 0) {
2125       dist += (dist < 0 ? (-delta) :delta);
2126     }
2127     assert(is8bit(dist), "Dispacement too large for a short jmp at %s:%d", file, line);
2128 #endif
2129     intptr_t offs = entry - pc();
2130     emit_int8((unsigned char)0xEB);
2131     emit_int8((offs - short_size) & 0xFF);
2132   } else {
2133     InstructionMark im(this);
2134     L.add_patch_at(code(), locator(), file, line);
2135     emit_int8((unsigned char)0xEB);
2136     emit_int8(0);
2137   }
2138 }
2139 
2140 void Assembler::ldmxcsr( Address src) {
2141   if (UseAVX > 0 ) {
2142     InstructionMark im(this);
2143     InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2144     vex_prefix(src, 0, 0, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2145     emit_int8((unsigned char)0xAE);
2146     emit_operand(as_Register(2), src);
2147   } else {
2148     NOT_LP64(assert(VM_Version::supports_sse(), ""));
2149     InstructionMark im(this);
2150     prefix(src);
2151     emit_int8(0x0F);
2152     emit_int8((unsigned char)0xAE);
2153     emit_operand(as_Register(2), src);
2154   }
2155 }
2156 
2157 void Assembler::leal(Register dst, Address src) {
2158   InstructionMark im(this);
2159 #ifdef _LP64
2160   emit_int8(0x67); // addr32
2161   prefix(src, dst);
2162 #endif // LP64
2163   emit_int8((unsigned char)0x8D);
2164   emit_operand(dst, src);
2165 }
2166 
2167 void Assembler::lfence() {
2168   emit_int8(0x0F);
2169   emit_int8((unsigned char)0xAE);
2170   emit_int8((unsigned char)0xE8);
2171 }
2172 
2173 void Assembler::lock() {
2174   emit_int8((unsigned char)0xF0);
2175 }
2176 
2177 void Assembler::lzcntl(Register dst, Register src) {
2178   assert(VM_Version::supports_lzcnt(), "encoding is treated as BSR");
2179   emit_int8((unsigned char)0xF3);
2180   int encode = prefix_and_encode(dst->encoding(), src->encoding());
2181   emit_int8(0x0F);
2182   emit_int8((unsigned char)0xBD);
2183   emit_int8((unsigned char)(0xC0 | encode));
2184 }
2185 
2186 // Emit mfence instruction
2187 void Assembler::mfence() {
2188   NOT_LP64(assert(VM_Version::supports_sse2(), "unsupported");)
2189   emit_int8(0x0F);
2190   emit_int8((unsigned char)0xAE);
2191   emit_int8((unsigned char)0xF0);
2192 }
2193 
2194 void Assembler::mov(Register dst, Register src) {
2195   LP64_ONLY(movq(dst, src)) NOT_LP64(movl(dst, src));
2196 }
2197 
2198 void Assembler::movapd(XMMRegister dst, XMMRegister src) {
2199   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2200   int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_128bit;
2201   InstructionAttr attributes(vector_len, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
2202   attributes.set_rex_vex_w_reverted();
2203   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2204   emit_int8(0x28);
2205   emit_int8((unsigned char)(0xC0 | encode));
2206 }
2207 
2208 void Assembler::movaps(XMMRegister dst, XMMRegister src) {
2209   NOT_LP64(assert(VM_Version::supports_sse(), ""));
2210   int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_128bit;
2211   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
2212   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2213   emit_int8(0x28);
2214   emit_int8((unsigned char)(0xC0 | encode));
2215 }
2216 
2217 void Assembler::movlhps(XMMRegister dst, XMMRegister src) {
2218   NOT_LP64(assert(VM_Version::supports_sse(), ""));
2219   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2220   int encode = simd_prefix_and_encode(dst, src, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2221   emit_int8(0x16);
2222   emit_int8((unsigned char)(0xC0 | encode));
2223 }
2224 
2225 void Assembler::movb(Register dst, Address src) {
2226   NOT_LP64(assert(dst->has_byte_register(), "must have byte register"));
2227   InstructionMark im(this);
2228   prefix(src, dst, true);
2229   emit_int8((unsigned char)0x8A);
2230   emit_operand(dst, src);
2231 }
2232 
2233 void Assembler::movddup(XMMRegister dst, XMMRegister src) {
2234   NOT_LP64(assert(VM_Version::supports_sse3(), ""));
2235   int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_128bit;
2236   InstructionAttr attributes(vector_len, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2237   attributes.set_rex_vex_w_reverted();
2238   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2239   emit_int8(0x12);
2240   emit_int8(0xC0 | encode);
2241 }
2242 
2243 void Assembler::kmovbl(KRegister dst, Register src) {
2244   assert(VM_Version::supports_avx512dq(), "");
2245   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2246   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2247   emit_int8((unsigned char)0x92);
2248   emit_int8((unsigned char)(0xC0 | encode));
2249 }
2250 
2251 void Assembler::kmovbl(Register dst, KRegister src) {
2252   assert(VM_Version::supports_avx512dq(), "");
2253   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2254   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2255   emit_int8((unsigned char)0x93);
2256   emit_int8((unsigned char)(0xC0 | encode));
2257 }
2258 
2259 void Assembler::kmovwl(KRegister dst, Register src) {
2260   assert(VM_Version::supports_evex(), "");
2261   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2262   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2263   emit_int8((unsigned char)0x92);
2264   emit_int8((unsigned char)(0xC0 | encode));
2265 }
2266 
2267 void Assembler::kmovwl(Register dst, KRegister src) {
2268   assert(VM_Version::supports_evex(), "");
2269   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2270   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2271   emit_int8((unsigned char)0x93);
2272   emit_int8((unsigned char)(0xC0 | encode));
2273 }
2274 
2275 void Assembler::kmovwl(KRegister dst, Address src) {
2276   assert(VM_Version::supports_evex(), "");
2277   InstructionMark im(this);
2278   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2279   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2280   emit_int8((unsigned char)0x90);
2281   emit_operand((Register)dst, src);
2282 }
2283 
2284 void Assembler::kmovdl(KRegister dst, Register src) {
2285   assert(VM_Version::supports_avx512bw(), "");
2286   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2287   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2288   emit_int8((unsigned char)0x92);
2289   emit_int8((unsigned char)(0xC0 | encode));
2290 }
2291 
2292 void Assembler::kmovdl(Register dst, KRegister src) {
2293   assert(VM_Version::supports_avx512bw(), "");
2294   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2295   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2296   emit_int8((unsigned char)0x93);
2297   emit_int8((unsigned char)(0xC0 | encode));
2298 }
2299 
2300 void Assembler::kmovql(KRegister dst, KRegister src) {
2301   assert(VM_Version::supports_avx512bw(), "");
2302   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2303   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2304   emit_int8((unsigned char)0x90);
2305   emit_int8((unsigned char)(0xC0 | encode));
2306 }
2307 
2308 void Assembler::kmovql(KRegister dst, Address src) {
2309   assert(VM_Version::supports_avx512bw(), "");
2310   InstructionMark im(this);
2311   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2312   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2313   emit_int8((unsigned char)0x90);
2314   emit_operand((Register)dst, src);
2315 }
2316 
2317 void Assembler::kmovql(Address dst, KRegister src) {
2318   assert(VM_Version::supports_avx512bw(), "");
2319   InstructionMark im(this);
2320   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2321   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2322   emit_int8((unsigned char)0x90);
2323   emit_operand((Register)src, dst);
2324 }
2325 
2326 void Assembler::kmovql(KRegister dst, Register src) {
2327   assert(VM_Version::supports_avx512bw(), "");
2328   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2329   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2330   emit_int8((unsigned char)0x92);
2331   emit_int8((unsigned char)(0xC0 | encode));
2332 }
2333 
2334 void Assembler::kmovql(Register dst, KRegister src) {
2335   assert(VM_Version::supports_avx512bw(), "");
2336   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2337   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2338   emit_int8((unsigned char)0x93);
2339   emit_int8((unsigned char)(0xC0 | encode));
2340 }
2341 
2342 void Assembler::knotwl(KRegister dst, KRegister src) {
2343   assert(VM_Version::supports_evex(), "");
2344   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2345   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2346   emit_int8((unsigned char)0x44);
2347   emit_int8((unsigned char)(0xC0 | encode));
2348 }
2349 
2350 // This instruction produces ZF or CF flags
2351 void Assembler::kortestbl(KRegister src1, KRegister src2) {
2352   assert(VM_Version::supports_avx512dq(), "");
2353   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2354   int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2355   emit_int8((unsigned char)0x98);
2356   emit_int8((unsigned char)(0xC0 | encode));
2357 }
2358 
2359 // This instruction produces ZF or CF flags
2360 void Assembler::kortestwl(KRegister src1, KRegister src2) {
2361   assert(VM_Version::supports_evex(), "");
2362   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2363   int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2364   emit_int8((unsigned char)0x98);
2365   emit_int8((unsigned char)(0xC0 | encode));
2366 }
2367 
2368 // This instruction produces ZF or CF flags
2369 void Assembler::kortestdl(KRegister src1, KRegister src2) {
2370   assert(VM_Version::supports_avx512bw(), "");
2371   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2372   int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2373   emit_int8((unsigned char)0x98);
2374   emit_int8((unsigned char)(0xC0 | encode));
2375 }
2376 
2377 // This instruction produces ZF or CF flags
2378 void Assembler::kortestql(KRegister src1, KRegister src2) {
2379   assert(VM_Version::supports_avx512bw(), "");
2380   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2381   int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2382   emit_int8((unsigned char)0x98);
2383   emit_int8((unsigned char)(0xC0 | encode));
2384 }
2385 
2386 // This instruction produces ZF or CF flags
2387 void Assembler::ktestql(KRegister src1, KRegister src2) {
2388   assert(VM_Version::supports_avx512bw(), "");
2389   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2390   int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2391   emit_int8((unsigned char)0x99);
2392   emit_int8((unsigned char)(0xC0 | encode));
2393 }
2394 
2395 void Assembler::ktestq(KRegister src1, KRegister src2) {
2396   assert(VM_Version::supports_avx512bw(), "");
2397   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2398   int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2399   emit_int8((unsigned char)0x99);
2400   emit_int8((unsigned char)(0xC0 | encode));
2401 }
2402 
2403 void Assembler::ktestd(KRegister src1, KRegister src2) {
2404   assert(VM_Version::supports_avx512bw(), "");
2405   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2406   int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2407   emit_int8((unsigned char)0x99);
2408   emit_int8((unsigned char)(0xC0 | encode));
2409 }
2410 
2411 void Assembler::movb(Address dst, int imm8) {
2412   InstructionMark im(this);
2413    prefix(dst);
2414   emit_int8((unsigned char)0xC6);
2415   emit_operand(rax, dst, 1);
2416   emit_int8(imm8);
2417 }
2418 
2419 
2420 void Assembler::movb(Address dst, Register src) {
2421   assert(src->has_byte_register(), "must have byte register");
2422   InstructionMark im(this);
2423   prefix(dst, src, true);
2424   emit_int8((unsigned char)0x88);
2425   emit_operand(src, dst);
2426 }
2427 
2428 void Assembler::movdl(XMMRegister dst, Register src) {
2429   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2430   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2431   int encode = simd_prefix_and_encode(dst, xnoreg, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2432   emit_int8(0x6E);
2433   emit_int8((unsigned char)(0xC0 | encode));
2434 }
2435 
2436 void Assembler::movdl(Register dst, XMMRegister src) {
2437   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2438   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2439   // swap src/dst to get correct prefix
2440   int encode = simd_prefix_and_encode(src, xnoreg, as_XMMRegister(dst->encoding()), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2441   emit_int8(0x7E);
2442   emit_int8((unsigned char)(0xC0 | encode));
2443 }
2444 
2445 void Assembler::movdl(XMMRegister dst, Address src) {
2446   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2447   InstructionMark im(this);
2448   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2449   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
2450   simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2451   emit_int8(0x6E);
2452   emit_operand(dst, src);
2453 }
2454 
2455 void Assembler::movdl(Address dst, XMMRegister src) {
2456   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2457   InstructionMark im(this);
2458   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2459   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
2460   simd_prefix(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2461   emit_int8(0x7E);
2462   emit_operand(src, dst);
2463 }
2464 
2465 void Assembler::movdqa(XMMRegister dst, XMMRegister src) {
2466   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2467   int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_128bit;
2468   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2469   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2470   emit_int8(0x6F);
2471   emit_int8((unsigned char)(0xC0 | encode));
2472 }
2473 
2474 void Assembler::movdqa(XMMRegister dst, Address src) {
2475   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2476   InstructionMark im(this);
2477   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2478   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2479   simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2480   emit_int8(0x6F);
2481   emit_operand(dst, src);
2482 }
2483 
2484 void Assembler::movdqu(XMMRegister dst, Address src) {
2485   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2486   InstructionMark im(this);
2487   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2488   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2489   simd_prefix(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2490   emit_int8(0x6F);
2491   emit_operand(dst, src);
2492 }
2493 
2494 void Assembler::movdqu(XMMRegister dst, XMMRegister src) {
2495   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2496   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2497   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2498   emit_int8(0x6F);
2499   emit_int8((unsigned char)(0xC0 | encode));
2500 }
2501 
2502 void Assembler::movdqu(Address dst, XMMRegister src) {
2503   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2504   InstructionMark im(this);
2505   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2506   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2507   attributes.reset_is_clear_context();
2508   simd_prefix(src, xnoreg, dst, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2509   emit_int8(0x7F);
2510   emit_operand(src, dst);
2511 }
2512 
2513 // Move Unaligned 256bit Vector
2514 void Assembler::vmovdqu(XMMRegister dst, XMMRegister src) {
2515   assert(UseAVX > 0, "");
2516   InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2517   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2518   emit_int8(0x6F);
2519   emit_int8((unsigned char)(0xC0 | encode));
2520 }
2521 
2522 void Assembler::vmovdqu(XMMRegister dst, Address src) {
2523   assert(UseAVX > 0, "");
2524   InstructionMark im(this);
2525   InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2526   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2527   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2528   emit_int8(0x6F);
2529   emit_operand(dst, src);
2530 }
2531 
2532 void Assembler::vmovdqu(Address dst, XMMRegister src) {
2533   assert(UseAVX > 0, "");
2534   InstructionMark im(this);
2535   InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2536   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2537   attributes.reset_is_clear_context();
2538   // swap src<->dst for encoding
2539   assert(src != xnoreg, "sanity");
2540   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2541   emit_int8(0x7F);
2542   emit_operand(src, dst);
2543 }
2544 
2545 // Move Unaligned EVEX enabled Vector (programmable : 8,16,32,64)
2546 void Assembler::evmovdqub(XMMRegister dst, XMMRegister src, int vector_len) {
2547   assert(VM_Version::supports_evex(), "");
2548   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
2549   attributes.set_is_evex_instruction();
2550   int prefix = (_legacy_mode_bw) ? VEX_SIMD_F2 : VEX_SIMD_F3;
2551   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), (Assembler::VexSimdPrefix)prefix, VEX_OPCODE_0F, &attributes);
2552   emit_int8(0x6F);
2553   emit_int8((unsigned char)(0xC0 | encode));
2554 }
2555 
2556 void Assembler::evmovdqub(XMMRegister dst, Address src, int vector_len) {
2557   assert(VM_Version::supports_evex(), "");
2558   InstructionMark im(this);
2559   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
2560   int prefix = (_legacy_mode_bw) ? VEX_SIMD_F2 : VEX_SIMD_F3;
2561   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2562   attributes.set_is_evex_instruction();
2563   vex_prefix(src, 0, dst->encoding(), (Assembler::VexSimdPrefix)prefix, VEX_OPCODE_0F, &attributes);
2564   emit_int8(0x6F);
2565   emit_operand(dst, src);
2566 }
2567 
2568 void Assembler::evmovdqub(Address dst, XMMRegister src, int vector_len) {
2569   assert(VM_Version::supports_evex(), "");
2570   assert(src != xnoreg, "sanity");
2571   InstructionMark im(this);
2572   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
2573   int prefix = (_legacy_mode_bw) ? VEX_SIMD_F2 : VEX_SIMD_F3;
2574   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2575   attributes.set_is_evex_instruction();
2576   vex_prefix(dst, 0, src->encoding(), (Assembler::VexSimdPrefix)prefix, VEX_OPCODE_0F, &attributes);
2577   emit_int8(0x7F);
2578   emit_operand(src, dst);
2579 }
2580 
2581 void Assembler::evmovdqub(XMMRegister dst, KRegister mask, Address src, int vector_len) {
2582   assert(VM_Version::supports_avx512vlbw(), "");
2583   assert(is_vector_masking(), "");    // For stub code use only
2584   InstructionMark im(this);
2585   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
2586   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2587   attributes.set_embedded_opmask_register_specifier(mask);
2588   attributes.set_is_evex_instruction();
2589   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2590   emit_int8(0x6F);
2591   emit_operand(dst, src);
2592 }
2593 
2594 void Assembler::evmovdquw(XMMRegister dst, Address src, int vector_len) {
2595   assert(VM_Version::supports_evex(), "");
2596   InstructionMark im(this);
2597   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
2598   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2599   attributes.set_is_evex_instruction();
2600   int prefix = (_legacy_mode_bw) ? VEX_SIMD_F2 : VEX_SIMD_F3;
2601   vex_prefix(src, 0, dst->encoding(), (Assembler::VexSimdPrefix)prefix, VEX_OPCODE_0F, &attributes);
2602   emit_int8(0x6F);
2603   emit_operand(dst, src);
2604 }
2605 
2606 void Assembler::evmovdquw(XMMRegister dst, KRegister mask, Address src, int vector_len) {
2607   assert(is_vector_masking(), "");
2608   assert(VM_Version::supports_avx512vlbw(), "");
2609   InstructionMark im(this);
2610   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
2611   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2612   attributes.set_embedded_opmask_register_specifier(mask);
2613   attributes.set_is_evex_instruction();
2614   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2615   emit_int8(0x6F);
2616   emit_operand(dst, src);
2617 }
2618 
2619 void Assembler::evmovdquw(Address dst, XMMRegister src, int vector_len) {
2620   assert(VM_Version::supports_evex(), "");
2621   assert(src != xnoreg, "sanity");
2622   InstructionMark im(this);
2623   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
2624   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2625   attributes.set_is_evex_instruction();
2626   int prefix = (_legacy_mode_bw) ? VEX_SIMD_F2 : VEX_SIMD_F3;
2627   vex_prefix(dst, 0, src->encoding(), (Assembler::VexSimdPrefix)prefix, VEX_OPCODE_0F, &attributes);
2628   emit_int8(0x7F);
2629   emit_operand(src, dst);
2630 }
2631 
2632 void Assembler::evmovdquw(Address dst, KRegister mask, XMMRegister src, int vector_len) {
2633   assert(VM_Version::supports_avx512vlbw(), "");
2634   assert(src != xnoreg, "sanity");
2635   InstructionMark im(this);
2636   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2637   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2638   attributes.reset_is_clear_context();
2639   attributes.set_embedded_opmask_register_specifier(mask);
2640   attributes.set_is_evex_instruction();
2641   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2642   emit_int8(0x7F);
2643   emit_operand(src, dst);
2644 }
2645 
2646 void Assembler::evmovdqul(XMMRegister dst, XMMRegister src, int vector_len) {
2647   assert(VM_Version::supports_evex(), "");
2648   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2649   attributes.set_is_evex_instruction();
2650   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2651   emit_int8(0x6F);
2652   emit_int8((unsigned char)(0xC0 | encode));
2653 }
2654 
2655 void Assembler::evmovdqul(XMMRegister dst, Address src, int vector_len) {
2656   assert(VM_Version::supports_evex(), "");
2657   InstructionMark im(this);
2658   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false , /* uses_vl */ true);
2659   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2660   attributes.set_is_evex_instruction();
2661   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2662   emit_int8(0x6F);
2663   emit_operand(dst, src);
2664 }
2665 
2666 void Assembler::evmovdqul(Address dst, XMMRegister src, int vector_len) {
2667   assert(VM_Version::supports_evex(), "");
2668   assert(src != xnoreg, "sanity");
2669   InstructionMark im(this);
2670   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2671   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2672   attributes.reset_is_clear_context();
2673   attributes.set_is_evex_instruction();
2674   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2675   emit_int8(0x7F);
2676   emit_operand(src, dst);
2677 }
2678 
2679 void Assembler::evmovdquq(XMMRegister dst, XMMRegister src, int vector_len) {
2680   assert(VM_Version::supports_evex(), "");
2681   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2682   attributes.set_is_evex_instruction();
2683   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2684   emit_int8(0x6F);
2685   emit_int8((unsigned char)(0xC0 | encode));
2686 }
2687 
2688 void Assembler::evmovdquq(XMMRegister dst, Address src, int vector_len) {
2689   assert(VM_Version::supports_evex(), "");
2690   InstructionMark im(this);
2691   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2692   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2693   attributes.set_is_evex_instruction();
2694   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2695   emit_int8(0x6F);
2696   emit_operand(dst, src);
2697 }
2698 
2699 void Assembler::evmovdquq(Address dst, XMMRegister src, int vector_len) {
2700   assert(VM_Version::supports_evex(), "");
2701   assert(src != xnoreg, "sanity");
2702   InstructionMark im(this);
2703   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2704   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2705   attributes.reset_is_clear_context();
2706   attributes.set_is_evex_instruction();
2707   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2708   emit_int8(0x7F);
2709   emit_operand(src, dst);
2710 }
2711 
2712 // Uses zero extension on 64bit
2713 
2714 void Assembler::movl(Register dst, int32_t imm32) {
2715   int encode = prefix_and_encode(dst->encoding());
2716   emit_int8((unsigned char)(0xB8 | encode));
2717   emit_int32(imm32);
2718 }
2719 
2720 void Assembler::movl(Register dst, Register src) {
2721   int encode = prefix_and_encode(dst->encoding(), src->encoding());
2722   emit_int8((unsigned char)0x8B);
2723   emit_int8((unsigned char)(0xC0 | encode));
2724 }
2725 
2726 void Assembler::movl(Register dst, Address src) {
2727   InstructionMark im(this);
2728   prefix(src, dst);
2729   emit_int8((unsigned char)0x8B);
2730   emit_operand(dst, src);
2731 }
2732 
2733 void Assembler::movl(Address dst, int32_t imm32) {
2734   InstructionMark im(this);
2735   prefix(dst);
2736   emit_int8((unsigned char)0xC7);
2737   emit_operand(rax, dst, 4);
2738   emit_int32(imm32);
2739 }
2740 
2741 void Assembler::movl(Address dst, Register src) {
2742   InstructionMark im(this);
2743   prefix(dst, src);
2744   emit_int8((unsigned char)0x89);
2745   emit_operand(src, dst);
2746 }
2747 
2748 // New cpus require to use movsd and movss to avoid partial register stall
2749 // when loading from memory. But for old Opteron use movlpd instead of movsd.
2750 // The selection is done in MacroAssembler::movdbl() and movflt().
2751 void Assembler::movlpd(XMMRegister dst, Address src) {
2752   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2753   InstructionMark im(this);
2754   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2755   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
2756   attributes.set_rex_vex_w_reverted();
2757   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2758   emit_int8(0x12);
2759   emit_operand(dst, src);
2760 }
2761 
2762 void Assembler::movq( MMXRegister dst, Address src ) {
2763   assert( VM_Version::supports_mmx(), "" );
2764   emit_int8(0x0F);
2765   emit_int8(0x6F);
2766   emit_operand(dst, src);
2767 }
2768 
2769 void Assembler::movq( Address dst, MMXRegister src ) {
2770   assert( VM_Version::supports_mmx(), "" );
2771   emit_int8(0x0F);
2772   emit_int8(0x7F);
2773   // workaround gcc (3.2.1-7a) bug
2774   // In that version of gcc with only an emit_operand(MMX, Address)
2775   // gcc will tail jump and try and reverse the parameters completely
2776   // obliterating dst in the process. By having a version available
2777   // that doesn't need to swap the args at the tail jump the bug is
2778   // avoided.
2779   emit_operand(dst, src);
2780 }
2781 
2782 void Assembler::movq(XMMRegister dst, Address src) {
2783   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2784   InstructionMark im(this);
2785   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2786   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
2787   attributes.set_rex_vex_w_reverted();
2788   simd_prefix(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2789   emit_int8(0x7E);
2790   emit_operand(dst, src);
2791 }
2792 
2793 void Assembler::movq(Address dst, XMMRegister src) {
2794   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2795   InstructionMark im(this);
2796   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2797   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
2798   attributes.set_rex_vex_w_reverted();
2799   simd_prefix(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2800   emit_int8((unsigned char)0xD6);
2801   emit_operand(src, dst);
2802 }
2803 
2804 void Assembler::movsbl(Register dst, Address src) { // movsxb
2805   InstructionMark im(this);
2806   prefix(src, dst);
2807   emit_int8(0x0F);
2808   emit_int8((unsigned char)0xBE);
2809   emit_operand(dst, src);
2810 }
2811 
2812 void Assembler::movsbl(Register dst, Register src) { // movsxb
2813   NOT_LP64(assert(src->has_byte_register(), "must have byte register"));
2814   int encode = prefix_and_encode(dst->encoding(), false, src->encoding(), true);
2815   emit_int8(0x0F);
2816   emit_int8((unsigned char)0xBE);
2817   emit_int8((unsigned char)(0xC0 | encode));
2818 }
2819 
2820 void Assembler::movsd(XMMRegister dst, XMMRegister src) {
2821   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2822   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2823   attributes.set_rex_vex_w_reverted();
2824   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2825   emit_int8(0x10);
2826   emit_int8((unsigned char)(0xC0 | encode));
2827 }
2828 
2829 void Assembler::movsd(XMMRegister dst, Address src) {
2830   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2831   InstructionMark im(this);
2832   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2833   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
2834   attributes.set_rex_vex_w_reverted();
2835   simd_prefix(dst, xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2836   emit_int8(0x10);
2837   emit_operand(dst, src);
2838 }
2839 
2840 void Assembler::movsd(Address dst, XMMRegister src) {
2841   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2842   InstructionMark im(this);
2843   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2844   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
2845   attributes.reset_is_clear_context();
2846   attributes.set_rex_vex_w_reverted();
2847   simd_prefix(src, xnoreg, dst, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2848   emit_int8(0x11);
2849   emit_operand(src, dst);
2850 }
2851 
2852 void Assembler::movss(XMMRegister dst, XMMRegister src) {
2853   NOT_LP64(assert(VM_Version::supports_sse(), ""));
2854   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2855   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2856   emit_int8(0x10);
2857   emit_int8((unsigned char)(0xC0 | encode));
2858 }
2859 
2860 void Assembler::movss(XMMRegister dst, Address src) {
2861   NOT_LP64(assert(VM_Version::supports_sse(), ""));
2862   InstructionMark im(this);
2863   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2864   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
2865   simd_prefix(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2866   emit_int8(0x10);
2867   emit_operand(dst, src);
2868 }
2869 
2870 void Assembler::movss(Address dst, XMMRegister src) {
2871   NOT_LP64(assert(VM_Version::supports_sse(), ""));
2872   InstructionMark im(this);
2873   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2874   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
2875   attributes.reset_is_clear_context();
2876   simd_prefix(src, xnoreg, dst, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2877   emit_int8(0x11);
2878   emit_operand(src, dst);
2879 }
2880 
2881 void Assembler::movswl(Register dst, Address src) { // movsxw
2882   InstructionMark im(this);
2883   prefix(src, dst);
2884   emit_int8(0x0F);
2885   emit_int8((unsigned char)0xBF);
2886   emit_operand(dst, src);
2887 }
2888 
2889 void Assembler::movswl(Register dst, Register src) { // movsxw
2890   int encode = prefix_and_encode(dst->encoding(), src->encoding());
2891   emit_int8(0x0F);
2892   emit_int8((unsigned char)0xBF);
2893   emit_int8((unsigned char)(0xC0 | encode));
2894 }
2895 
2896 void Assembler::movw(Address dst, int imm16) {
2897   InstructionMark im(this);
2898 
2899   emit_int8(0x66); // switch to 16-bit mode
2900   prefix(dst);
2901   emit_int8((unsigned char)0xC7);
2902   emit_operand(rax, dst, 2);
2903   emit_int16(imm16);
2904 }
2905 
2906 void Assembler::movw(Register dst, Address src) {
2907   InstructionMark im(this);
2908   emit_int8(0x66);
2909   prefix(src, dst);
2910   emit_int8((unsigned char)0x8B);
2911   emit_operand(dst, src);
2912 }
2913 
2914 void Assembler::movw(Address dst, Register src) {
2915   InstructionMark im(this);
2916   emit_int8(0x66);
2917   prefix(dst, src);
2918   emit_int8((unsigned char)0x89);
2919   emit_operand(src, dst);
2920 }
2921 
2922 void Assembler::movzbl(Register dst, Address src) { // movzxb
2923   InstructionMark im(this);
2924   prefix(src, dst);
2925   emit_int8(0x0F);
2926   emit_int8((unsigned char)0xB6);
2927   emit_operand(dst, src);
2928 }
2929 
2930 void Assembler::movzbl(Register dst, Register src) { // movzxb
2931   NOT_LP64(assert(src->has_byte_register(), "must have byte register"));
2932   int encode = prefix_and_encode(dst->encoding(), false, src->encoding(), true);
2933   emit_int8(0x0F);
2934   emit_int8((unsigned char)0xB6);
2935   emit_int8(0xC0 | encode);
2936 }
2937 
2938 void Assembler::movzwl(Register dst, Address src) { // movzxw
2939   InstructionMark im(this);
2940   prefix(src, dst);
2941   emit_int8(0x0F);
2942   emit_int8((unsigned char)0xB7);
2943   emit_operand(dst, src);
2944 }
2945 
2946 void Assembler::movzwl(Register dst, Register src) { // movzxw
2947   int encode = prefix_and_encode(dst->encoding(), src->encoding());
2948   emit_int8(0x0F);
2949   emit_int8((unsigned char)0xB7);
2950   emit_int8(0xC0 | encode);
2951 }
2952 
2953 void Assembler::mull(Address src) {
2954   InstructionMark im(this);
2955   prefix(src);
2956   emit_int8((unsigned char)0xF7);
2957   emit_operand(rsp, src);
2958 }
2959 
2960 void Assembler::mull(Register src) {
2961   int encode = prefix_and_encode(src->encoding());
2962   emit_int8((unsigned char)0xF7);
2963   emit_int8((unsigned char)(0xE0 | encode));
2964 }
2965 
2966 void Assembler::mulsd(XMMRegister dst, Address src) {
2967   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2968   InstructionMark im(this);
2969   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2970   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
2971   attributes.set_rex_vex_w_reverted();
2972   simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2973   emit_int8(0x59);
2974   emit_operand(dst, src);
2975 }
2976 
2977 void Assembler::mulsd(XMMRegister dst, XMMRegister src) {
2978   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2979   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2980   attributes.set_rex_vex_w_reverted();
2981   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2982   emit_int8(0x59);
2983   emit_int8((unsigned char)(0xC0 | encode));
2984 }
2985 
2986 void Assembler::mulss(XMMRegister dst, Address src) {
2987   NOT_LP64(assert(VM_Version::supports_sse(), ""));
2988   InstructionMark im(this);
2989   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2990   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
2991   simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2992   emit_int8(0x59);
2993   emit_operand(dst, src);
2994 }
2995 
2996 void Assembler::mulss(XMMRegister dst, XMMRegister src) {
2997   NOT_LP64(assert(VM_Version::supports_sse(), ""));
2998   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2999   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
3000   emit_int8(0x59);
3001   emit_int8((unsigned char)(0xC0 | encode));
3002 }
3003 
3004 void Assembler::negl(Register dst) {
3005   int encode = prefix_and_encode(dst->encoding());
3006   emit_int8((unsigned char)0xF7);
3007   emit_int8((unsigned char)(0xD8 | encode));
3008 }
3009 
3010 void Assembler::nop(int i) {
3011 #ifdef ASSERT
3012   assert(i > 0, " ");
3013   // The fancy nops aren't currently recognized by debuggers making it a
3014   // pain to disassemble code while debugging. If asserts are on clearly
3015   // speed is not an issue so simply use the single byte traditional nop
3016   // to do alignment.
3017 
3018   for (; i > 0 ; i--) emit_int8((unsigned char)0x90);
3019   return;
3020 
3021 #endif // ASSERT
3022 
3023   if (UseAddressNop && VM_Version::is_intel()) {
3024     //
3025     // Using multi-bytes nops "0x0F 0x1F [address]" for Intel
3026     //  1: 0x90
3027     //  2: 0x66 0x90
3028     //  3: 0x66 0x66 0x90 (don't use "0x0F 0x1F 0x00" - need patching safe padding)
3029     //  4: 0x0F 0x1F 0x40 0x00
3030     //  5: 0x0F 0x1F 0x44 0x00 0x00
3031     //  6: 0x66 0x0F 0x1F 0x44 0x00 0x00
3032     //  7: 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00
3033     //  8: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
3034     //  9: 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
3035     // 10: 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
3036     // 11: 0x66 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
3037 
3038     // The rest coding is Intel specific - don't use consecutive address nops
3039 
3040     // 12: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
3041     // 13: 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
3042     // 14: 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
3043     // 15: 0x66 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
3044 
3045     while(i >= 15) {
3046       // For Intel don't generate consecutive addess nops (mix with regular nops)
3047       i -= 15;
3048       emit_int8(0x66);   // size prefix
3049       emit_int8(0x66);   // size prefix
3050       emit_int8(0x66);   // size prefix
3051       addr_nop_8();
3052       emit_int8(0x66);   // size prefix
3053       emit_int8(0x66);   // size prefix
3054       emit_int8(0x66);   // size prefix
3055       emit_int8((unsigned char)0x90);
3056                          // nop
3057     }
3058     switch (i) {
3059       case 14:
3060         emit_int8(0x66); // size prefix
3061       case 13:
3062         emit_int8(0x66); // size prefix
3063       case 12:
3064         addr_nop_8();
3065         emit_int8(0x66); // size prefix
3066         emit_int8(0x66); // size prefix
3067         emit_int8(0x66); // size prefix
3068         emit_int8((unsigned char)0x90);
3069                          // nop
3070         break;
3071       case 11:
3072         emit_int8(0x66); // size prefix
3073       case 10:
3074         emit_int8(0x66); // size prefix
3075       case 9:
3076         emit_int8(0x66); // size prefix
3077       case 8:
3078         addr_nop_8();
3079         break;
3080       case 7:
3081         addr_nop_7();
3082         break;
3083       case 6:
3084         emit_int8(0x66); // size prefix
3085       case 5:
3086         addr_nop_5();
3087         break;
3088       case 4:
3089         addr_nop_4();
3090         break;
3091       case 3:
3092         // Don't use "0x0F 0x1F 0x00" - need patching safe padding
3093         emit_int8(0x66); // size prefix
3094       case 2:
3095         emit_int8(0x66); // size prefix
3096       case 1:
3097         emit_int8((unsigned char)0x90);
3098                          // nop
3099         break;
3100       default:
3101         assert(i == 0, " ");
3102     }
3103     return;
3104   }
3105   if (UseAddressNop && VM_Version::is_amd()) {
3106     //
3107     // Using multi-bytes nops "0x0F 0x1F [address]" for AMD.
3108     //  1: 0x90
3109     //  2: 0x66 0x90
3110     //  3: 0x66 0x66 0x90 (don't use "0x0F 0x1F 0x00" - need patching safe padding)
3111     //  4: 0x0F 0x1F 0x40 0x00
3112     //  5: 0x0F 0x1F 0x44 0x00 0x00
3113     //  6: 0x66 0x0F 0x1F 0x44 0x00 0x00
3114     //  7: 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00
3115     //  8: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
3116     //  9: 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
3117     // 10: 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
3118     // 11: 0x66 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
3119 
3120     // The rest coding is AMD specific - use consecutive address nops
3121 
3122     // 12: 0x66 0x0F 0x1F 0x44 0x00 0x00 0x66 0x0F 0x1F 0x44 0x00 0x00
3123     // 13: 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00 0x66 0x0F 0x1F 0x44 0x00 0x00
3124     // 14: 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00
3125     // 15: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00
3126     // 16: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
3127     //     Size prefixes (0x66) are added for larger sizes
3128 
3129     while(i >= 22) {
3130       i -= 11;
3131       emit_int8(0x66); // size prefix
3132       emit_int8(0x66); // size prefix
3133       emit_int8(0x66); // size prefix
3134       addr_nop_8();
3135     }
3136     // Generate first nop for size between 21-12
3137     switch (i) {
3138       case 21:
3139         i -= 1;
3140         emit_int8(0x66); // size prefix
3141       case 20:
3142       case 19:
3143         i -= 1;
3144         emit_int8(0x66); // size prefix
3145       case 18:
3146       case 17:
3147         i -= 1;
3148         emit_int8(0x66); // size prefix
3149       case 16:
3150       case 15:
3151         i -= 8;
3152         addr_nop_8();
3153         break;
3154       case 14:
3155       case 13:
3156         i -= 7;
3157         addr_nop_7();
3158         break;
3159       case 12:
3160         i -= 6;
3161         emit_int8(0x66); // size prefix
3162         addr_nop_5();
3163         break;
3164       default:
3165         assert(i < 12, " ");
3166     }
3167 
3168     // Generate second nop for size between 11-1
3169     switch (i) {
3170       case 11:
3171         emit_int8(0x66); // size prefix
3172       case 10:
3173         emit_int8(0x66); // size prefix
3174       case 9:
3175         emit_int8(0x66); // size prefix
3176       case 8:
3177         addr_nop_8();
3178         break;
3179       case 7:
3180         addr_nop_7();
3181         break;
3182       case 6:
3183         emit_int8(0x66); // size prefix
3184       case 5:
3185         addr_nop_5();
3186         break;
3187       case 4:
3188         addr_nop_4();
3189         break;
3190       case 3:
3191         // Don't use "0x0F 0x1F 0x00" - need patching safe padding
3192         emit_int8(0x66); // size prefix
3193       case 2:
3194         emit_int8(0x66); // size prefix
3195       case 1:
3196         emit_int8((unsigned char)0x90);
3197                          // nop
3198         break;
3199       default:
3200         assert(i == 0, " ");
3201     }
3202     return;
3203   }
3204 
3205   if (UseAddressNop && VM_Version::is_zx()) {
3206     //
3207     // Using multi-bytes nops "0x0F 0x1F [address]" for ZX
3208     //  1: 0x90
3209     //  2: 0x66 0x90
3210     //  3: 0x66 0x66 0x90 (don't use "0x0F 0x1F 0x00" - need patching safe padding)
3211     //  4: 0x0F 0x1F 0x40 0x00
3212     //  5: 0x0F 0x1F 0x44 0x00 0x00
3213     //  6: 0x66 0x0F 0x1F 0x44 0x00 0x00
3214     //  7: 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00
3215     //  8: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
3216     //  9: 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
3217     // 10: 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
3218     // 11: 0x66 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
3219 
3220     // The rest coding is ZX specific - don't use consecutive address nops
3221 
3222     // 12: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
3223     // 13: 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
3224     // 14: 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
3225     // 15: 0x66 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
3226 
3227     while (i >= 15) {
3228       // For ZX don't generate consecutive addess nops (mix with regular nops)
3229       i -= 15;
3230       emit_int8(0x66);   // size prefix
3231       emit_int8(0x66);   // size prefix
3232       emit_int8(0x66);   // size prefix
3233       addr_nop_8();
3234       emit_int8(0x66);   // size prefix
3235       emit_int8(0x66);   // size prefix
3236       emit_int8(0x66);   // size prefix
3237       emit_int8((unsigned char)0x90);
3238                          // nop
3239     }
3240     switch (i) {
3241       case 14:
3242         emit_int8(0x66); // size prefix
3243       case 13:
3244         emit_int8(0x66); // size prefix
3245       case 12:
3246         addr_nop_8();
3247         emit_int8(0x66); // size prefix
3248         emit_int8(0x66); // size prefix
3249         emit_int8(0x66); // size prefix
3250         emit_int8((unsigned char)0x90);
3251                          // nop
3252         break;
3253       case 11:
3254         emit_int8(0x66); // size prefix
3255       case 10:
3256         emit_int8(0x66); // size prefix
3257       case 9:
3258         emit_int8(0x66); // size prefix
3259       case 8:
3260         addr_nop_8();
3261         break;
3262       case 7:
3263         addr_nop_7();
3264         break;
3265       case 6:
3266         emit_int8(0x66); // size prefix
3267       case 5:
3268         addr_nop_5();
3269         break;
3270       case 4:
3271         addr_nop_4();
3272         break;
3273       case 3:
3274         // Don't use "0x0F 0x1F 0x00" - need patching safe padding
3275         emit_int8(0x66); // size prefix
3276       case 2:
3277         emit_int8(0x66); // size prefix
3278       case 1:
3279         emit_int8((unsigned char)0x90);
3280                          // nop
3281         break;
3282       default:
3283         assert(i == 0, " ");
3284     }
3285     return;
3286   }
3287 
3288   // Using nops with size prefixes "0x66 0x90".
3289   // From AMD Optimization Guide:
3290   //  1: 0x90
3291   //  2: 0x66 0x90
3292   //  3: 0x66 0x66 0x90
3293   //  4: 0x66 0x66 0x66 0x90
3294   //  5: 0x66 0x66 0x90 0x66 0x90
3295   //  6: 0x66 0x66 0x90 0x66 0x66 0x90
3296   //  7: 0x66 0x66 0x66 0x90 0x66 0x66 0x90
3297   //  8: 0x66 0x66 0x66 0x90 0x66 0x66 0x66 0x90
3298   //  9: 0x66 0x66 0x90 0x66 0x66 0x90 0x66 0x66 0x90
3299   // 10: 0x66 0x66 0x66 0x90 0x66 0x66 0x90 0x66 0x66 0x90
3300   //
3301   while(i > 12) {
3302     i -= 4;
3303     emit_int8(0x66); // size prefix
3304     emit_int8(0x66);
3305     emit_int8(0x66);
3306     emit_int8((unsigned char)0x90);
3307                      // nop
3308   }
3309   // 1 - 12 nops
3310   if(i > 8) {
3311     if(i > 9) {
3312       i -= 1;
3313       emit_int8(0x66);
3314     }
3315     i -= 3;
3316     emit_int8(0x66);
3317     emit_int8(0x66);
3318     emit_int8((unsigned char)0x90);
3319   }
3320   // 1 - 8 nops
3321   if(i > 4) {
3322     if(i > 6) {
3323       i -= 1;
3324       emit_int8(0x66);
3325     }
3326     i -= 3;
3327     emit_int8(0x66);
3328     emit_int8(0x66);
3329     emit_int8((unsigned char)0x90);
3330   }
3331   switch (i) {
3332     case 4:
3333       emit_int8(0x66);
3334     case 3:
3335       emit_int8(0x66);
3336     case 2:
3337       emit_int8(0x66);
3338     case 1:
3339       emit_int8((unsigned char)0x90);
3340       break;
3341     default:
3342       assert(i == 0, " ");
3343   }
3344 }
3345 
3346 void Assembler::notl(Register dst) {
3347   int encode = prefix_and_encode(dst->encoding());
3348   emit_int8((unsigned char)0xF7);
3349   emit_int8((unsigned char)(0xD0 | encode));
3350 }
3351 
3352 void Assembler::orl(Address dst, int32_t imm32) {
3353   InstructionMark im(this);
3354   prefix(dst);
3355   emit_arith_operand(0x81, rcx, dst, imm32);
3356 }
3357 
3358 void Assembler::orl(Register dst, int32_t imm32) {
3359   prefix(dst);
3360   emit_arith(0x81, 0xC8, dst, imm32);
3361 }
3362 
3363 void Assembler::orl(Register dst, Address src) {
3364   InstructionMark im(this);
3365   prefix(src, dst);
3366   emit_int8(0x0B);
3367   emit_operand(dst, src);
3368 }
3369 
3370 void Assembler::orl(Register dst, Register src) {
3371   (void) prefix_and_encode(dst->encoding(), src->encoding());
3372   emit_arith(0x0B, 0xC0, dst, src);
3373 }
3374 
3375 void Assembler::orl(Address dst, Register src) {
3376   InstructionMark im(this);
3377   prefix(dst, src);
3378   emit_int8(0x09);
3379   emit_operand(src, dst);
3380 }
3381 
3382 void Assembler::orb(Address dst, int imm8) {
3383   InstructionMark im(this);
3384   prefix(dst);
3385   emit_int8((unsigned char)0x80);
3386   emit_operand(rcx, dst, 1);
3387   emit_int8(imm8);
3388 }
3389 
3390 void Assembler::packuswb(XMMRegister dst, Address src) {
3391   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3392   assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
3393   InstructionMark im(this);
3394   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
3395   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
3396   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3397   emit_int8(0x67);
3398   emit_operand(dst, src);
3399 }
3400 
3401 void Assembler::packuswb(XMMRegister dst, XMMRegister src) {
3402   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3403   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
3404   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3405   emit_int8(0x67);
3406   emit_int8((unsigned char)(0xC0 | encode));
3407 }
3408 
3409 void Assembler::vpackuswb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
3410   assert(UseAVX > 0, "some form of AVX must be enabled");
3411   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
3412   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3413   emit_int8(0x67);
3414   emit_int8((unsigned char)(0xC0 | encode));
3415 }
3416 
3417 void Assembler::vpermq(XMMRegister dst, XMMRegister src, int imm8, int vector_len) {
3418   assert(VM_Version::supports_avx2(), "");
3419   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
3420   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3421   emit_int8(0x00);
3422   emit_int8(0xC0 | encode);
3423   emit_int8(imm8);
3424 }
3425 
3426 void Assembler::vperm2i128(XMMRegister dst,  XMMRegister nds, XMMRegister src, int imm8) {
3427   assert(VM_Version::supports_avx2(), "");
3428   InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3429   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3430   emit_int8(0x46);
3431   emit_int8(0xC0 | encode);
3432   emit_int8(imm8);
3433 }
3434 
3435 void Assembler::vperm2f128(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8) {
3436   assert(VM_Version::supports_avx(), "");
3437   InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3438   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3439   emit_int8(0x06);
3440   emit_int8(0xC0 | encode);
3441   emit_int8(imm8);
3442 }
3443 
3444 void Assembler::evpermi2q(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
3445   assert(VM_Version::supports_evex(), "");
3446   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
3447   attributes.set_is_evex_instruction();
3448   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3449   emit_int8(0x76);
3450   emit_int8((unsigned char)(0xC0 | encode));
3451 }
3452 
3453 
3454 void Assembler::pause() {
3455   emit_int8((unsigned char)0xF3);
3456   emit_int8((unsigned char)0x90);
3457 }
3458 
3459 void Assembler::ud2() {
3460   emit_int8(0x0F);
3461   emit_int8(0x0B);
3462 }
3463 
3464 void Assembler::pcmpestri(XMMRegister dst, Address src, int imm8) {
3465   assert(VM_Version::supports_sse4_2(), "");
3466   InstructionMark im(this);
3467   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3468   simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3469   emit_int8(0x61);
3470   emit_operand(dst, src);
3471   emit_int8(imm8);
3472 }
3473 
3474 void Assembler::pcmpestri(XMMRegister dst, XMMRegister src, int imm8) {
3475   assert(VM_Version::supports_sse4_2(), "");
3476   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3477   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3478   emit_int8(0x61);
3479   emit_int8((unsigned char)(0xC0 | encode));
3480   emit_int8(imm8);
3481 }
3482 
3483 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
3484 void Assembler::pcmpeqb(XMMRegister dst, XMMRegister src) {
3485   assert(VM_Version::supports_sse2(), "");
3486   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3487   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3488   emit_int8(0x74);
3489   emit_int8((unsigned char)(0xC0 | encode));
3490 }
3491 
3492 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
3493 void Assembler::vpcmpeqb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
3494   assert(VM_Version::supports_avx(), "");
3495   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3496   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3497   emit_int8(0x74);
3498   emit_int8((unsigned char)(0xC0 | encode));
3499 }
3500 
3501 // In this context, kdst is written the mask used to process the equal components
3502 void Assembler::evpcmpeqb(KRegister kdst, XMMRegister nds, XMMRegister src, int vector_len) {
3503   assert(VM_Version::supports_avx512bw(), "");
3504   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
3505   attributes.set_is_evex_instruction();
3506   int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3507   emit_int8(0x74);
3508   emit_int8((unsigned char)(0xC0 | encode));
3509 }
3510 
3511 void Assembler::evpcmpgtb(KRegister kdst, XMMRegister nds, Address src, int vector_len) {
3512   assert(VM_Version::supports_avx512vlbw(), "");
3513   InstructionMark im(this);
3514   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3515   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
3516   attributes.set_is_evex_instruction();
3517   int dst_enc = kdst->encoding();
3518   vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3519   emit_int8(0x64);
3520   emit_operand(as_Register(dst_enc), src);
3521 }
3522 
3523 void Assembler::evpcmpgtb(KRegister kdst, KRegister mask, XMMRegister nds, Address src, int vector_len) {
3524   assert(is_vector_masking(), "");
3525   assert(VM_Version::supports_avx512vlbw(), "");
3526   InstructionMark im(this);
3527   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
3528   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
3529   attributes.reset_is_clear_context();
3530   attributes.set_embedded_opmask_register_specifier(mask);
3531   attributes.set_is_evex_instruction();
3532   int dst_enc = kdst->encoding();
3533   vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3534   emit_int8(0x64);
3535   emit_operand(as_Register(dst_enc), src);
3536 }
3537 
3538 void Assembler::evpcmpuw(KRegister kdst, XMMRegister nds, XMMRegister src, ComparisonPredicate vcc, int vector_len) {
3539   assert(VM_Version::supports_avx512vlbw(), "");
3540   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3541   attributes.set_is_evex_instruction();
3542   int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3543   emit_int8(0x3E);
3544   emit_int8((unsigned char)(0xC0 | encode));
3545   emit_int8(vcc);
3546 }
3547 
3548 void Assembler::evpcmpuw(KRegister kdst, KRegister mask, XMMRegister nds, XMMRegister src, ComparisonPredicate vcc, int vector_len) {
3549   assert(is_vector_masking(), "");
3550   assert(VM_Version::supports_avx512vlbw(), "");
3551   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
3552   attributes.reset_is_clear_context();
3553   attributes.set_embedded_opmask_register_specifier(mask);
3554   attributes.set_is_evex_instruction();
3555   int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3556   emit_int8(0x3E);
3557   emit_int8((unsigned char)(0xC0 | encode));
3558   emit_int8(vcc);
3559 }
3560 
3561 void Assembler::evpcmpuw(KRegister kdst, XMMRegister nds, Address src, ComparisonPredicate vcc, int vector_len) {
3562   assert(VM_Version::supports_avx512vlbw(), "");
3563   InstructionMark im(this);
3564   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3565   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
3566   attributes.set_is_evex_instruction();
3567   int dst_enc = kdst->encoding();
3568   vex_prefix(src, nds->encoding(), kdst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3569   emit_int8(0x3E);
3570   emit_operand(as_Register(dst_enc), src);
3571   emit_int8(vcc);
3572 }
3573 
3574 void Assembler::evpcmpeqb(KRegister kdst, XMMRegister nds, Address src, int vector_len) {
3575   assert(VM_Version::supports_avx512bw(), "");
3576   InstructionMark im(this);
3577   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3578   attributes.set_is_evex_instruction();
3579   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
3580   int dst_enc = kdst->encoding();
3581   vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3582   emit_int8(0x74);
3583   emit_operand(as_Register(dst_enc), src);
3584 }
3585 
3586 void Assembler::evpcmpeqb(KRegister kdst, KRegister mask, XMMRegister nds, Address src, int vector_len) {
3587   assert(VM_Version::supports_avx512vlbw(), "");
3588   assert(is_vector_masking(), "");    // For stub code use only
3589   InstructionMark im(this);
3590   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_reg_mask */ false, /* uses_vl */ false);
3591   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
3592   attributes.reset_is_clear_context();
3593   attributes.set_embedded_opmask_register_specifier(mask);
3594   attributes.set_is_evex_instruction();
3595   vex_prefix(src, nds->encoding(), kdst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3596   emit_int8(0x74);
3597   emit_operand(as_Register(kdst->encoding()), src);
3598 }
3599 
3600 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
3601 void Assembler::pcmpeqw(XMMRegister dst, XMMRegister src) {
3602   assert(VM_Version::supports_sse2(), "");
3603   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3604   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3605   emit_int8(0x75);
3606   emit_int8((unsigned char)(0xC0 | encode));
3607 }
3608 
3609 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
3610 void Assembler::vpcmpeqw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
3611   assert(VM_Version::supports_avx(), "");
3612   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3613   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3614   emit_int8(0x75);
3615   emit_int8((unsigned char)(0xC0 | encode));
3616 }
3617 
3618 // In this context, kdst is written the mask used to process the equal components
3619 void Assembler::evpcmpeqw(KRegister kdst, XMMRegister nds, XMMRegister src, int vector_len) {
3620   assert(VM_Version::supports_avx512bw(), "");
3621   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
3622   attributes.set_is_evex_instruction();
3623   int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3624   emit_int8(0x75);
3625   emit_int8((unsigned char)(0xC0 | encode));
3626 }
3627 
3628 void Assembler::evpcmpeqw(KRegister kdst, XMMRegister nds, Address src, int vector_len) {
3629   assert(VM_Version::supports_avx512bw(), "");
3630   InstructionMark im(this);
3631   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
3632   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
3633   attributes.set_is_evex_instruction();
3634   int dst_enc = kdst->encoding();
3635   vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3636   emit_int8(0x75);
3637   emit_operand(as_Register(dst_enc), src);
3638 }
3639 
3640 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
3641 void Assembler::pcmpeqd(XMMRegister dst, XMMRegister src) {
3642   assert(VM_Version::supports_sse2(), "");
3643   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3644   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3645   emit_int8(0x76);
3646   emit_int8((unsigned char)(0xC0 | encode));
3647 }
3648 
3649 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
3650 void Assembler::vpcmpeqd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
3651   assert(VM_Version::supports_avx(), "");
3652   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3653   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3654   emit_int8(0x76);
3655   emit_int8((unsigned char)(0xC0 | encode));
3656 }
3657 
3658 // In this context, kdst is written the mask used to process the equal components
3659 void Assembler::evpcmpeqd(KRegister kdst, XMMRegister nds, XMMRegister src, int vector_len) {
3660   assert(VM_Version::supports_evex(), "");
3661   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
3662   attributes.set_is_evex_instruction();
3663   attributes.reset_is_clear_context();
3664   int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3665   emit_int8(0x76);
3666   emit_int8((unsigned char)(0xC0 | encode));
3667 }
3668 
3669 void Assembler::evpcmpeqd(KRegister kdst, XMMRegister nds, Address src, int vector_len) {
3670   assert(VM_Version::supports_evex(), "");
3671   InstructionMark im(this);
3672   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
3673   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
3674   attributes.reset_is_clear_context();
3675   attributes.set_is_evex_instruction();
3676   int dst_enc = kdst->encoding();
3677   vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3678   emit_int8(0x76);
3679   emit_operand(as_Register(dst_enc), src);
3680 }
3681 
3682 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
3683 void Assembler::pcmpeqq(XMMRegister dst, XMMRegister src) {
3684   assert(VM_Version::supports_sse4_1(), "");
3685   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3686   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3687   emit_int8(0x29);
3688   emit_int8((unsigned char)(0xC0 | encode));
3689 }
3690 
3691 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
3692 void Assembler::vpcmpeqq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
3693   assert(VM_Version::supports_avx(), "");
3694   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3695   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3696   emit_int8(0x29);
3697   emit_int8((unsigned char)(0xC0 | encode));
3698 }
3699 
3700 // In this context, kdst is written the mask used to process the equal components
3701 void Assembler::evpcmpeqq(KRegister kdst, XMMRegister nds, XMMRegister src, int vector_len) {
3702   assert(VM_Version::supports_evex(), "");
3703   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
3704   attributes.reset_is_clear_context();
3705   attributes.set_is_evex_instruction();
3706   int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3707   emit_int8(0x29);
3708   emit_int8((unsigned char)(0xC0 | encode));
3709 }
3710 
3711 // In this context, kdst is written the mask used to process the equal components
3712 void Assembler::evpcmpeqq(KRegister kdst, XMMRegister nds, Address src, int vector_len) {
3713   assert(VM_Version::supports_evex(), "");
3714   InstructionMark im(this);
3715   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
3716   attributes.reset_is_clear_context();
3717   attributes.set_is_evex_instruction();
3718   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
3719   int dst_enc = kdst->encoding();
3720   vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3721   emit_int8(0x29);
3722   emit_operand(as_Register(dst_enc), src);
3723 }
3724 
3725 void Assembler::pmovmskb(Register dst, XMMRegister src) {
3726   assert(VM_Version::supports_sse2(), "");
3727   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3728   int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3729   emit_int8((unsigned char)0xD7);
3730   emit_int8((unsigned char)(0xC0 | encode));
3731 }
3732 
3733 void Assembler::vpmovmskb(Register dst, XMMRegister src) {
3734   assert(VM_Version::supports_avx2(), "");
3735   InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3736   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3737   emit_int8((unsigned char)0xD7);
3738   emit_int8((unsigned char)(0xC0 | encode));
3739 }
3740 
3741 void Assembler::pextrd(Register dst, XMMRegister src, int imm8) {
3742   assert(VM_Version::supports_sse4_1(), "");
3743   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
3744   int encode = simd_prefix_and_encode(src, xnoreg, as_XMMRegister(dst->encoding()), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3745   emit_int8(0x16);
3746   emit_int8((unsigned char)(0xC0 | encode));
3747   emit_int8(imm8);
3748 }
3749 
3750 void Assembler::pextrd(Address dst, XMMRegister src, int imm8) {
3751   assert(VM_Version::supports_sse4_1(), "");
3752   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
3753   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
3754   simd_prefix(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3755   emit_int8(0x16);
3756   emit_operand(src, dst);
3757   emit_int8(imm8);
3758 }
3759 
3760 void Assembler::pextrq(Register dst, XMMRegister src, int imm8) {
3761   assert(VM_Version::supports_sse4_1(), "");
3762   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
3763   int encode = simd_prefix_and_encode(src, xnoreg, as_XMMRegister(dst->encoding()), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3764   emit_int8(0x16);
3765   emit_int8((unsigned char)(0xC0 | encode));
3766   emit_int8(imm8);
3767 }
3768 
3769 void Assembler::pextrq(Address dst, XMMRegister src, int imm8) {
3770   assert(VM_Version::supports_sse4_1(), "");
3771   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
3772   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
3773   simd_prefix(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3774   emit_int8(0x16);
3775   emit_operand(src, dst);
3776   emit_int8(imm8);
3777 }
3778 
3779 void Assembler::pextrw(Register dst, XMMRegister src, int imm8) {
3780   assert(VM_Version::supports_sse2(), "");
3781   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
3782   int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3783   emit_int8((unsigned char)0xC5);
3784   emit_int8((unsigned char)(0xC0 | encode));
3785   emit_int8(imm8);
3786 }
3787 
3788 void Assembler::pextrw(Address dst, XMMRegister src, int imm8) {
3789   assert(VM_Version::supports_sse4_1(), "");
3790   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
3791   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_16bit);
3792   simd_prefix(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3793   emit_int8((unsigned char)0x15);
3794   emit_operand(src, dst);
3795   emit_int8(imm8);
3796 }
3797 
3798 void Assembler::pextrb(Address dst, XMMRegister src, int imm8) {
3799   assert(VM_Version::supports_sse4_1(), "");
3800   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
3801   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_8bit);
3802   simd_prefix(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3803   emit_int8(0x14);
3804   emit_operand(src, dst);
3805   emit_int8(imm8);
3806 }
3807 
3808 void Assembler::pinsrd(XMMRegister dst, Register src, int imm8) {
3809   assert(VM_Version::supports_sse4_1(), "");
3810   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
3811   int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3812   emit_int8(0x22);
3813   emit_int8((unsigned char)(0xC0 | encode));
3814   emit_int8(imm8);
3815 }
3816 
3817 void Assembler::pinsrd(XMMRegister dst, Address src, int imm8) {
3818   assert(VM_Version::supports_sse4_1(), "");
3819   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
3820   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
3821   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3822   emit_int8(0x22);
3823   emit_operand(dst,src);
3824   emit_int8(imm8);
3825 }
3826 
3827 void Assembler::pinsrq(XMMRegister dst, Register src, int imm8) {
3828   assert(VM_Version::supports_sse4_1(), "");
3829   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
3830   int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3831   emit_int8(0x22);
3832   emit_int8((unsigned char)(0xC0 | encode));
3833   emit_int8(imm8);
3834 }
3835 
3836 void Assembler::pinsrq(XMMRegister dst, Address src, int imm8) {
3837   assert(VM_Version::supports_sse4_1(), "");
3838   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
3839   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
3840   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3841   emit_int8(0x22);
3842   emit_operand(dst, src);
3843   emit_int8(imm8);
3844 }
3845 
3846 void Assembler::pinsrw(XMMRegister dst, Register src, int imm8) {
3847   assert(VM_Version::supports_sse2(), "");
3848   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
3849   int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3850   emit_int8((unsigned char)0xC4);
3851   emit_int8((unsigned char)(0xC0 | encode));
3852   emit_int8(imm8);
3853 }
3854 
3855 void Assembler::pinsrw(XMMRegister dst, Address src, int imm8) {
3856   assert(VM_Version::supports_sse2(), "");
3857   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
3858   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_16bit);
3859   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3860   emit_int8((unsigned char)0xC4);
3861   emit_operand(dst, src);
3862   emit_int8(imm8);
3863 }
3864 
3865 void Assembler::pinsrb(XMMRegister dst, Address src, int imm8) {
3866   assert(VM_Version::supports_sse4_1(), "");
3867   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
3868   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_8bit);
3869   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3870   emit_int8(0x20);
3871   emit_operand(dst, src);
3872   emit_int8(imm8);
3873 }
3874 
3875 void Assembler::pmovzxbw(XMMRegister dst, Address src) {
3876   assert(VM_Version::supports_sse4_1(), "");
3877   InstructionMark im(this);
3878   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
3879   attributes.set_address_attributes(/* tuple_type */ EVEX_HVM, /* input_size_in_bits */ EVEX_NObit);
3880   simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3881   emit_int8(0x30);
3882   emit_operand(dst, src);
3883 }
3884 
3885 void Assembler::pmovzxbw(XMMRegister dst, XMMRegister src) {
3886   assert(VM_Version::supports_sse4_1(), "");
3887   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
3888   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3889   emit_int8(0x30);
3890   emit_int8((unsigned char)(0xC0 | encode));
3891 }
3892 
3893 void Assembler::vpmovzxbw(XMMRegister dst, Address src, int vector_len) {
3894   assert(VM_Version::supports_avx(), "");
3895   InstructionMark im(this);
3896   assert(dst != xnoreg, "sanity");
3897   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
3898   attributes.set_address_attributes(/* tuple_type */ EVEX_HVM, /* input_size_in_bits */ EVEX_NObit);
3899   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3900   emit_int8(0x30);
3901   emit_operand(dst, src);
3902 }
3903 
3904 void Assembler::vpmovzxbw(XMMRegister dst, XMMRegister src, int vector_len) {
3905   assert(vector_len == AVX_128bit? VM_Version::supports_avx() :
3906   vector_len == AVX_256bit? VM_Version::supports_avx2() :
3907   vector_len == AVX_512bit? VM_Version::supports_avx512bw() : 0, "");
3908   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
3909   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3910   emit_int8(0x30);
3911   emit_int8((unsigned char) (0xC0 | encode));
3912 }
3913 
3914 
3915 void Assembler::evpmovzxbw(XMMRegister dst, KRegister mask, Address src, int vector_len) {
3916   assert(is_vector_masking(), "");
3917   assert(VM_Version::supports_avx512vlbw(), "");
3918   assert(dst != xnoreg, "sanity");
3919   InstructionMark im(this);
3920   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
3921   attributes.set_address_attributes(/* tuple_type */ EVEX_HVM, /* input_size_in_bits */ EVEX_NObit);
3922   attributes.set_embedded_opmask_register_specifier(mask);
3923   attributes.set_is_evex_instruction();
3924   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3925   emit_int8(0x30);
3926   emit_operand(dst, src);
3927 }
3928 void Assembler::evpmovwb(Address dst, XMMRegister src, int vector_len) {
3929   assert(VM_Version::supports_avx512vlbw(), "");
3930   assert(src != xnoreg, "sanity");
3931   InstructionMark im(this);
3932   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3933   attributes.set_address_attributes(/* tuple_type */ EVEX_HVM, /* input_size_in_bits */ EVEX_NObit);
3934   attributes.set_is_evex_instruction();
3935   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
3936   emit_int8(0x30);
3937   emit_operand(src, dst);
3938 }
3939 
3940 void Assembler::evpmovwb(Address dst, KRegister mask, XMMRegister src, int vector_len) {
3941   assert(is_vector_masking(), "");
3942   assert(VM_Version::supports_avx512vlbw(), "");
3943   assert(src != xnoreg, "sanity");
3944   InstructionMark im(this);
3945   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
3946   attributes.set_address_attributes(/* tuple_type */ EVEX_HVM, /* input_size_in_bits */ EVEX_NObit);
3947   attributes.reset_is_clear_context();
3948   attributes.set_embedded_opmask_register_specifier(mask);
3949   attributes.set_is_evex_instruction();
3950   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
3951   emit_int8(0x30);
3952   emit_operand(src, dst);
3953 }
3954 
3955 void Assembler::evpmovdb(Address dst, XMMRegister src, int vector_len) {
3956   assert(VM_Version::supports_evex(), "");
3957   assert(src != xnoreg, "sanity");
3958   InstructionMark im(this);
3959   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3960   attributes.set_address_attributes(/* tuple_type */ EVEX_QVM, /* input_size_in_bits */ EVEX_NObit);
3961   attributes.set_is_evex_instruction();
3962   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
3963   emit_int8(0x31);
3964   emit_operand(src, dst);
3965 }
3966 
3967 void Assembler::vpmovzxwd(XMMRegister dst, XMMRegister src, int vector_len) {
3968   assert(vector_len == AVX_128bit? VM_Version::supports_avx() :
3969   vector_len == AVX_256bit? VM_Version::supports_avx2() :
3970   vector_len == AVX_512bit? VM_Version::supports_evex() : 0, " ");
3971   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
3972   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3973   emit_int8(0x33);
3974   emit_int8((unsigned char)(0xC0 | encode));
3975 }
3976 
3977 // generic
3978 void Assembler::pop(Register dst) {
3979   int encode = prefix_and_encode(dst->encoding());
3980   emit_int8(0x58 | encode);
3981 }
3982 
3983 void Assembler::popcntl(Register dst, Address src) {
3984   assert(VM_Version::supports_popcnt(), "must support");
3985   InstructionMark im(this);
3986   emit_int8((unsigned char)0xF3);
3987   prefix(src, dst);
3988   emit_int8(0x0F);
3989   emit_int8((unsigned char)0xB8);
3990   emit_operand(dst, src);
3991 }
3992 
3993 void Assembler::popcntl(Register dst, Register src) {
3994   assert(VM_Version::supports_popcnt(), "must support");
3995   emit_int8((unsigned char)0xF3);
3996   int encode = prefix_and_encode(dst->encoding(), src->encoding());
3997   emit_int8(0x0F);
3998   emit_int8((unsigned char)0xB8);
3999   emit_int8((unsigned char)(0xC0 | encode));
4000 }
4001 
4002 void Assembler::vpopcntd(XMMRegister dst, XMMRegister src, int vector_len) {
4003   assert(VM_Version::supports_vpopcntdq(), "must support vpopcntdq feature");
4004   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4005   attributes.set_is_evex_instruction();
4006   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4007   emit_int8(0x55);
4008   emit_int8((unsigned char)(0xC0 | encode));
4009 }
4010 
4011 void Assembler::popf() {
4012   emit_int8((unsigned char)0x9D);
4013 }
4014 
4015 #ifndef _LP64 // no 32bit push/pop on amd64
4016 void Assembler::popl(Address dst) {
4017   // NOTE: this will adjust stack by 8byte on 64bits
4018   InstructionMark im(this);
4019   prefix(dst);
4020   emit_int8((unsigned char)0x8F);
4021   emit_operand(rax, dst);
4022 }
4023 #endif
4024 
4025 void Assembler::prefetch_prefix(Address src) {
4026   prefix(src);
4027   emit_int8(0x0F);
4028 }
4029 
4030 void Assembler::prefetchnta(Address src) {
4031   NOT_LP64(assert(VM_Version::supports_sse(), "must support"));
4032   InstructionMark im(this);
4033   prefetch_prefix(src);
4034   emit_int8(0x18);
4035   emit_operand(rax, src); // 0, src
4036 }
4037 
4038 void Assembler::prefetchr(Address src) {
4039   assert(VM_Version::supports_3dnow_prefetch(), "must support");
4040   InstructionMark im(this);
4041   prefetch_prefix(src);
4042   emit_int8(0x0D);
4043   emit_operand(rax, src); // 0, src
4044 }
4045 
4046 void Assembler::prefetcht0(Address src) {
4047   NOT_LP64(assert(VM_Version::supports_sse(), "must support"));
4048   InstructionMark im(this);
4049   prefetch_prefix(src);
4050   emit_int8(0x18);
4051   emit_operand(rcx, src); // 1, src
4052 }
4053 
4054 void Assembler::prefetcht1(Address src) {
4055   NOT_LP64(assert(VM_Version::supports_sse(), "must support"));
4056   InstructionMark im(this);
4057   prefetch_prefix(src);
4058   emit_int8(0x18);
4059   emit_operand(rdx, src); // 2, src
4060 }
4061 
4062 void Assembler::prefetcht2(Address src) {
4063   NOT_LP64(assert(VM_Version::supports_sse(), "must support"));
4064   InstructionMark im(this);
4065   prefetch_prefix(src);
4066   emit_int8(0x18);
4067   emit_operand(rbx, src); // 3, src
4068 }
4069 
4070 void Assembler::prefetchw(Address src) {
4071   assert(VM_Version::supports_3dnow_prefetch(), "must support");
4072   InstructionMark im(this);
4073   prefetch_prefix(src);
4074   emit_int8(0x0D);
4075   emit_operand(rcx, src); // 1, src
4076 }
4077 
4078 void Assembler::prefix(Prefix p) {
4079   emit_int8(p);
4080 }
4081 
4082 void Assembler::pshufb(XMMRegister dst, XMMRegister src) {
4083   assert(VM_Version::supports_ssse3(), "");
4084   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
4085   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4086   emit_int8(0x00);
4087   emit_int8((unsigned char)(0xC0 | encode));
4088 }
4089 
4090 void Assembler::vpshufb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4091   assert(vector_len == AVX_128bit? VM_Version::supports_avx() :
4092          vector_len == AVX_256bit? VM_Version::supports_avx2() :
4093          0, "");
4094   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
4095   int encode = simd_prefix_and_encode(dst, nds, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4096   emit_int8(0x00);
4097   emit_int8((unsigned char)(0xC0 | encode));
4098 }
4099 
4100 void Assembler::pshufb(XMMRegister dst, Address src) {
4101   assert(VM_Version::supports_ssse3(), "");
4102   InstructionMark im(this);
4103   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
4104   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
4105   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4106   emit_int8(0x00);
4107   emit_operand(dst, src);
4108 }
4109 
4110 void Assembler::pshufd(XMMRegister dst, XMMRegister src, int mode) {
4111   assert(isByte(mode), "invalid value");
4112   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4113   int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_128bit;
4114   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4115   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4116   emit_int8(0x70);
4117   emit_int8((unsigned char)(0xC0 | encode));
4118   emit_int8(mode & 0xFF);
4119 }
4120 
4121 void Assembler::vpshufd(XMMRegister dst, XMMRegister src, int mode, int vector_len) {
4122   assert(vector_len == AVX_128bit? VM_Version::supports_avx() :
4123          vector_len == AVX_256bit? VM_Version::supports_avx2() :
4124          0, "");
4125   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4126   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4127   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4128   emit_int8(0x70);
4129   emit_int8((unsigned char)(0xC0 | encode));
4130   emit_int8(mode & 0xFF);
4131 }
4132 
4133 void Assembler::pshufd(XMMRegister dst, Address src, int mode) {
4134   assert(isByte(mode), "invalid value");
4135   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4136   assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
4137   InstructionMark im(this);
4138   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4139   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
4140   simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4141   emit_int8(0x70);
4142   emit_operand(dst, src);
4143   emit_int8(mode & 0xFF);
4144 }
4145 
4146 void Assembler::pshuflw(XMMRegister dst, XMMRegister src, int mode) {
4147   assert(isByte(mode), "invalid value");
4148   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4149   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
4150   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
4151   emit_int8(0x70);
4152   emit_int8((unsigned char)(0xC0 | encode));
4153   emit_int8(mode & 0xFF);
4154 }
4155 
4156 void Assembler::pshuflw(XMMRegister dst, Address src, int mode) {
4157   assert(isByte(mode), "invalid value");
4158   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4159   assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
4160   InstructionMark im(this);
4161   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
4162   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
4163   simd_prefix(dst, xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
4164   emit_int8(0x70);
4165   emit_operand(dst, src);
4166   emit_int8(mode & 0xFF);
4167 }
4168 void Assembler::evshufi64x2(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8, int vector_len) {
4169   assert(VM_Version::supports_evex(), "requires EVEX support");
4170   assert(vector_len == Assembler::AVX_256bit || vector_len == Assembler::AVX_512bit, "");
4171   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
4172   attributes.set_is_evex_instruction();
4173   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
4174   emit_int8(0x43);
4175   emit_int8((unsigned char)(0xC0 | encode));
4176   emit_int8(imm8 & 0xFF);
4177 }
4178 
4179 void Assembler::psrldq(XMMRegister dst, int shift) {
4180   // Shift left 128 bit value in dst XMMRegister by shift number of bytes.
4181   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4182   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
4183   int encode = simd_prefix_and_encode(xmm3, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4184   emit_int8(0x73);
4185   emit_int8((unsigned char)(0xC0 | encode));
4186   emit_int8(shift);
4187 }
4188 
4189 void Assembler::pslldq(XMMRegister dst, int shift) {
4190   // Shift left 128 bit value in dst XMMRegister by shift number of bytes.
4191   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4192   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
4193   // XMM7 is for /7 encoding: 66 0F 73 /7 ib
4194   int encode = simd_prefix_and_encode(xmm7, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4195   emit_int8(0x73);
4196   emit_int8((unsigned char)(0xC0 | encode));
4197   emit_int8(shift);
4198 }
4199 
4200 void Assembler::ptest(XMMRegister dst, Address src) {
4201   assert(VM_Version::supports_sse4_1(), "");
4202   assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
4203   InstructionMark im(this);
4204   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
4205   simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4206   emit_int8(0x17);
4207   emit_operand(dst, src);
4208 }
4209 
4210 void Assembler::ptest(XMMRegister dst, XMMRegister src) {
4211   assert(VM_Version::supports_sse4_1(), "");
4212   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
4213   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4214   emit_int8(0x17);
4215   emit_int8((unsigned char)(0xC0 | encode));
4216 }
4217 
4218 void Assembler::vptest(XMMRegister dst, Address src) {
4219   assert(VM_Version::supports_avx(), "");
4220   InstructionMark im(this);
4221   InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
4222   assert(dst != xnoreg, "sanity");
4223   // swap src<->dst for encoding
4224   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4225   emit_int8(0x17);
4226   emit_operand(dst, src);
4227 }
4228 
4229 void Assembler::vptest(XMMRegister dst, XMMRegister src) {
4230   assert(VM_Version::supports_avx(), "");
4231   InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
4232   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4233   emit_int8(0x17);
4234   emit_int8((unsigned char)(0xC0 | encode));
4235 }
4236 
4237 void Assembler::punpcklbw(XMMRegister dst, Address src) {
4238   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4239   assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
4240   InstructionMark im(this);
4241   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_vlbw, /* no_mask_reg */ false, /* uses_vl */ true);
4242   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
4243   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4244   emit_int8(0x60);
4245   emit_operand(dst, src);
4246 }
4247 
4248 void Assembler::punpcklbw(XMMRegister dst, XMMRegister src) {
4249   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4250   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_vlbw, /* no_mask_reg */ false, /* uses_vl */ true);
4251   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4252   emit_int8(0x60);
4253   emit_int8((unsigned char)(0xC0 | encode));
4254 }
4255 
4256 void Assembler::punpckldq(XMMRegister dst, Address src) {
4257   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4258   assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
4259   InstructionMark im(this);
4260   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4261   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
4262   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4263   emit_int8(0x62);
4264   emit_operand(dst, src);
4265 }
4266 
4267 void Assembler::punpckldq(XMMRegister dst, XMMRegister src) {
4268   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4269   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4270   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4271   emit_int8(0x62);
4272   emit_int8((unsigned char)(0xC0 | encode));
4273 }
4274 
4275 void Assembler::punpcklqdq(XMMRegister dst, XMMRegister src) {
4276   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4277   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4278   attributes.set_rex_vex_w_reverted();
4279   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4280   emit_int8(0x6C);
4281   emit_int8((unsigned char)(0xC0 | encode));
4282 }
4283 
4284 void Assembler::push(int32_t imm32) {
4285   // in 64bits we push 64bits onto the stack but only
4286   // take a 32bit immediate
4287   emit_int8(0x68);
4288   emit_int32(imm32);
4289 }
4290 
4291 void Assembler::push(Register src) {
4292   int encode = prefix_and_encode(src->encoding());
4293 
4294   emit_int8(0x50 | encode);
4295 }
4296 
4297 void Assembler::pushf() {
4298   emit_int8((unsigned char)0x9C);
4299 }
4300 
4301 #ifndef _LP64 // no 32bit push/pop on amd64
4302 void Assembler::pushl(Address src) {
4303   // Note this will push 64bit on 64bit
4304   InstructionMark im(this);
4305   prefix(src);
4306   emit_int8((unsigned char)0xFF);
4307   emit_operand(rsi, src);
4308 }
4309 #endif
4310 
4311 void Assembler::rcll(Register dst, int imm8) {
4312   assert(isShiftCount(imm8), "illegal shift count");
4313   int encode = prefix_and_encode(dst->encoding());
4314   if (imm8 == 1) {
4315     emit_int8((unsigned char)0xD1);
4316     emit_int8((unsigned char)(0xD0 | encode));
4317   } else {
4318     emit_int8((unsigned char)0xC1);
4319     emit_int8((unsigned char)0xD0 | encode);
4320     emit_int8(imm8);
4321   }
4322 }
4323 
4324 void Assembler::rcpps(XMMRegister dst, XMMRegister src) {
4325   NOT_LP64(assert(VM_Version::supports_sse(), ""));
4326   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
4327   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
4328   emit_int8(0x53);
4329   emit_int8((unsigned char)(0xC0 | encode));
4330 }
4331 
4332 void Assembler::rcpss(XMMRegister dst, XMMRegister src) {
4333   NOT_LP64(assert(VM_Version::supports_sse(), ""));
4334   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
4335   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
4336   emit_int8(0x53);
4337   emit_int8((unsigned char)(0xC0 | encode));
4338 }
4339 
4340 void Assembler::rdtsc() {
4341   emit_int8((unsigned char)0x0F);
4342   emit_int8((unsigned char)0x31);
4343 }
4344 
4345 // copies data from [esi] to [edi] using rcx pointer sized words
4346 // generic
4347 void Assembler::rep_mov() {
4348   emit_int8((unsigned char)0xF3);
4349   // MOVSQ
4350   LP64_ONLY(prefix(REX_W));
4351   emit_int8((unsigned char)0xA5);
4352 }
4353 
4354 // sets rcx bytes with rax, value at [edi]
4355 void Assembler::rep_stosb() {
4356   emit_int8((unsigned char)0xF3); // REP
4357   LP64_ONLY(prefix(REX_W));
4358   emit_int8((unsigned char)0xAA); // STOSB
4359 }
4360 
4361 // sets rcx pointer sized words with rax, value at [edi]
4362 // generic
4363 void Assembler::rep_stos() {
4364   emit_int8((unsigned char)0xF3); // REP
4365   LP64_ONLY(prefix(REX_W));       // LP64:STOSQ, LP32:STOSD
4366   emit_int8((unsigned char)0xAB);
4367 }
4368 
4369 // scans rcx pointer sized words at [edi] for occurance of rax,
4370 // generic
4371 void Assembler::repne_scan() { // repne_scan
4372   emit_int8((unsigned char)0xF2);
4373   // SCASQ
4374   LP64_ONLY(prefix(REX_W));
4375   emit_int8((unsigned char)0xAF);
4376 }
4377 
4378 #ifdef _LP64
4379 // scans rcx 4 byte words at [edi] for occurance of rax,
4380 // generic
4381 void Assembler::repne_scanl() { // repne_scan
4382   emit_int8((unsigned char)0xF2);
4383   // SCASL
4384   emit_int8((unsigned char)0xAF);
4385 }
4386 #endif
4387 
4388 void Assembler::ret(int imm16) {
4389   if (imm16 == 0) {
4390     emit_int8((unsigned char)0xC3);
4391   } else {
4392     emit_int8((unsigned char)0xC2);
4393     emit_int16(imm16);
4394   }
4395 }
4396 
4397 void Assembler::sahf() {
4398 #ifdef _LP64
4399   // Not supported in 64bit mode
4400   ShouldNotReachHere();
4401 #endif
4402   emit_int8((unsigned char)0x9E);
4403 }
4404 
4405 void Assembler::sarl(Register dst, int imm8) {
4406   int encode = prefix_and_encode(dst->encoding());
4407   assert(isShiftCount(imm8), "illegal shift count");
4408   if (imm8 == 1) {
4409     emit_int8((unsigned char)0xD1);
4410     emit_int8((unsigned char)(0xF8 | encode));
4411   } else {
4412     emit_int8((unsigned char)0xC1);
4413     emit_int8((unsigned char)(0xF8 | encode));
4414     emit_int8(imm8);
4415   }
4416 }
4417 
4418 void Assembler::sarl(Register dst) {
4419   int encode = prefix_and_encode(dst->encoding());
4420   emit_int8((unsigned char)0xD3);
4421   emit_int8((unsigned char)(0xF8 | encode));
4422 }
4423 
4424 void Assembler::sbbl(Address dst, int32_t imm32) {
4425   InstructionMark im(this);
4426   prefix(dst);
4427   emit_arith_operand(0x81, rbx, dst, imm32);
4428 }
4429 
4430 void Assembler::sbbl(Register dst, int32_t imm32) {
4431   prefix(dst);
4432   emit_arith(0x81, 0xD8, dst, imm32);
4433 }
4434 
4435 
4436 void Assembler::sbbl(Register dst, Address src) {
4437   InstructionMark im(this);
4438   prefix(src, dst);
4439   emit_int8(0x1B);
4440   emit_operand(dst, src);
4441 }
4442 
4443 void Assembler::sbbl(Register dst, Register src) {
4444   (void) prefix_and_encode(dst->encoding(), src->encoding());
4445   emit_arith(0x1B, 0xC0, dst, src);
4446 }
4447 
4448 void Assembler::setb(Condition cc, Register dst) {
4449   assert(0 <= cc && cc < 16, "illegal cc");
4450   int encode = prefix_and_encode(dst->encoding(), true);
4451   emit_int8(0x0F);
4452   emit_int8((unsigned char)0x90 | cc);
4453   emit_int8((unsigned char)(0xC0 | encode));
4454 }
4455 
4456 void Assembler::palignr(XMMRegister dst, XMMRegister src, int imm8) {
4457   assert(VM_Version::supports_ssse3(), "");
4458   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ false);
4459   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
4460   emit_int8((unsigned char)0x0F);
4461   emit_int8((unsigned char)(0xC0 | encode));
4462   emit_int8(imm8);
4463 }
4464 
4465 void Assembler::vpalignr(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8, int vector_len) {
4466   assert(vector_len == AVX_128bit? VM_Version::supports_avx() :
4467          vector_len == AVX_256bit? VM_Version::supports_avx2() :
4468          0, "");
4469   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
4470   int encode = simd_prefix_and_encode(dst, nds, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
4471   emit_int8((unsigned char)0x0F);
4472   emit_int8((unsigned char)(0xC0 | encode));
4473   emit_int8(imm8);
4474 }
4475 
4476 void Assembler::evalignq(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) {
4477   assert(VM_Version::supports_evex(), "");
4478   InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4479   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
4480   emit_int8(0x3);
4481   emit_int8((unsigned char)(0xC0 | encode));
4482   emit_int8(imm8);
4483 }
4484 
4485 void Assembler::pblendw(XMMRegister dst, XMMRegister src, int imm8) {
4486   assert(VM_Version::supports_sse4_1(), "");
4487   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
4488   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
4489   emit_int8((unsigned char)0x0E);
4490   emit_int8((unsigned char)(0xC0 | encode));
4491   emit_int8(imm8);
4492 }
4493 
4494 void Assembler::sha1rnds4(XMMRegister dst, XMMRegister src, int imm8) {
4495   assert(VM_Version::supports_sha(), "");
4496   int encode = rex_prefix_and_encode(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3A, /* rex_w */ false);
4497   emit_int8((unsigned char)0xCC);
4498   emit_int8((unsigned char)(0xC0 | encode));
4499   emit_int8((unsigned char)imm8);
4500 }
4501 
4502 void Assembler::sha1nexte(XMMRegister dst, XMMRegister src) {
4503   assert(VM_Version::supports_sha(), "");
4504   int encode = rex_prefix_and_encode(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, /* rex_w */ false);
4505   emit_int8((unsigned char)0xC8);
4506   emit_int8((unsigned char)(0xC0 | encode));
4507 }
4508 
4509 void Assembler::sha1msg1(XMMRegister dst, XMMRegister src) {
4510   assert(VM_Version::supports_sha(), "");
4511   int encode = rex_prefix_and_encode(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, /* rex_w */ false);
4512   emit_int8((unsigned char)0xC9);
4513   emit_int8((unsigned char)(0xC0 | encode));
4514 }
4515 
4516 void Assembler::sha1msg2(XMMRegister dst, XMMRegister src) {
4517   assert(VM_Version::supports_sha(), "");
4518   int encode = rex_prefix_and_encode(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, /* rex_w */ false);
4519   emit_int8((unsigned char)0xCA);
4520   emit_int8((unsigned char)(0xC0 | encode));
4521 }
4522 
4523 // xmm0 is implicit additional source to this instruction.
4524 void Assembler::sha256rnds2(XMMRegister dst, XMMRegister src) {
4525   assert(VM_Version::supports_sha(), "");
4526   int encode = rex_prefix_and_encode(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, /* rex_w */ false);
4527   emit_int8((unsigned char)0xCB);
4528   emit_int8((unsigned char)(0xC0 | encode));
4529 }
4530 
4531 void Assembler::sha256msg1(XMMRegister dst, XMMRegister src) {
4532   assert(VM_Version::supports_sha(), "");
4533   int encode = rex_prefix_and_encode(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, /* rex_w */ false);
4534   emit_int8((unsigned char)0xCC);
4535   emit_int8((unsigned char)(0xC0 | encode));
4536 }
4537 
4538 void Assembler::sha256msg2(XMMRegister dst, XMMRegister src) {
4539   assert(VM_Version::supports_sha(), "");
4540   int encode = rex_prefix_and_encode(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, /* rex_w */ false);
4541   emit_int8((unsigned char)0xCD);
4542   emit_int8((unsigned char)(0xC0 | encode));
4543 }
4544 
4545 
4546 void Assembler::shll(Register dst, int imm8) {
4547   assert(isShiftCount(imm8), "illegal shift count");
4548   int encode = prefix_and_encode(dst->encoding());
4549   if (imm8 == 1 ) {
4550     emit_int8((unsigned char)0xD1);
4551     emit_int8((unsigned char)(0xE0 | encode));
4552   } else {
4553     emit_int8((unsigned char)0xC1);
4554     emit_int8((unsigned char)(0xE0 | encode));
4555     emit_int8(imm8);
4556   }
4557 }
4558 
4559 void Assembler::shll(Register dst) {
4560   int encode = prefix_and_encode(dst->encoding());
4561   emit_int8((unsigned char)0xD3);
4562   emit_int8((unsigned char)(0xE0 | encode));
4563 }
4564 
4565 void Assembler::shrl(Register dst, int imm8) {
4566   assert(isShiftCount(imm8), "illegal shift count");
4567   int encode = prefix_and_encode(dst->encoding());
4568   emit_int8((unsigned char)0xC1);
4569   emit_int8((unsigned char)(0xE8 | encode));
4570   emit_int8(imm8);
4571 }
4572 
4573 void Assembler::shrl(Register dst) {
4574   int encode = prefix_and_encode(dst->encoding());
4575   emit_int8((unsigned char)0xD3);
4576   emit_int8((unsigned char)(0xE8 | encode));
4577 }
4578 
4579 // copies a single word from [esi] to [edi]
4580 void Assembler::smovl() {
4581   emit_int8((unsigned char)0xA5);
4582 }
4583 
4584 void Assembler::sqrtsd(XMMRegister dst, XMMRegister src) {
4585   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4586   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4587   attributes.set_rex_vex_w_reverted();
4588   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
4589   emit_int8(0x51);
4590   emit_int8((unsigned char)(0xC0 | encode));
4591 }
4592 
4593 void Assembler::sqrtsd(XMMRegister dst, Address src) {
4594   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4595   InstructionMark im(this);
4596   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4597   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
4598   attributes.set_rex_vex_w_reverted();
4599   simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
4600   emit_int8(0x51);
4601   emit_operand(dst, src);
4602 }
4603 
4604 void Assembler::sqrtss(XMMRegister dst, XMMRegister src) {
4605   NOT_LP64(assert(VM_Version::supports_sse(), ""));
4606   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4607   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
4608   emit_int8(0x51);
4609   emit_int8((unsigned char)(0xC0 | encode));
4610 }
4611 
4612 void Assembler::std() {
4613   emit_int8((unsigned char)0xFD);
4614 }
4615 
4616 void Assembler::sqrtss(XMMRegister dst, Address src) {
4617   NOT_LP64(assert(VM_Version::supports_sse(), ""));
4618   InstructionMark im(this);
4619   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4620   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
4621   simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
4622   emit_int8(0x51);
4623   emit_operand(dst, src);
4624 }
4625 
4626 void Assembler::stmxcsr( Address dst) {
4627   if (UseAVX > 0 ) {
4628     assert(VM_Version::supports_avx(), "");
4629     InstructionMark im(this);
4630     InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
4631     vex_prefix(dst, 0, 0, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
4632     emit_int8((unsigned char)0xAE);
4633     emit_operand(as_Register(3), dst);
4634   } else {
4635     NOT_LP64(assert(VM_Version::supports_sse(), ""));
4636     InstructionMark im(this);
4637     prefix(dst);
4638     emit_int8(0x0F);
4639     emit_int8((unsigned char)0xAE);
4640     emit_operand(as_Register(3), dst);
4641   }
4642 }
4643 
4644 void Assembler::subl(Address dst, int32_t imm32) {
4645   InstructionMark im(this);
4646   prefix(dst);
4647   emit_arith_operand(0x81, rbp, dst, imm32);
4648 }
4649 
4650 void Assembler::subl(Address dst, Register src) {
4651   InstructionMark im(this);
4652   prefix(dst, src);
4653   emit_int8(0x29);
4654   emit_operand(src, dst);
4655 }
4656 
4657 void Assembler::subl(Register dst, int32_t imm32) {
4658   prefix(dst);
4659   emit_arith(0x81, 0xE8, dst, imm32);
4660 }
4661 
4662 // Force generation of a 4 byte immediate value even if it fits into 8bit
4663 void Assembler::subl_imm32(Register dst, int32_t imm32) {
4664   prefix(dst);
4665   emit_arith_imm32(0x81, 0xE8, dst, imm32);
4666 }
4667 
4668 void Assembler::subl(Register dst, Address src) {
4669   InstructionMark im(this);
4670   prefix(src, dst);
4671   emit_int8(0x2B);
4672   emit_operand(dst, src);
4673 }
4674 
4675 void Assembler::subl(Register dst, Register src) {
4676   (void) prefix_and_encode(dst->encoding(), src->encoding());
4677   emit_arith(0x2B, 0xC0, dst, src);
4678 }
4679 
4680 void Assembler::subsd(XMMRegister dst, XMMRegister src) {
4681   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4682   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4683   attributes.set_rex_vex_w_reverted();
4684   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
4685   emit_int8(0x5C);
4686   emit_int8((unsigned char)(0xC0 | encode));
4687 }
4688 
4689 void Assembler::subsd(XMMRegister dst, Address src) {
4690   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4691   InstructionMark im(this);
4692   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4693   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
4694   attributes.set_rex_vex_w_reverted();
4695   simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
4696   emit_int8(0x5C);
4697   emit_operand(dst, src);
4698 }
4699 
4700 void Assembler::subss(XMMRegister dst, XMMRegister src) {
4701   NOT_LP64(assert(VM_Version::supports_sse(), ""));
4702   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true , /* uses_vl */ false);
4703   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
4704   emit_int8(0x5C);
4705   emit_int8((unsigned char)(0xC0 | encode));
4706 }
4707 
4708 void Assembler::subss(XMMRegister dst, Address src) {
4709   NOT_LP64(assert(VM_Version::supports_sse(), ""));
4710   InstructionMark im(this);
4711   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4712   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
4713   simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
4714   emit_int8(0x5C);
4715   emit_operand(dst, src);
4716 }
4717 
4718 void Assembler::testb(Register dst, int imm8) {
4719   NOT_LP64(assert(dst->has_byte_register(), "must have byte register"));
4720   (void) prefix_and_encode(dst->encoding(), true);
4721   emit_arith_b(0xF6, 0xC0, dst, imm8);
4722 }
4723 
4724 void Assembler::testb(Address dst, int imm8) {
4725   InstructionMark im(this);
4726   prefix(dst);
4727   emit_int8((unsigned char)0xF6);
4728   emit_operand(rax, dst, 1);
4729   emit_int8(imm8);
4730 }
4731 
4732 void Assembler::testl(Register dst, int32_t imm32) {
4733   // not using emit_arith because test
4734   // doesn't support sign-extension of
4735   // 8bit operands
4736   int encode = dst->encoding();
4737   if (encode == 0) {
4738     emit_int8((unsigned char)0xA9);
4739   } else {
4740     encode = prefix_and_encode(encode);
4741     emit_int8((unsigned char)0xF7);
4742     emit_int8((unsigned char)(0xC0 | encode));
4743   }
4744   emit_int32(imm32);
4745 }
4746 
4747 void Assembler::testl(Register dst, Register src) {
4748   (void) prefix_and_encode(dst->encoding(), src->encoding());
4749   emit_arith(0x85, 0xC0, dst, src);
4750 }
4751 
4752 void Assembler::testl(Register dst, Address src) {
4753   InstructionMark im(this);
4754   prefix(src, dst);
4755   emit_int8((unsigned char)0x85);
4756   emit_operand(dst, src);
4757 }
4758 
4759 void Assembler::tzcntl(Register dst, Register src) {
4760   assert(VM_Version::supports_bmi1(), "tzcnt instruction not supported");
4761   emit_int8((unsigned char)0xF3);
4762   int encode = prefix_and_encode(dst->encoding(), src->encoding());
4763   emit_int8(0x0F);
4764   emit_int8((unsigned char)0xBC);
4765   emit_int8((unsigned char)0xC0 | encode);
4766 }
4767 
4768 void Assembler::tzcntq(Register dst, Register src) {
4769   assert(VM_Version::supports_bmi1(), "tzcnt instruction not supported");
4770   emit_int8((unsigned char)0xF3);
4771   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
4772   emit_int8(0x0F);
4773   emit_int8((unsigned char)0xBC);
4774   emit_int8((unsigned char)(0xC0 | encode));
4775 }
4776 
4777 void Assembler::ucomisd(XMMRegister dst, Address src) {
4778   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4779   InstructionMark im(this);
4780   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4781   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
4782   attributes.set_rex_vex_w_reverted();
4783   simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4784   emit_int8(0x2E);
4785   emit_operand(dst, src);
4786 }
4787 
4788 void Assembler::ucomisd(XMMRegister dst, XMMRegister src) {
4789   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4790   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4791   attributes.set_rex_vex_w_reverted();
4792   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4793   emit_int8(0x2E);
4794   emit_int8((unsigned char)(0xC0 | encode));
4795 }
4796 
4797 void Assembler::ucomiss(XMMRegister dst, Address src) {
4798   NOT_LP64(assert(VM_Version::supports_sse(), ""));
4799   InstructionMark im(this);
4800   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4801   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
4802   simd_prefix(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
4803   emit_int8(0x2E);
4804   emit_operand(dst, src);
4805 }
4806 
4807 void Assembler::ucomiss(XMMRegister dst, XMMRegister src) {
4808   NOT_LP64(assert(VM_Version::supports_sse(), ""));
4809   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4810   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
4811   emit_int8(0x2E);
4812   emit_int8((unsigned char)(0xC0 | encode));
4813 }
4814 
4815 void Assembler::xabort(int8_t imm8) {
4816   emit_int8((unsigned char)0xC6);
4817   emit_int8((unsigned char)0xF8);
4818   emit_int8((unsigned char)(imm8 & 0xFF));
4819 }
4820 
4821 void Assembler::xaddb(Address dst, Register src) {
4822   InstructionMark im(this);
4823   prefix(dst, src, true);
4824   emit_int8(0x0F);
4825   emit_int8((unsigned char)0xC0);
4826   emit_operand(src, dst);
4827 }
4828 
4829 void Assembler::xaddw(Address dst, Register src) {
4830   InstructionMark im(this);
4831   emit_int8(0x66);
4832   prefix(dst, src);
4833   emit_int8(0x0F);
4834   emit_int8((unsigned char)0xC1);
4835   emit_operand(src, dst);
4836 }
4837 
4838 void Assembler::xaddl(Address dst, Register src) {
4839   InstructionMark im(this);
4840   prefix(dst, src);
4841   emit_int8(0x0F);
4842   emit_int8((unsigned char)0xC1);
4843   emit_operand(src, dst);
4844 }
4845 
4846 void Assembler::xbegin(Label& abort, relocInfo::relocType rtype) {
4847   InstructionMark im(this);
4848   relocate(rtype);
4849   if (abort.is_bound()) {
4850     address entry = target(abort);
4851     assert(entry != NULL, "abort entry NULL");
4852     intptr_t offset = entry - pc();
4853     emit_int8((unsigned char)0xC7);
4854     emit_int8((unsigned char)0xF8);
4855     emit_int32(offset - 6); // 2 opcode + 4 address
4856   } else {
4857     abort.add_patch_at(code(), locator());
4858     emit_int8((unsigned char)0xC7);
4859     emit_int8((unsigned char)0xF8);
4860     emit_int32(0);
4861   }
4862 }
4863 
4864 void Assembler::xchgb(Register dst, Address src) { // xchg
4865   InstructionMark im(this);
4866   prefix(src, dst, true);
4867   emit_int8((unsigned char)0x86);
4868   emit_operand(dst, src);
4869 }
4870 
4871 void Assembler::xchgw(Register dst, Address src) { // xchg
4872   InstructionMark im(this);
4873   emit_int8(0x66);
4874   prefix(src, dst);
4875   emit_int8((unsigned char)0x87);
4876   emit_operand(dst, src);
4877 }
4878 
4879 void Assembler::xchgl(Register dst, Address src) { // xchg
4880   InstructionMark im(this);
4881   prefix(src, dst);
4882   emit_int8((unsigned char)0x87);
4883   emit_operand(dst, src);
4884 }
4885 
4886 void Assembler::xchgl(Register dst, Register src) {
4887   int encode = prefix_and_encode(dst->encoding(), src->encoding());
4888   emit_int8((unsigned char)0x87);
4889   emit_int8((unsigned char)(0xC0 | encode));
4890 }
4891 
4892 void Assembler::xend() {
4893   emit_int8((unsigned char)0x0F);
4894   emit_int8((unsigned char)0x01);
4895   emit_int8((unsigned char)0xD5);
4896 }
4897 
4898 void Assembler::xgetbv() {
4899   emit_int8(0x0F);
4900   emit_int8(0x01);
4901   emit_int8((unsigned char)0xD0);
4902 }
4903 
4904 void Assembler::xorl(Register dst, int32_t imm32) {
4905   prefix(dst);
4906   emit_arith(0x81, 0xF0, dst, imm32);
4907 }
4908 
4909 void Assembler::xorl(Register dst, Address src) {
4910   InstructionMark im(this);
4911   prefix(src, dst);
4912   emit_int8(0x33);
4913   emit_operand(dst, src);
4914 }
4915 
4916 void Assembler::xorl(Register dst, Register src) {
4917   (void) prefix_and_encode(dst->encoding(), src->encoding());
4918   emit_arith(0x33, 0xC0, dst, src);
4919 }
4920 
4921 void Assembler::xorb(Register dst, Address src) {
4922   InstructionMark im(this);
4923   prefix(src, dst);
4924   emit_int8(0x32);
4925   emit_operand(dst, src);
4926 }
4927 
4928 // AVX 3-operands scalar float-point arithmetic instructions
4929 
4930 void Assembler::vaddsd(XMMRegister dst, XMMRegister nds, Address src) {
4931   assert(VM_Version::supports_avx(), "");
4932   InstructionMark im(this);
4933   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4934   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
4935   attributes.set_rex_vex_w_reverted();
4936   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
4937   emit_int8(0x58);
4938   emit_operand(dst, src);
4939 }
4940 
4941 void Assembler::vaddsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
4942   assert(VM_Version::supports_avx(), "");
4943   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4944   attributes.set_rex_vex_w_reverted();
4945   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
4946   emit_int8(0x58);
4947   emit_int8((unsigned char)(0xC0 | encode));
4948 }
4949 
4950 void Assembler::vaddss(XMMRegister dst, XMMRegister nds, Address src) {
4951   assert(VM_Version::supports_avx(), "");
4952   InstructionMark im(this);
4953   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4954   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
4955   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
4956   emit_int8(0x58);
4957   emit_operand(dst, src);
4958 }
4959 
4960 void Assembler::vaddss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
4961   assert(VM_Version::supports_avx(), "");
4962   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4963   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
4964   emit_int8(0x58);
4965   emit_int8((unsigned char)(0xC0 | encode));
4966 }
4967 
4968 void Assembler::vdivsd(XMMRegister dst, XMMRegister nds, Address src) {
4969   assert(VM_Version::supports_avx(), "");
4970   InstructionMark im(this);
4971   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4972   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
4973   attributes.set_rex_vex_w_reverted();
4974   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
4975   emit_int8(0x5E);
4976   emit_operand(dst, src);
4977 }
4978 
4979 void Assembler::vdivsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
4980   assert(VM_Version::supports_avx(), "");
4981   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4982   attributes.set_rex_vex_w_reverted();
4983   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
4984   emit_int8(0x5E);
4985   emit_int8((unsigned char)(0xC0 | encode));
4986 }
4987 
4988 void Assembler::vdivss(XMMRegister dst, XMMRegister nds, Address src) {
4989   assert(VM_Version::supports_avx(), "");
4990   InstructionMark im(this);
4991   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4992   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
4993   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
4994   emit_int8(0x5E);
4995   emit_operand(dst, src);
4996 }
4997 
4998 void Assembler::vdivss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
4999   assert(VM_Version::supports_avx(), "");
5000   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
5001   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
5002   emit_int8(0x5E);
5003   emit_int8((unsigned char)(0xC0 | encode));
5004 }
5005 
5006 void Assembler::vfmadd231sd(XMMRegister dst, XMMRegister src1, XMMRegister src2) {
5007   assert(VM_Version::supports_fma(), "");
5008   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
5009   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5010   emit_int8((unsigned char)0xB9);
5011   emit_int8((unsigned char)(0xC0 | encode));
5012 }
5013 
5014 void Assembler::vfmadd231ss(XMMRegister dst, XMMRegister src1, XMMRegister src2) {
5015   assert(VM_Version::supports_fma(), "");
5016   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
5017   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5018   emit_int8((unsigned char)0xB9);
5019   emit_int8((unsigned char)(0xC0 | encode));
5020 }
5021 
5022 void Assembler::vmulsd(XMMRegister dst, XMMRegister nds, Address src) {
5023   assert(VM_Version::supports_avx(), "");
5024   InstructionMark im(this);
5025   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
5026   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
5027   attributes.set_rex_vex_w_reverted();
5028   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
5029   emit_int8(0x59);
5030   emit_operand(dst, src);
5031 }
5032 
5033 void Assembler::vmulsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
5034   assert(VM_Version::supports_avx(), "");
5035   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
5036   attributes.set_rex_vex_w_reverted();
5037   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
5038   emit_int8(0x59);
5039   emit_int8((unsigned char)(0xC0 | encode));
5040 }
5041 
5042 void Assembler::vmulss(XMMRegister dst, XMMRegister nds, Address src) {
5043   assert(VM_Version::supports_avx(), "");
5044   InstructionMark im(this);
5045   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
5046   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
5047   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
5048   emit_int8(0x59);
5049   emit_operand(dst, src);
5050 }
5051 
5052 void Assembler::vmulss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
5053   assert(VM_Version::supports_avx(), "");
5054   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
5055   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
5056   emit_int8(0x59);
5057   emit_int8((unsigned char)(0xC0 | encode));
5058 }
5059 
5060 void Assembler::vsubsd(XMMRegister dst, XMMRegister nds, Address src) {
5061   assert(VM_Version::supports_avx(), "");
5062   InstructionMark im(this);
5063   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
5064   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
5065   attributes.set_rex_vex_w_reverted();
5066   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
5067   emit_int8(0x5C);
5068   emit_operand(dst, src);
5069 }
5070 
5071 void Assembler::vsubsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
5072   assert(VM_Version::supports_avx(), "");
5073   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
5074   attributes.set_rex_vex_w_reverted();
5075   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
5076   emit_int8(0x5C);
5077   emit_int8((unsigned char)(0xC0 | encode));
5078 }
5079 
5080 void Assembler::vsubss(XMMRegister dst, XMMRegister nds, Address src) {
5081   assert(VM_Version::supports_avx(), "");
5082   InstructionMark im(this);
5083   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
5084   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
5085   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
5086   emit_int8(0x5C);
5087   emit_operand(dst, src);
5088 }
5089 
5090 void Assembler::vsubss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
5091   assert(VM_Version::supports_avx(), "");
5092   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
5093   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
5094   emit_int8(0x5C);
5095   emit_int8((unsigned char)(0xC0 | encode));
5096 }
5097 
5098 //====================VECTOR ARITHMETIC=====================================
5099 
5100 // Float-point vector arithmetic
5101 
5102 void Assembler::addpd(XMMRegister dst, XMMRegister src) {
5103   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5104   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5105   attributes.set_rex_vex_w_reverted();
5106   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5107   emit_int8(0x58);
5108   emit_int8((unsigned char)(0xC0 | encode));
5109 }
5110 
5111 void Assembler::addpd(XMMRegister dst, Address src) {
5112   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5113   InstructionMark im(this);
5114   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5115   attributes.set_rex_vex_w_reverted();
5116   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
5117   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5118   emit_int8(0x58);
5119   emit_operand(dst, src);
5120 }
5121 
5122 
5123 void Assembler::addps(XMMRegister dst, XMMRegister src) {
5124   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5125   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5126   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
5127   emit_int8(0x58);
5128   emit_int8((unsigned char)(0xC0 | encode));
5129 }
5130 
5131 void Assembler::vaddpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5132   assert(VM_Version::supports_avx(), "");
5133   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5134   attributes.set_rex_vex_w_reverted();
5135   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5136   emit_int8(0x58);
5137   emit_int8((unsigned char)(0xC0 | encode));
5138 }
5139 
5140 void Assembler::vaddps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5141   assert(VM_Version::supports_avx(), "");
5142   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5143   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
5144   emit_int8(0x58);
5145   emit_int8((unsigned char)(0xC0 | encode));
5146 }
5147 
5148 void Assembler::vaddpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5149   assert(VM_Version::supports_avx(), "");
5150   InstructionMark im(this);
5151   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5152   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
5153   attributes.set_rex_vex_w_reverted();
5154   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5155   emit_int8(0x58);
5156   emit_operand(dst, src);
5157 }
5158 
5159 void Assembler::vaddps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5160   assert(VM_Version::supports_avx(), "");
5161   InstructionMark im(this);
5162   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5163   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
5164   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
5165   emit_int8(0x58);
5166   emit_operand(dst, src);
5167 }
5168 
5169 void Assembler::subpd(XMMRegister dst, XMMRegister src) {
5170   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5171   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5172   attributes.set_rex_vex_w_reverted();
5173   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5174   emit_int8(0x5C);
5175   emit_int8((unsigned char)(0xC0 | encode));
5176 }
5177 
5178 void Assembler::subps(XMMRegister dst, XMMRegister src) {
5179   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5180   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5181   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
5182   emit_int8(0x5C);
5183   emit_int8((unsigned char)(0xC0 | encode));
5184 }
5185 
5186 void Assembler::vsubpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5187   assert(VM_Version::supports_avx(), "");
5188   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5189   attributes.set_rex_vex_w_reverted();
5190   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5191   emit_int8(0x5C);
5192   emit_int8((unsigned char)(0xC0 | encode));
5193 }
5194 
5195 void Assembler::vsubps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5196   assert(VM_Version::supports_avx(), "");
5197   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5198   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
5199   emit_int8(0x5C);
5200   emit_int8((unsigned char)(0xC0 | encode));
5201 }
5202 
5203 void Assembler::vsubpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5204   assert(VM_Version::supports_avx(), "");
5205   InstructionMark im(this);
5206   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5207   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
5208   attributes.set_rex_vex_w_reverted();
5209   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5210   emit_int8(0x5C);
5211   emit_operand(dst, src);
5212 }
5213 
5214 void Assembler::vsubps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5215   assert(VM_Version::supports_avx(), "");
5216   InstructionMark im(this);
5217   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5218   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
5219   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
5220   emit_int8(0x5C);
5221   emit_operand(dst, src);
5222 }
5223 
5224 void Assembler::mulpd(XMMRegister dst, XMMRegister src) {
5225   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5226   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5227   attributes.set_rex_vex_w_reverted();
5228   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5229   emit_int8(0x59);
5230   emit_int8((unsigned char)(0xC0 | encode));
5231 }
5232 
5233 void Assembler::mulpd(XMMRegister dst, Address src) {
5234   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5235   InstructionMark im(this);
5236   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5237   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
5238   attributes.set_rex_vex_w_reverted();
5239   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5240   emit_int8(0x59);
5241   emit_operand(dst, src);
5242 }
5243 
5244 void Assembler::mulps(XMMRegister dst, XMMRegister src) {
5245   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5246   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5247   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
5248   emit_int8(0x59);
5249   emit_int8((unsigned char)(0xC0 | encode));
5250 }
5251 
5252 void Assembler::vmulpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5253   assert(VM_Version::supports_avx(), "");
5254   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5255   attributes.set_rex_vex_w_reverted();
5256   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5257   emit_int8(0x59);
5258   emit_int8((unsigned char)(0xC0 | encode));
5259 }
5260 
5261 void Assembler::vmulps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5262   assert(VM_Version::supports_avx(), "");
5263   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5264   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
5265   emit_int8(0x59);
5266   emit_int8((unsigned char)(0xC0 | encode));
5267 }
5268 
5269 void Assembler::vmulpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5270   assert(VM_Version::supports_avx(), "");
5271   InstructionMark im(this);
5272   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5273   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
5274   attributes.set_rex_vex_w_reverted();
5275   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5276   emit_int8(0x59);
5277   emit_operand(dst, src);
5278 }
5279 
5280 void Assembler::vmulps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5281   assert(VM_Version::supports_avx(), "");
5282   InstructionMark im(this);
5283   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5284   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
5285   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
5286   emit_int8(0x59);
5287   emit_operand(dst, src);
5288 }
5289 
5290 void Assembler::vfmadd231pd(XMMRegister dst, XMMRegister src1, XMMRegister src2, int vector_len) {
5291   assert(VM_Version::supports_fma(), "");
5292   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5293   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5294   emit_int8((unsigned char)0xB8);
5295   emit_int8((unsigned char)(0xC0 | encode));
5296 }
5297 
5298 void Assembler::vfmadd231ps(XMMRegister dst, XMMRegister src1, XMMRegister src2, int vector_len) {
5299   assert(VM_Version::supports_fma(), "");
5300   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5301   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5302   emit_int8((unsigned char)0xB8);
5303   emit_int8((unsigned char)(0xC0 | encode));
5304 }
5305 
5306 void Assembler::vfmadd231pd(XMMRegister dst, XMMRegister src1, Address src2, int vector_len) {
5307   assert(VM_Version::supports_fma(), "");
5308   InstructionMark im(this);
5309   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5310   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
5311   vex_prefix(src2, src1->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5312   emit_int8((unsigned char)0xB8);
5313   emit_operand(dst, src2);
5314 }
5315 
5316 void Assembler::vfmadd231ps(XMMRegister dst, XMMRegister src1, Address src2, int vector_len) {
5317   assert(VM_Version::supports_fma(), "");
5318   InstructionMark im(this);
5319   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5320   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
5321   vex_prefix(src2, src1->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5322   emit_int8((unsigned char)0xB8);
5323   emit_operand(dst, src2);
5324 }
5325 
5326 void Assembler::divpd(XMMRegister dst, XMMRegister src) {
5327   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5328   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5329   attributes.set_rex_vex_w_reverted();
5330   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5331   emit_int8(0x5E);
5332   emit_int8((unsigned char)(0xC0 | encode));
5333 }
5334 
5335 void Assembler::divps(XMMRegister dst, XMMRegister src) {
5336   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5337   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5338   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
5339   emit_int8(0x5E);
5340   emit_int8((unsigned char)(0xC0 | encode));
5341 }
5342 
5343 void Assembler::vdivpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5344   assert(VM_Version::supports_avx(), "");
5345   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5346   attributes.set_rex_vex_w_reverted();
5347   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5348   emit_int8(0x5E);
5349   emit_int8((unsigned char)(0xC0 | encode));
5350 }
5351 
5352 void Assembler::vdivps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5353   assert(VM_Version::supports_avx(), "");
5354   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5355   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
5356   emit_int8(0x5E);
5357   emit_int8((unsigned char)(0xC0 | encode));
5358 }
5359 
5360 void Assembler::vdivpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5361   assert(VM_Version::supports_avx(), "");
5362   InstructionMark im(this);
5363   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5364   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
5365   attributes.set_rex_vex_w_reverted();
5366   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5367   emit_int8(0x5E);
5368   emit_operand(dst, src);
5369 }
5370 
5371 void Assembler::vdivps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5372   assert(VM_Version::supports_avx(), "");
5373   InstructionMark im(this);
5374   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5375   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
5376   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
5377   emit_int8(0x5E);
5378   emit_operand(dst, src);
5379 }
5380 
5381 void Assembler::vsqrtpd(XMMRegister dst, XMMRegister src, int vector_len) {
5382   assert(VM_Version::supports_avx(), "");
5383   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5384   attributes.set_rex_vex_w_reverted();
5385   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5386   emit_int8(0x51);
5387   emit_int8((unsigned char)(0xC0 | encode));
5388 }
5389 
5390 void Assembler::vsqrtpd(XMMRegister dst, Address src, int vector_len) {
5391   assert(VM_Version::supports_avx(), "");
5392   InstructionMark im(this);
5393   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5394   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
5395   attributes.set_rex_vex_w_reverted();
5396   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5397   emit_int8(0x51);
5398   emit_operand(dst, src);
5399 }
5400 
5401 void Assembler::vsqrtps(XMMRegister dst, XMMRegister src, int vector_len) {
5402   assert(VM_Version::supports_avx(), "");
5403   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5404   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
5405   emit_int8(0x51);
5406   emit_int8((unsigned char)(0xC0 | encode));
5407 }
5408 
5409 void Assembler::vsqrtps(XMMRegister dst, Address src, int vector_len) {
5410   assert(VM_Version::supports_avx(), "");
5411   InstructionMark im(this);
5412   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5413   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
5414   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
5415   emit_int8(0x51);
5416   emit_operand(dst, src);
5417 }
5418 
5419 void Assembler::andpd(XMMRegister dst, XMMRegister src) {
5420   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5421   InstructionAttr attributes(AVX_128bit, /* rex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
5422   attributes.set_rex_vex_w_reverted();
5423   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5424   emit_int8(0x54);
5425   emit_int8((unsigned char)(0xC0 | encode));
5426 }
5427 
5428 void Assembler::andps(XMMRegister dst, XMMRegister src) {
5429   NOT_LP64(assert(VM_Version::supports_sse(), ""));
5430   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
5431   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
5432   emit_int8(0x54);
5433   emit_int8((unsigned char)(0xC0 | encode));
5434 }
5435 
5436 void Assembler::andps(XMMRegister dst, Address src) {
5437   NOT_LP64(assert(VM_Version::supports_sse(), ""));
5438   InstructionMark im(this);
5439   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
5440   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
5441   simd_prefix(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
5442   emit_int8(0x54);
5443   emit_operand(dst, src);
5444 }
5445 
5446 void Assembler::andpd(XMMRegister dst, Address src) {
5447   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5448   InstructionMark im(this);
5449   InstructionAttr attributes(AVX_128bit, /* rex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
5450   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
5451   attributes.set_rex_vex_w_reverted();
5452   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5453   emit_int8(0x54);
5454   emit_operand(dst, src);
5455 }
5456 
5457 void Assembler::vandpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5458   assert(VM_Version::supports_avx(), "");
5459   InstructionAttr attributes(vector_len, /* vex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
5460   attributes.set_rex_vex_w_reverted();
5461   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5462   emit_int8(0x54);
5463   emit_int8((unsigned char)(0xC0 | encode));
5464 }
5465 
5466 void Assembler::vandps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5467   assert(VM_Version::supports_avx(), "");
5468   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
5469   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
5470   emit_int8(0x54);
5471   emit_int8((unsigned char)(0xC0 | encode));
5472 }
5473 
5474 void Assembler::vandpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5475   assert(VM_Version::supports_avx(), "");
5476   InstructionMark im(this);
5477   InstructionAttr attributes(vector_len, /* vex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
5478   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
5479   attributes.set_rex_vex_w_reverted();
5480   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5481   emit_int8(0x54);
5482   emit_operand(dst, src);
5483 }
5484 
5485 void Assembler::vandps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5486   assert(VM_Version::supports_avx(), "");
5487   InstructionMark im(this);
5488   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
5489   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
5490   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
5491   emit_int8(0x54);
5492   emit_operand(dst, src);
5493 }
5494 
5495 void Assembler::unpckhpd(XMMRegister dst, XMMRegister src) {
5496   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5497   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5498   attributes.set_rex_vex_w_reverted();
5499   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5500   emit_int8(0x15);
5501   emit_int8((unsigned char)(0xC0 | encode));
5502 }
5503 
5504 void Assembler::unpcklpd(XMMRegister dst, XMMRegister src) {
5505   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5506   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5507   attributes.set_rex_vex_w_reverted();
5508   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5509   emit_int8(0x14);
5510   emit_int8((unsigned char)(0xC0 | encode));
5511 }
5512 
5513 void Assembler::xorpd(XMMRegister dst, XMMRegister src) {
5514   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5515   InstructionAttr attributes(AVX_128bit, /* rex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
5516   attributes.set_rex_vex_w_reverted();
5517   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5518   emit_int8(0x57);
5519   emit_int8((unsigned char)(0xC0 | encode));
5520 }
5521 
5522 void Assembler::xorps(XMMRegister dst, XMMRegister src) {
5523   NOT_LP64(assert(VM_Version::supports_sse(), ""));
5524   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
5525   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
5526   emit_int8(0x57);
5527   emit_int8((unsigned char)(0xC0 | encode));
5528 }
5529 
5530 void Assembler::xorpd(XMMRegister dst, Address src) {
5531   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5532   InstructionMark im(this);
5533   InstructionAttr attributes(AVX_128bit, /* rex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
5534   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
5535   attributes.set_rex_vex_w_reverted();
5536   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5537   emit_int8(0x57);
5538   emit_operand(dst, src);
5539 }
5540 
5541 void Assembler::xorps(XMMRegister dst, Address src) {
5542   NOT_LP64(assert(VM_Version::supports_sse(), ""));
5543   InstructionMark im(this);
5544   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
5545   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
5546   simd_prefix(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
5547   emit_int8(0x57);
5548   emit_operand(dst, src);
5549 }
5550 
5551 void Assembler::vxorpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5552   assert(VM_Version::supports_avx(), "");
5553   InstructionAttr attributes(vector_len, /* vex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
5554   attributes.set_rex_vex_w_reverted();
5555   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5556   emit_int8(0x57);
5557   emit_int8((unsigned char)(0xC0 | encode));
5558 }
5559 
5560 void Assembler::vxorps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5561   assert(VM_Version::supports_avx(), "");
5562   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
5563   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
5564   emit_int8(0x57);
5565   emit_int8((unsigned char)(0xC0 | encode));
5566 }
5567 
5568 void Assembler::vxorpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5569   assert(VM_Version::supports_avx(), "");
5570   InstructionMark im(this);
5571   InstructionAttr attributes(vector_len, /* vex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
5572   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
5573   attributes.set_rex_vex_w_reverted();
5574   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5575   emit_int8(0x57);
5576   emit_operand(dst, src);
5577 }
5578 
5579 void Assembler::vxorps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5580   assert(VM_Version::supports_avx(), "");
5581   InstructionMark im(this);
5582   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
5583   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
5584   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
5585   emit_int8(0x57);
5586   emit_operand(dst, src);
5587 }
5588 
5589 // Integer vector arithmetic
5590 void Assembler::vphaddw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5591   assert(VM_Version::supports_avx() && (vector_len == 0) ||
5592          VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
5593   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
5594   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5595   emit_int8(0x01);
5596   emit_int8((unsigned char)(0xC0 | encode));
5597 }
5598 
5599 void Assembler::vphaddd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5600   assert(VM_Version::supports_avx() && (vector_len == 0) ||
5601          VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
5602   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
5603   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5604   emit_int8(0x02);
5605   emit_int8((unsigned char)(0xC0 | encode));
5606 }
5607 
5608 void Assembler::paddb(XMMRegister dst, XMMRegister src) {
5609   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5610   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5611   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5612   emit_int8((unsigned char)0xFC);
5613   emit_int8((unsigned char)(0xC0 | encode));
5614 }
5615 
5616 void Assembler::paddw(XMMRegister dst, XMMRegister src) {
5617   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5618   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5619   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5620   emit_int8((unsigned char)0xFD);
5621   emit_int8((unsigned char)(0xC0 | encode));
5622 }
5623 
5624 void Assembler::paddd(XMMRegister dst, XMMRegister src) {
5625   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5626   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5627   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5628   emit_int8((unsigned char)0xFE);
5629   emit_int8((unsigned char)(0xC0 | encode));
5630 }
5631 
5632 void Assembler::paddd(XMMRegister dst, Address src) {
5633   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5634   InstructionMark im(this);
5635   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5636   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5637   emit_int8((unsigned char)0xFE);
5638   emit_operand(dst, src);
5639 }
5640 
5641 void Assembler::paddq(XMMRegister dst, XMMRegister src) {
5642   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5643   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5644   attributes.set_rex_vex_w_reverted();
5645   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5646   emit_int8((unsigned char)0xD4);
5647   emit_int8((unsigned char)(0xC0 | encode));
5648 }
5649 
5650 void Assembler::phaddw(XMMRegister dst, XMMRegister src) {
5651   assert(VM_Version::supports_sse3(), "");
5652   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
5653   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5654   emit_int8(0x01);
5655   emit_int8((unsigned char)(0xC0 | encode));
5656 }
5657 
5658 void Assembler::phaddd(XMMRegister dst, XMMRegister src) {
5659   assert(VM_Version::supports_sse3(), "");
5660   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
5661   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5662   emit_int8(0x02);
5663   emit_int8((unsigned char)(0xC0 | encode));
5664 }
5665 
5666 void Assembler::vpaddb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5667   assert(UseAVX > 0, "requires some form of AVX");
5668   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5669   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5670   emit_int8((unsigned char)0xFC);
5671   emit_int8((unsigned char)(0xC0 | encode));
5672 }
5673 
5674 void Assembler::vpaddw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5675   assert(UseAVX > 0, "requires some form of AVX");
5676   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5677   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5678   emit_int8((unsigned char)0xFD);
5679   emit_int8((unsigned char)(0xC0 | encode));
5680 }
5681 
5682 void Assembler::vpaddd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5683   assert(UseAVX > 0, "requires some form of AVX");
5684   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5685   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5686   emit_int8((unsigned char)0xFE);
5687   emit_int8((unsigned char)(0xC0 | encode));
5688 }
5689 
5690 void Assembler::vpaddq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5691   assert(UseAVX > 0, "requires some form of AVX");
5692   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5693   attributes.set_rex_vex_w_reverted();
5694   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5695   emit_int8((unsigned char)0xD4);
5696   emit_int8((unsigned char)(0xC0 | encode));
5697 }
5698 
5699 void Assembler::vpaddb(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5700   assert(UseAVX > 0, "requires some form of AVX");
5701   InstructionMark im(this);
5702   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5703   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
5704   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5705   emit_int8((unsigned char)0xFC);
5706   emit_operand(dst, src);
5707 }
5708 
5709 void Assembler::vpaddw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5710   assert(UseAVX > 0, "requires some form of AVX");
5711   InstructionMark im(this);
5712   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5713   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
5714   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5715   emit_int8((unsigned char)0xFD);
5716   emit_operand(dst, src);
5717 }
5718 
5719 void Assembler::vpaddd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5720   assert(UseAVX > 0, "requires some form of AVX");
5721   InstructionMark im(this);
5722   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5723   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
5724   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5725   emit_int8((unsigned char)0xFE);
5726   emit_operand(dst, src);
5727 }
5728 
5729 void Assembler::vpaddq(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5730   assert(UseAVX > 0, "requires some form of AVX");
5731   InstructionMark im(this);
5732   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5733   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
5734   attributes.set_rex_vex_w_reverted();
5735   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5736   emit_int8((unsigned char)0xD4);
5737   emit_operand(dst, src);
5738 }
5739 
5740 void Assembler::psubb(XMMRegister dst, XMMRegister src) {
5741   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5742   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5743   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5744   emit_int8((unsigned char)0xF8);
5745   emit_int8((unsigned char)(0xC0 | encode));
5746 }
5747 
5748 void Assembler::psubw(XMMRegister dst, XMMRegister src) {
5749   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5750   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5751   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5752   emit_int8((unsigned char)0xF9);
5753   emit_int8((unsigned char)(0xC0 | encode));
5754 }
5755 
5756 void Assembler::psubd(XMMRegister dst, XMMRegister src) {
5757   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5758   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5759   emit_int8((unsigned char)0xFA);
5760   emit_int8((unsigned char)(0xC0 | encode));
5761 }
5762 
5763 void Assembler::psubq(XMMRegister dst, XMMRegister src) {
5764   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5765   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5766   attributes.set_rex_vex_w_reverted();
5767   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5768   emit_int8((unsigned char)0xFB);
5769   emit_int8((unsigned char)(0xC0 | encode));
5770 }
5771 
5772 void Assembler::vpsubb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5773   assert(UseAVX > 0, "requires some form of AVX");
5774   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5775   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5776   emit_int8((unsigned char)0xF8);
5777   emit_int8((unsigned char)(0xC0 | encode));
5778 }
5779 
5780 void Assembler::vpsubw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5781   assert(UseAVX > 0, "requires some form of AVX");
5782   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5783   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5784   emit_int8((unsigned char)0xF9);
5785   emit_int8((unsigned char)(0xC0 | encode));
5786 }
5787 
5788 void Assembler::vpsubd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5789   assert(UseAVX > 0, "requires some form of AVX");
5790   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5791   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5792   emit_int8((unsigned char)0xFA);
5793   emit_int8((unsigned char)(0xC0 | encode));
5794 }
5795 
5796 void Assembler::vpsubq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5797   assert(UseAVX > 0, "requires some form of AVX");
5798   InstructionAttr attributes(vector_len, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5799   attributes.set_rex_vex_w_reverted();
5800   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5801   emit_int8((unsigned char)0xFB);
5802   emit_int8((unsigned char)(0xC0 | encode));
5803 }
5804 
5805 void Assembler::vpsubb(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5806   assert(UseAVX > 0, "requires some form of AVX");
5807   InstructionMark im(this);
5808   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5809   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
5810   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5811   emit_int8((unsigned char)0xF8);
5812   emit_operand(dst, src);
5813 }
5814 
5815 void Assembler::vpsubw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5816   assert(UseAVX > 0, "requires some form of AVX");
5817   InstructionMark im(this);
5818   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5819   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
5820   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5821   emit_int8((unsigned char)0xF9);
5822   emit_operand(dst, src);
5823 }
5824 
5825 void Assembler::vpsubd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5826   assert(UseAVX > 0, "requires some form of AVX");
5827   InstructionMark im(this);
5828   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5829   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
5830   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5831   emit_int8((unsigned char)0xFA);
5832   emit_operand(dst, src);
5833 }
5834 
5835 void Assembler::vpsubq(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5836   assert(UseAVX > 0, "requires some form of AVX");
5837   InstructionMark im(this);
5838   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5839   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
5840   attributes.set_rex_vex_w_reverted();
5841   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5842   emit_int8((unsigned char)0xFB);
5843   emit_operand(dst, src);
5844 }
5845 
5846 void Assembler::pmullw(XMMRegister dst, XMMRegister src) {
5847   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5848   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5849   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5850   emit_int8((unsigned char)0xD5);
5851   emit_int8((unsigned char)(0xC0 | encode));
5852 }
5853 
5854 void Assembler::pmulld(XMMRegister dst, XMMRegister src) {
5855   assert(VM_Version::supports_sse4_1(), "");
5856   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5857   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5858   emit_int8(0x40);
5859   emit_int8((unsigned char)(0xC0 | encode));
5860 }
5861 
5862 void Assembler::vpmullw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5863   assert(UseAVX > 0, "requires some form of AVX");
5864   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5865   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5866   emit_int8((unsigned char)0xD5);
5867   emit_int8((unsigned char)(0xC0 | encode));
5868 }
5869 
5870 void Assembler::vpmulld(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5871   assert(UseAVX > 0, "requires some form of AVX");
5872   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5873   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5874   emit_int8(0x40);
5875   emit_int8((unsigned char)(0xC0 | encode));
5876 }
5877 
5878 void Assembler::vpmullq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5879   assert(UseAVX > 2, "requires some form of EVEX");
5880   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
5881   attributes.set_is_evex_instruction();
5882   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5883   emit_int8(0x40);
5884   emit_int8((unsigned char)(0xC0 | encode));
5885 }
5886 
5887 void Assembler::vpmullw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5888   assert(UseAVX > 0, "requires some form of AVX");
5889   InstructionMark im(this);
5890   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5891   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
5892   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5893   emit_int8((unsigned char)0xD5);
5894   emit_operand(dst, src);
5895 }
5896 
5897 void Assembler::vpmulld(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5898   assert(UseAVX > 0, "requires some form of AVX");
5899   InstructionMark im(this);
5900   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5901   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
5902   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5903   emit_int8(0x40);
5904   emit_operand(dst, src);
5905 }
5906 
5907 void Assembler::vpmullq(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5908   assert(UseAVX > 2, "requires some form of EVEX");
5909   InstructionMark im(this);
5910   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
5911   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
5912   attributes.set_is_evex_instruction();
5913   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5914   emit_int8(0x40);
5915   emit_operand(dst, src);
5916 }
5917 
5918 // Shift packed integers left by specified number of bits.
5919 void Assembler::psllw(XMMRegister dst, int shift) {
5920   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5921   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5922   // XMM6 is for /6 encoding: 66 0F 71 /6 ib
5923   int encode = simd_prefix_and_encode(xmm6, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5924   emit_int8(0x71);
5925   emit_int8((unsigned char)(0xC0 | encode));
5926   emit_int8(shift & 0xFF);
5927 }
5928 
5929 void Assembler::pslld(XMMRegister dst, int shift) {
5930   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5931   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5932   // XMM6 is for /6 encoding: 66 0F 72 /6 ib
5933   int encode = simd_prefix_and_encode(xmm6, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5934   emit_int8(0x72);
5935   emit_int8((unsigned char)(0xC0 | encode));
5936   emit_int8(shift & 0xFF);
5937 }
5938 
5939 void Assembler::psllq(XMMRegister dst, int shift) {
5940   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5941   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5942   // XMM6 is for /6 encoding: 66 0F 73 /6 ib
5943   int encode = simd_prefix_and_encode(xmm6, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5944   emit_int8(0x73);
5945   emit_int8((unsigned char)(0xC0 | encode));
5946   emit_int8(shift & 0xFF);
5947 }
5948 
5949 void Assembler::psllw(XMMRegister dst, XMMRegister shift) {
5950   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5951   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5952   int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5953   emit_int8((unsigned char)0xF1);
5954   emit_int8((unsigned char)(0xC0 | encode));
5955 }
5956 
5957 void Assembler::pslld(XMMRegister dst, XMMRegister shift) {
5958   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5959   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5960   int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5961   emit_int8((unsigned char)0xF2);
5962   emit_int8((unsigned char)(0xC0 | encode));
5963 }
5964 
5965 void Assembler::psllq(XMMRegister dst, XMMRegister shift) {
5966   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5967   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5968   attributes.set_rex_vex_w_reverted();
5969   int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5970   emit_int8((unsigned char)0xF3);
5971   emit_int8((unsigned char)(0xC0 | encode));
5972 }
5973 
5974 void Assembler::vpsllw(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
5975   assert(UseAVX > 0, "requires some form of AVX");
5976   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5977   // XMM6 is for /6 encoding: 66 0F 71 /6 ib
5978   int encode = vex_prefix_and_encode(xmm6->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5979   emit_int8(0x71);
5980   emit_int8((unsigned char)(0xC0 | encode));
5981   emit_int8(shift & 0xFF);
5982 }
5983 
5984 void Assembler::vpslld(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
5985   assert(UseAVX > 0, "requires some form of AVX");
5986   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5987   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5988   // XMM6 is for /6 encoding: 66 0F 72 /6 ib
5989   int encode = vex_prefix_and_encode(xmm6->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5990   emit_int8(0x72);
5991   emit_int8((unsigned char)(0xC0 | encode));
5992   emit_int8(shift & 0xFF);
5993 }
5994 
5995 void Assembler::vpsllq(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
5996   assert(UseAVX > 0, "requires some form of AVX");
5997   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5998   attributes.set_rex_vex_w_reverted();
5999   // XMM6 is for /6 encoding: 66 0F 73 /6 ib
6000   int encode = vex_prefix_and_encode(xmm6->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6001   emit_int8(0x73);
6002   emit_int8((unsigned char)(0xC0 | encode));
6003   emit_int8(shift & 0xFF);
6004 }
6005 
6006 void Assembler::vpsllw(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
6007   assert(UseAVX > 0, "requires some form of AVX");
6008   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
6009   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6010   emit_int8((unsigned char)0xF1);
6011   emit_int8((unsigned char)(0xC0 | encode));
6012 }
6013 
6014 void Assembler::vpslld(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
6015   assert(UseAVX > 0, "requires some form of AVX");
6016   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6017   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6018   emit_int8((unsigned char)0xF2);
6019   emit_int8((unsigned char)(0xC0 | encode));
6020 }
6021 
6022 void Assembler::vpsllq(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
6023   assert(UseAVX > 0, "requires some form of AVX");
6024   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6025   attributes.set_rex_vex_w_reverted();
6026   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6027   emit_int8((unsigned char)0xF3);
6028   emit_int8((unsigned char)(0xC0 | encode));
6029 }
6030 
6031 // Shift packed integers logically right by specified number of bits.
6032 void Assembler::psrlw(XMMRegister dst, int shift) {
6033   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
6034   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
6035   // XMM2 is for /2 encoding: 66 0F 71 /2 ib
6036   int encode = simd_prefix_and_encode(xmm2, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6037   emit_int8(0x71);
6038   emit_int8((unsigned char)(0xC0 | encode));
6039   emit_int8(shift & 0xFF);
6040 }
6041 
6042 void Assembler::psrld(XMMRegister dst, int shift) {
6043   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
6044   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6045   // XMM2 is for /2 encoding: 66 0F 72 /2 ib
6046   int encode = simd_prefix_and_encode(xmm2, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6047   emit_int8(0x72);
6048   emit_int8((unsigned char)(0xC0 | encode));
6049   emit_int8(shift & 0xFF);
6050 }
6051 
6052 void Assembler::psrlq(XMMRegister dst, int shift) {
6053   // Do not confuse it with psrldq SSE2 instruction which
6054   // shifts 128 bit value in xmm register by number of bytes.
6055   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
6056   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6057   attributes.set_rex_vex_w_reverted();
6058   // XMM2 is for /2 encoding: 66 0F 73 /2 ib
6059   int encode = simd_prefix_and_encode(xmm2, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6060   emit_int8(0x73);
6061   emit_int8((unsigned char)(0xC0 | encode));
6062   emit_int8(shift & 0xFF);
6063 }
6064 
6065 void Assembler::psrlw(XMMRegister dst, XMMRegister shift) {
6066   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
6067   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
6068   int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6069   emit_int8((unsigned char)0xD1);
6070   emit_int8((unsigned char)(0xC0 | encode));
6071 }
6072 
6073 void Assembler::psrld(XMMRegister dst, XMMRegister shift) {
6074   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
6075   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6076   int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6077   emit_int8((unsigned char)0xD2);
6078   emit_int8((unsigned char)(0xC0 | encode));
6079 }
6080 
6081 void Assembler::psrlq(XMMRegister dst, XMMRegister shift) {
6082   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
6083   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6084   attributes.set_rex_vex_w_reverted();
6085   int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6086   emit_int8((unsigned char)0xD3);
6087   emit_int8((unsigned char)(0xC0 | encode));
6088 }
6089 
6090 void Assembler::vpsrlw(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
6091   assert(UseAVX > 0, "requires some form of AVX");
6092   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
6093   // XMM2 is for /2 encoding: 66 0F 71 /2 ib
6094   int encode = vex_prefix_and_encode(xmm2->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6095   emit_int8(0x71);
6096   emit_int8((unsigned char)(0xC0 | encode));
6097   emit_int8(shift & 0xFF);
6098 }
6099 
6100 void Assembler::vpsrld(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
6101   assert(UseAVX > 0, "requires some form of AVX");
6102   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6103   // XMM2 is for /2 encoding: 66 0F 72 /2 ib
6104   int encode = vex_prefix_and_encode(xmm2->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6105   emit_int8(0x72);
6106   emit_int8((unsigned char)(0xC0 | encode));
6107   emit_int8(shift & 0xFF);
6108 }
6109 
6110 void Assembler::vpsrlq(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
6111   assert(UseAVX > 0, "requires some form of AVX");
6112   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6113   attributes.set_rex_vex_w_reverted();
6114   // XMM2 is for /2 encoding: 66 0F 73 /2 ib
6115   int encode = vex_prefix_and_encode(xmm2->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6116   emit_int8(0x73);
6117   emit_int8((unsigned char)(0xC0 | encode));
6118   emit_int8(shift & 0xFF);
6119 }
6120 
6121 void Assembler::vpsrlw(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
6122   assert(UseAVX > 0, "requires some form of AVX");
6123   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
6124   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6125   emit_int8((unsigned char)0xD1);
6126   emit_int8((unsigned char)(0xC0 | encode));
6127 }
6128 
6129 void Assembler::vpsrld(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
6130   assert(UseAVX > 0, "requires some form of AVX");
6131   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6132   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6133   emit_int8((unsigned char)0xD2);
6134   emit_int8((unsigned char)(0xC0 | encode));
6135 }
6136 
6137 void Assembler::vpsrlq(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
6138   assert(UseAVX > 0, "requires some form of AVX");
6139   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6140   attributes.set_rex_vex_w_reverted();
6141   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6142   emit_int8((unsigned char)0xD3);
6143   emit_int8((unsigned char)(0xC0 | encode));
6144 }
6145 
6146 void Assembler::evpsrlvw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
6147   assert(VM_Version::supports_avx512bw(), "");
6148   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6149   attributes.set_is_evex_instruction();
6150   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6151   emit_int8(0x10);
6152   emit_int8((unsigned char)(0xC0 | encode));
6153 }
6154 
6155 void Assembler::evpsllvw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
6156   assert(VM_Version::supports_avx512bw(), "");
6157   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6158   attributes.set_is_evex_instruction();
6159   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6160   emit_int8(0x12);
6161   emit_int8((unsigned char)(0xC0 | encode));
6162 }
6163 
6164 // Shift packed integers arithmetically right by specified number of bits.
6165 void Assembler::psraw(XMMRegister dst, int shift) {
6166   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
6167   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
6168   // XMM4 is for /4 encoding: 66 0F 71 /4 ib
6169   int encode = simd_prefix_and_encode(xmm4, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6170   emit_int8(0x71);
6171   emit_int8((unsigned char)(0xC0 | encode));
6172   emit_int8(shift & 0xFF);
6173 }
6174 
6175 void Assembler::psrad(XMMRegister dst, int shift) {
6176   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
6177   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6178   // XMM4 is for /4 encoding: 66 0F 72 /4 ib
6179   int encode = simd_prefix_and_encode(xmm4, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6180   emit_int8(0x72);
6181   emit_int8((unsigned char)(0xC0 | encode));
6182   emit_int8(shift & 0xFF);
6183 }
6184 
6185 void Assembler::psraw(XMMRegister dst, XMMRegister shift) {
6186   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
6187   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
6188   int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6189   emit_int8((unsigned char)0xE1);
6190   emit_int8((unsigned char)(0xC0 | encode));
6191 }
6192 
6193 void Assembler::psrad(XMMRegister dst, XMMRegister shift) {
6194   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
6195   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6196   int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6197   emit_int8((unsigned char)0xE2);
6198   emit_int8((unsigned char)(0xC0 | encode));
6199 }
6200 
6201 void Assembler::vpsraw(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
6202   assert(UseAVX > 0, "requires some form of AVX");
6203   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
6204   // XMM4 is for /4 encoding: 66 0F 71 /4 ib
6205   int encode = vex_prefix_and_encode(xmm4->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6206   emit_int8(0x71);
6207   emit_int8((unsigned char)(0xC0 | encode));
6208   emit_int8(shift & 0xFF);
6209 }
6210 
6211 void Assembler::vpsrad(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
6212   assert(UseAVX > 0, "requires some form of AVX");
6213   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6214   // XMM4 is for /4 encoding: 66 0F 71 /4 ib
6215   int encode = vex_prefix_and_encode(xmm4->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6216   emit_int8(0x72);
6217   emit_int8((unsigned char)(0xC0 | encode));
6218   emit_int8(shift & 0xFF);
6219 }
6220 
6221 void Assembler::vpsraw(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
6222   assert(UseAVX > 0, "requires some form of AVX");
6223   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
6224   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6225   emit_int8((unsigned char)0xE1);
6226   emit_int8((unsigned char)(0xC0 | encode));
6227 }
6228 
6229 void Assembler::vpsrad(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
6230   assert(UseAVX > 0, "requires some form of AVX");
6231   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6232   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6233   emit_int8((unsigned char)0xE2);
6234   emit_int8((unsigned char)(0xC0 | encode));
6235 }
6236 
6237 
6238 // logical operations packed integers
6239 void Assembler::pand(XMMRegister dst, XMMRegister src) {
6240   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
6241   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6242   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6243   emit_int8((unsigned char)0xDB);
6244   emit_int8((unsigned char)(0xC0 | encode));
6245 }
6246 
6247 void Assembler::vpand(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
6248   assert(UseAVX > 0, "requires some form of AVX");
6249   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6250   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6251   emit_int8((unsigned char)0xDB);
6252   emit_int8((unsigned char)(0xC0 | encode));
6253 }
6254 
6255 void Assembler::vpand(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
6256   assert(UseAVX > 0, "requires some form of AVX");
6257   InstructionMark im(this);
6258   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6259   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
6260   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6261   emit_int8((unsigned char)0xDB);
6262   emit_operand(dst, src);
6263 }
6264 
6265 void Assembler::vpandq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
6266   assert(VM_Version::supports_evex(), "");
6267   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6268   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6269   emit_int8((unsigned char)0xDB);
6270   emit_int8((unsigned char)(0xC0 | encode));
6271 }
6272 
6273 
6274 void Assembler::pandn(XMMRegister dst, XMMRegister src) {
6275   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
6276   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6277   attributes.set_rex_vex_w_reverted();
6278   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6279   emit_int8((unsigned char)0xDF);
6280   emit_int8((unsigned char)(0xC0 | encode));
6281 }
6282 
6283 void Assembler::por(XMMRegister dst, XMMRegister src) {
6284   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
6285   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6286   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6287   emit_int8((unsigned char)0xEB);
6288   emit_int8((unsigned char)(0xC0 | encode));
6289 }
6290 
6291 void Assembler::vpor(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
6292   assert(UseAVX > 0, "requires some form of AVX");
6293   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6294   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6295   emit_int8((unsigned char)0xEB);
6296   emit_int8((unsigned char)(0xC0 | encode));
6297 }
6298 
6299 void Assembler::vpor(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
6300   assert(UseAVX > 0, "requires some form of AVX");
6301   InstructionMark im(this);
6302   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6303   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
6304   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6305   emit_int8((unsigned char)0xEB);
6306   emit_operand(dst, src);
6307 }
6308 
6309 void Assembler::vporq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
6310   assert(VM_Version::supports_evex(), "");
6311   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6312   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6313   emit_int8((unsigned char)0xEB);
6314   emit_int8((unsigned char)(0xC0 | encode));
6315 }
6316 
6317 
6318 void Assembler::pxor(XMMRegister dst, XMMRegister src) {
6319   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
6320   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6321   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6322   emit_int8((unsigned char)0xEF);
6323   emit_int8((unsigned char)(0xC0 | encode));
6324 }
6325 
6326 void Assembler::vpxor(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
6327   assert(UseAVX > 0, "requires some form of AVX");
6328   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6329   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6330   emit_int8((unsigned char)0xEF);
6331   emit_int8((unsigned char)(0xC0 | encode));
6332 }
6333 
6334 void Assembler::vpxor(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
6335   assert(UseAVX > 0, "requires some form of AVX");
6336   InstructionMark im(this);
6337   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6338   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
6339   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6340   emit_int8((unsigned char)0xEF);
6341   emit_operand(dst, src);
6342 }
6343 
6344 void Assembler::evpxorq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
6345   assert(VM_Version::supports_evex(), "requires EVEX support");
6346   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6347   attributes.set_is_evex_instruction();
6348   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6349   emit_int8((unsigned char)0xEF);
6350   emit_int8((unsigned char)(0xC0 | encode));
6351 }
6352 
6353 void Assembler::evpxorq(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
6354   assert(VM_Version::supports_evex(), "requires EVEX support");
6355   assert(dst != xnoreg, "sanity");
6356   InstructionMark im(this);
6357   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6358   attributes.set_is_evex_instruction();
6359   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
6360   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6361   emit_int8((unsigned char)0xEF);
6362   emit_operand(dst, src);
6363 }
6364 
6365 
6366 // vinserti forms
6367 
6368 void Assembler::vinserti128(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) {
6369   assert(VM_Version::supports_avx2(), "");
6370   assert(imm8 <= 0x01, "imm8: %u", imm8);
6371   int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_256bit;
6372   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6373   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6374   emit_int8(0x38);
6375   emit_int8((unsigned char)(0xC0 | encode));
6376   // 0x00 - insert into lower 128 bits
6377   // 0x01 - insert into upper 128 bits
6378   emit_int8(imm8 & 0x01);
6379 }
6380 
6381 void Assembler::vinserti128(XMMRegister dst, XMMRegister nds, Address src, uint8_t imm8) {
6382   assert(VM_Version::supports_avx2(), "");
6383   assert(dst != xnoreg, "sanity");
6384   assert(imm8 <= 0x01, "imm8: %u", imm8);
6385   int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_256bit;
6386   InstructionMark im(this);
6387   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6388   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
6389   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6390   emit_int8(0x38);
6391   emit_operand(dst, src);
6392   // 0x00 - insert into lower 128 bits
6393   // 0x01 - insert into upper 128 bits
6394   emit_int8(imm8 & 0x01);
6395 }
6396 
6397 void Assembler::vinserti32x4(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) {
6398   assert(VM_Version::supports_evex(), "");
6399   assert(imm8 <= 0x03, "imm8: %u", imm8);
6400   InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6401   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6402   emit_int8(0x38);
6403   emit_int8((unsigned char)(0xC0 | encode));
6404   // 0x00 - insert into q0 128 bits (0..127)
6405   // 0x01 - insert into q1 128 bits (128..255)
6406   // 0x02 - insert into q2 128 bits (256..383)
6407   // 0x03 - insert into q3 128 bits (384..511)
6408   emit_int8(imm8 & 0x03);
6409 }
6410 
6411 void Assembler::vinserti32x4(XMMRegister dst, XMMRegister nds, Address src, uint8_t imm8) {
6412   assert(VM_Version::supports_avx(), "");
6413   assert(dst != xnoreg, "sanity");
6414   assert(imm8 <= 0x03, "imm8: %u", imm8);
6415   int vector_len = VM_Version::supports_evex() ? AVX_512bit : AVX_256bit;
6416   InstructionMark im(this);
6417   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6418   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
6419   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6420   emit_int8(0x18);
6421   emit_operand(dst, src);
6422   // 0x00 - insert into q0 128 bits (0..127)
6423   // 0x01 - insert into q1 128 bits (128..255)
6424   // 0x02 - insert into q2 128 bits (256..383)
6425   // 0x03 - insert into q3 128 bits (384..511)
6426   emit_int8(imm8 & 0x03);
6427 }
6428 
6429 void Assembler::vinserti64x4(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) {
6430   assert(VM_Version::supports_evex(), "");
6431   assert(imm8 <= 0x01, "imm8: %u", imm8);
6432   InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6433   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6434   emit_int8(0x38);
6435   emit_int8((unsigned char)(0xC0 | encode));
6436   // 0x00 - insert into lower 256 bits
6437   // 0x01 - insert into upper 256 bits
6438   emit_int8(imm8 & 0x01);
6439 }
6440 
6441 
6442 // vinsertf forms
6443 
6444 void Assembler::vinsertf128(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) {
6445   assert(VM_Version::supports_avx(), "");
6446   assert(imm8 <= 0x01, "imm8: %u", imm8);
6447   int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_256bit;
6448   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6449   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6450   emit_int8(0x18);
6451   emit_int8((unsigned char)(0xC0 | encode));
6452   // 0x00 - insert into lower 128 bits
6453   // 0x01 - insert into upper 128 bits
6454   emit_int8(imm8 & 0x01);
6455 }
6456 
6457 void Assembler::vinsertf128(XMMRegister dst, XMMRegister nds, Address src, uint8_t imm8) {
6458   assert(VM_Version::supports_avx(), "");
6459   assert(dst != xnoreg, "sanity");
6460   assert(imm8 <= 0x01, "imm8: %u", imm8);
6461   int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_256bit;
6462   InstructionMark im(this);
6463   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6464   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
6465   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6466   emit_int8(0x18);
6467   emit_operand(dst, src);
6468   // 0x00 - insert into lower 128 bits
6469   // 0x01 - insert into upper 128 bits
6470   emit_int8(imm8 & 0x01);
6471 }
6472 
6473 void Assembler::vinsertf32x4(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) {
6474   assert(VM_Version::supports_evex(), "");
6475   assert(imm8 <= 0x03, "imm8: %u", imm8);
6476   InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6477   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6478   emit_int8(0x18);
6479   emit_int8((unsigned char)(0xC0 | encode));
6480   // 0x00 - insert into q0 128 bits (0..127)
6481   // 0x01 - insert into q1 128 bits (128..255)
6482   // 0x02 - insert into q2 128 bits (256..383)
6483   // 0x03 - insert into q3 128 bits (384..511)
6484   emit_int8(imm8 & 0x03);
6485 }
6486 
6487 void Assembler::vinsertf32x4(XMMRegister dst, XMMRegister nds, Address src, uint8_t imm8) {
6488   assert(VM_Version::supports_avx(), "");
6489   assert(dst != xnoreg, "sanity");
6490   assert(imm8 <= 0x03, "imm8: %u", imm8);
6491   int vector_len = VM_Version::supports_evex() ? AVX_512bit : AVX_256bit;
6492   InstructionMark im(this);
6493   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6494   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
6495   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6496   emit_int8(0x18);
6497   emit_operand(dst, src);
6498   // 0x00 - insert into q0 128 bits (0..127)
6499   // 0x01 - insert into q1 128 bits (128..255)
6500   // 0x02 - insert into q2 128 bits (256..383)
6501   // 0x03 - insert into q3 128 bits (384..511)
6502   emit_int8(imm8 & 0x03);
6503 }
6504 
6505 void Assembler::vinsertf64x4(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) {
6506   assert(VM_Version::supports_evex(), "");
6507   assert(imm8 <= 0x01, "imm8: %u", imm8);
6508   InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6509   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6510   emit_int8(0x1A);
6511   emit_int8((unsigned char)(0xC0 | encode));
6512   // 0x00 - insert into lower 256 bits
6513   // 0x01 - insert into upper 256 bits
6514   emit_int8(imm8 & 0x01);
6515 }
6516 
6517 void Assembler::vinsertf64x4(XMMRegister dst, XMMRegister nds, Address src, uint8_t imm8) {
6518   assert(VM_Version::supports_evex(), "");
6519   assert(dst != xnoreg, "sanity");
6520   assert(imm8 <= 0x01, "imm8: %u", imm8);
6521   InstructionMark im(this);
6522   InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6523   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_64bit);
6524   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6525   emit_int8(0x1A);
6526   emit_operand(dst, src);
6527   // 0x00 - insert into lower 256 bits
6528   // 0x01 - insert into upper 256 bits
6529   emit_int8(imm8 & 0x01);
6530 }
6531 
6532 
6533 // vextracti forms
6534 
6535 void Assembler::vextracti128(XMMRegister dst, XMMRegister src, uint8_t imm8) {
6536   assert(VM_Version::supports_avx(), "");
6537   assert(imm8 <= 0x01, "imm8: %u", imm8);
6538   int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_256bit;
6539   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6540   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6541   emit_int8(0x39);
6542   emit_int8((unsigned char)(0xC0 | encode));
6543   // 0x00 - extract from lower 128 bits
6544   // 0x01 - extract from upper 128 bits
6545   emit_int8(imm8 & 0x01);
6546 }
6547 
6548 void Assembler::vextracti128(Address dst, XMMRegister src, uint8_t imm8) {
6549   assert(VM_Version::supports_avx2(), "");
6550   assert(src != xnoreg, "sanity");
6551   assert(imm8 <= 0x01, "imm8: %u", imm8);
6552   int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_256bit;
6553   InstructionMark im(this);
6554   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6555   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
6556   attributes.reset_is_clear_context();
6557   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6558   emit_int8(0x39);
6559   emit_operand(src, dst);
6560   // 0x00 - extract from lower 128 bits
6561   // 0x01 - extract from upper 128 bits
6562   emit_int8(imm8 & 0x01);
6563 }
6564 
6565 void Assembler::vextracti32x4(XMMRegister dst, XMMRegister src, uint8_t imm8) {
6566   assert(VM_Version::supports_avx(), "");
6567   assert(imm8 <= 0x03, "imm8: %u", imm8);
6568   int vector_len = VM_Version::supports_evex() ? AVX_512bit : AVX_256bit;
6569   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6570   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6571   emit_int8(0x39);
6572   emit_int8((unsigned char)(0xC0 | encode));
6573   // 0x00 - extract from bits 127:0
6574   // 0x01 - extract from bits 255:128
6575   // 0x02 - extract from bits 383:256
6576   // 0x03 - extract from bits 511:384
6577   emit_int8(imm8 & 0x03);
6578 }
6579 
6580 void Assembler::vextracti32x4(Address dst, XMMRegister src, uint8_t imm8) {
6581   assert(VM_Version::supports_evex(), "");
6582   assert(src != xnoreg, "sanity");
6583   assert(imm8 <= 0x03, "imm8: %u", imm8);
6584   InstructionMark im(this);
6585   InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6586   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
6587   attributes.reset_is_clear_context();
6588   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6589   emit_int8(0x39);
6590   emit_operand(src, dst);
6591   // 0x00 - extract from bits 127:0
6592   // 0x01 - extract from bits 255:128
6593   // 0x02 - extract from bits 383:256
6594   // 0x03 - extract from bits 511:384
6595   emit_int8(imm8 & 0x03);
6596 }
6597 
6598 void Assembler::vextracti64x2(XMMRegister dst, XMMRegister src, uint8_t imm8) {
6599   assert(VM_Version::supports_avx512dq(), "");
6600   assert(imm8 <= 0x03, "imm8: %u", imm8);
6601   InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6602   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6603   emit_int8(0x39);
6604   emit_int8((unsigned char)(0xC0 | encode));
6605   // 0x00 - extract from bits 127:0
6606   // 0x01 - extract from bits 255:128
6607   // 0x02 - extract from bits 383:256
6608   // 0x03 - extract from bits 511:384
6609   emit_int8(imm8 & 0x03);
6610 }
6611 
6612 void Assembler::vextracti64x4(XMMRegister dst, XMMRegister src, uint8_t imm8) {
6613   assert(VM_Version::supports_evex(), "");
6614   assert(imm8 <= 0x01, "imm8: %u", imm8);
6615   InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6616   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6617   emit_int8(0x3B);
6618   emit_int8((unsigned char)(0xC0 | encode));
6619   // 0x00 - extract from lower 256 bits
6620   // 0x01 - extract from upper 256 bits
6621   emit_int8(imm8 & 0x01);
6622 }
6623 
6624 
6625 // vextractf forms
6626 
6627 void Assembler::vextractf128(XMMRegister dst, XMMRegister src, uint8_t imm8) {
6628   assert(VM_Version::supports_avx(), "");
6629   assert(imm8 <= 0x01, "imm8: %u", imm8);
6630   int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_256bit;
6631   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6632   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6633   emit_int8(0x19);
6634   emit_int8((unsigned char)(0xC0 | encode));
6635   // 0x00 - extract from lower 128 bits
6636   // 0x01 - extract from upper 128 bits
6637   emit_int8(imm8 & 0x01);
6638 }
6639 
6640 void Assembler::vextractf128(Address dst, XMMRegister src, uint8_t imm8) {
6641   assert(VM_Version::supports_avx(), "");
6642   assert(src != xnoreg, "sanity");
6643   assert(imm8 <= 0x01, "imm8: %u", imm8);
6644   int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_256bit;
6645   InstructionMark im(this);
6646   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6647   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
6648   attributes.reset_is_clear_context();
6649   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6650   emit_int8(0x19);
6651   emit_operand(src, dst);
6652   // 0x00 - extract from lower 128 bits
6653   // 0x01 - extract from upper 128 bits
6654   emit_int8(imm8 & 0x01);
6655 }
6656 
6657 void Assembler::vextractf32x4(XMMRegister dst, XMMRegister src, uint8_t imm8) {
6658   assert(VM_Version::supports_avx(), "");
6659   assert(imm8 <= 0x03, "imm8: %u", imm8);
6660   int vector_len = VM_Version::supports_evex() ? AVX_512bit : AVX_256bit;
6661   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6662   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6663   emit_int8(0x19);
6664   emit_int8((unsigned char)(0xC0 | encode));
6665   // 0x00 - extract from bits 127:0
6666   // 0x01 - extract from bits 255:128
6667   // 0x02 - extract from bits 383:256
6668   // 0x03 - extract from bits 511:384
6669   emit_int8(imm8 & 0x03);
6670 }
6671 
6672 void Assembler::vextractf32x4(Address dst, XMMRegister src, uint8_t imm8) {
6673   assert(VM_Version::supports_evex(), "");
6674   assert(src != xnoreg, "sanity");
6675   assert(imm8 <= 0x03, "imm8: %u", imm8);
6676   InstructionMark im(this);
6677   InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6678   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
6679   attributes.reset_is_clear_context();
6680   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6681   emit_int8(0x19);
6682   emit_operand(src, dst);
6683   // 0x00 - extract from bits 127:0
6684   // 0x01 - extract from bits 255:128
6685   // 0x02 - extract from bits 383:256
6686   // 0x03 - extract from bits 511:384
6687   emit_int8(imm8 & 0x03);
6688 }
6689 
6690 void Assembler::vextractf64x2(XMMRegister dst, XMMRegister src, uint8_t imm8) {
6691   assert(VM_Version::supports_avx512dq(), "");
6692   assert(imm8 <= 0x03, "imm8: %u", imm8);
6693   InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6694   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6695   emit_int8(0x19);
6696   emit_int8((unsigned char)(0xC0 | encode));
6697   // 0x00 - extract from bits 127:0
6698   // 0x01 - extract from bits 255:128
6699   // 0x02 - extract from bits 383:256
6700   // 0x03 - extract from bits 511:384
6701   emit_int8(imm8 & 0x03);
6702 }
6703 
6704 void Assembler::vextractf64x4(XMMRegister dst, XMMRegister src, uint8_t imm8) {
6705   assert(VM_Version::supports_evex(), "");
6706   assert(imm8 <= 0x01, "imm8: %u", imm8);
6707   InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6708   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6709   emit_int8(0x1B);
6710   emit_int8((unsigned char)(0xC0 | encode));
6711   // 0x00 - extract from lower 256 bits
6712   // 0x01 - extract from upper 256 bits
6713   emit_int8(imm8 & 0x01);
6714 }
6715 
6716 void Assembler::vextractf64x4(Address dst, XMMRegister src, uint8_t imm8) {
6717   assert(VM_Version::supports_evex(), "");
6718   assert(src != xnoreg, "sanity");
6719   assert(imm8 <= 0x01, "imm8: %u", imm8);
6720   InstructionMark im(this);
6721   InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6722   attributes.set_address_attributes(/* tuple_type */ EVEX_T4,/* input_size_in_bits */  EVEX_64bit);
6723   attributes.reset_is_clear_context();
6724   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6725   emit_int8(0x1B);
6726   emit_operand(src, dst);
6727   // 0x00 - extract from lower 256 bits
6728   // 0x01 - extract from upper 256 bits
6729   emit_int8(imm8 & 0x01);
6730 }
6731 
6732 
6733 // legacy word/dword replicate
6734 void Assembler::vpbroadcastw(XMMRegister dst, XMMRegister src) {
6735   assert(VM_Version::supports_avx2(), "");
6736   InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
6737   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6738   emit_int8(0x79);
6739   emit_int8((unsigned char)(0xC0 | encode));
6740 }
6741 
6742 void Assembler::vpbroadcastd(XMMRegister dst, XMMRegister src) {
6743   assert(VM_Version::supports_avx2(), "");
6744   InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6745   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6746   emit_int8(0x58);
6747   emit_int8((unsigned char)(0xC0 | encode));
6748 }
6749 
6750 
6751 // xmm/mem sourced byte/word/dword/qword replicate
6752 
6753 // duplicate 1-byte integer data from src into programmed locations in dest : requires AVX512BW and AVX512VL
6754 void Assembler::evpbroadcastb(XMMRegister dst, XMMRegister src, int vector_len) {
6755   assert(VM_Version::supports_evex(), "");
6756   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
6757   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6758   emit_int8(0x78);
6759   emit_int8((unsigned char)(0xC0 | encode));
6760 }
6761 
6762 void Assembler::evpbroadcastb(XMMRegister dst, Address src, int vector_len) {
6763   assert(VM_Version::supports_evex(), "");
6764   assert(dst != xnoreg, "sanity");
6765   InstructionMark im(this);
6766   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
6767   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_8bit);
6768   // swap src<->dst for encoding
6769   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6770   emit_int8(0x78);
6771   emit_operand(dst, src);
6772 }
6773 
6774 // duplicate 2-byte integer data from src into programmed locations in dest : requires AVX512BW and AVX512VL
6775 void Assembler::evpbroadcastw(XMMRegister dst, XMMRegister src, int vector_len) {
6776   assert(VM_Version::supports_evex(), "");
6777   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
6778   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6779   emit_int8(0x79);
6780   emit_int8((unsigned char)(0xC0 | encode));
6781 }
6782 
6783 void Assembler::evpbroadcastw(XMMRegister dst, Address src, int vector_len) {
6784   assert(VM_Version::supports_evex(), "");
6785   assert(dst != xnoreg, "sanity");
6786   InstructionMark im(this);
6787   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
6788   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_16bit);
6789   // swap src<->dst for encoding
6790   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6791   emit_int8(0x79);
6792   emit_operand(dst, src);
6793 }
6794 
6795 // duplicate 4-byte integer data from src into programmed locations in dest : requires AVX512VL
6796 void Assembler::evpbroadcastd(XMMRegister dst, XMMRegister src, int vector_len) {
6797   assert(VM_Version::supports_evex(), "");
6798   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6799   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6800   emit_int8(0x58);
6801   emit_int8((unsigned char)(0xC0 | encode));
6802 }
6803 
6804 void Assembler::evpbroadcastd(XMMRegister dst, Address src, int vector_len) {
6805   assert(VM_Version::supports_evex(), "");
6806   assert(dst != xnoreg, "sanity");
6807   InstructionMark im(this);
6808   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6809   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
6810   // swap src<->dst for encoding
6811   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6812   emit_int8(0x58);
6813   emit_operand(dst, src);
6814 }
6815 
6816 // duplicate 8-byte integer data from src into programmed locations in dest : requires AVX512VL
6817 void Assembler::evpbroadcastq(XMMRegister dst, XMMRegister src, int vector_len) {
6818   assert(VM_Version::supports_evex(), "");
6819   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6820   attributes.set_rex_vex_w_reverted();
6821   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6822   emit_int8(0x59);
6823   emit_int8((unsigned char)(0xC0 | encode));
6824 }
6825 
6826 void Assembler::evpbroadcastq(XMMRegister dst, Address src, int vector_len) {
6827   assert(VM_Version::supports_evex(), "");
6828   assert(dst != xnoreg, "sanity");
6829   InstructionMark im(this);
6830   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6831   attributes.set_rex_vex_w_reverted();
6832   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
6833   // swap src<->dst for encoding
6834   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6835   emit_int8(0x59);
6836   emit_operand(dst, src);
6837 }
6838 void Assembler::evbroadcasti64x2(XMMRegister dst, XMMRegister src, int vector_len) {
6839   assert(vector_len != Assembler::AVX_128bit, "");
6840   assert(VM_Version::supports_avx512dq(), "");
6841   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6842   attributes.set_rex_vex_w_reverted();
6843   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6844   emit_int8(0x5A);
6845   emit_int8((unsigned char)(0xC0 | encode));
6846 }
6847 
6848 void Assembler::evbroadcasti64x2(XMMRegister dst, Address src, int vector_len) {
6849   assert(vector_len != Assembler::AVX_128bit, "");
6850   assert(VM_Version::supports_avx512dq(), "");
6851   assert(dst != xnoreg, "sanity");
6852   InstructionMark im(this);
6853   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6854   attributes.set_rex_vex_w_reverted();
6855   attributes.set_address_attributes(/* tuple_type */ EVEX_T2, /* input_size_in_bits */ EVEX_64bit);
6856   // swap src<->dst for encoding
6857   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6858   emit_int8(0x5A);
6859   emit_operand(dst, src);
6860 }
6861 
6862 // scalar single/double precision replicate
6863 
6864 // duplicate single precision data from src into programmed locations in dest : requires AVX512VL
6865 void Assembler::evpbroadcastss(XMMRegister dst, XMMRegister src, int vector_len) {
6866   assert(VM_Version::supports_evex(), "");
6867   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6868   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6869   emit_int8(0x18);
6870   emit_int8((unsigned char)(0xC0 | encode));
6871 }
6872 
6873 void Assembler::evpbroadcastss(XMMRegister dst, Address src, int vector_len) {
6874   assert(VM_Version::supports_evex(), "");
6875   assert(dst != xnoreg, "sanity");
6876   InstructionMark im(this);
6877   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6878   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
6879   // swap src<->dst for encoding
6880   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6881   emit_int8(0x18);
6882   emit_operand(dst, src);
6883 }
6884 
6885 // duplicate double precision data from src into programmed locations in dest : requires AVX512VL
6886 void Assembler::evpbroadcastsd(XMMRegister dst, XMMRegister src, int vector_len) {
6887   assert(VM_Version::supports_evex(), "");
6888   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6889   attributes.set_rex_vex_w_reverted();
6890   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6891   emit_int8(0x19);
6892   emit_int8((unsigned char)(0xC0 | encode));
6893 }
6894 
6895 void Assembler::evpbroadcastsd(XMMRegister dst, Address src, int vector_len) {
6896   assert(VM_Version::supports_evex(), "");
6897   assert(dst != xnoreg, "sanity");
6898   InstructionMark im(this);
6899   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6900   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
6901   attributes.set_rex_vex_w_reverted();
6902   // swap src<->dst for encoding
6903   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6904   emit_int8(0x19);
6905   emit_operand(dst, src);
6906 }
6907 
6908 
6909 // gpr source broadcast forms
6910 
6911 // duplicate 1-byte integer data from src into programmed locations in dest : requires AVX512BW and AVX512VL
6912 void Assembler::evpbroadcastb(XMMRegister dst, Register src, int vector_len) {
6913   assert(VM_Version::supports_evex(), "");
6914   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
6915   attributes.set_is_evex_instruction();
6916   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6917   emit_int8(0x7A);
6918   emit_int8((unsigned char)(0xC0 | encode));
6919 }
6920 
6921 // duplicate 2-byte integer data from src into programmed locations in dest : requires AVX512BW and AVX512VL
6922 void Assembler::evpbroadcastw(XMMRegister dst, Register src, int vector_len) {
6923   assert(VM_Version::supports_evex(), "");
6924   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
6925   attributes.set_is_evex_instruction();
6926   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6927   emit_int8(0x7B);
6928   emit_int8((unsigned char)(0xC0 | encode));
6929 }
6930 
6931 // duplicate 4-byte integer data from src into programmed locations in dest : requires AVX512VL
6932 void Assembler::evpbroadcastd(XMMRegister dst, Register src, int vector_len) {
6933   assert(VM_Version::supports_evex(), "");
6934   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6935   attributes.set_is_evex_instruction();
6936   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6937   emit_int8(0x7C);
6938   emit_int8((unsigned char)(0xC0 | encode));
6939 }
6940 
6941 // duplicate 8-byte integer data from src into programmed locations in dest : requires AVX512VL
6942 void Assembler::evpbroadcastq(XMMRegister dst, Register src, int vector_len) {
6943   assert(VM_Version::supports_evex(), "");
6944   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6945   attributes.set_is_evex_instruction();
6946   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6947   emit_int8(0x7C);
6948   emit_int8((unsigned char)(0xC0 | encode));
6949 }
6950 
6951 void Assembler::evpgatherdd(XMMRegister dst, KRegister mask, Address src, int vector_len) {
6952   assert(VM_Version::supports_evex(), "");
6953   assert(dst != xnoreg, "sanity");
6954   InstructionMark im(this);
6955   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6956   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
6957   attributes.reset_is_clear_context();
6958   attributes.set_embedded_opmask_register_specifier(mask);
6959   attributes.set_is_evex_instruction();
6960   // swap src<->dst for encoding
6961   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6962   emit_int8((unsigned char)0x90);
6963   emit_operand(dst, src);
6964 }
6965 
6966 // Carry-Less Multiplication Quadword
6967 void Assembler::pclmulqdq(XMMRegister dst, XMMRegister src, int mask) {
6968   assert(VM_Version::supports_clmul(), "");
6969   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
6970   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6971   emit_int8(0x44);
6972   emit_int8((unsigned char)(0xC0 | encode));
6973   emit_int8((unsigned char)mask);
6974 }
6975 
6976 // Carry-Less Multiplication Quadword
6977 void Assembler::vpclmulqdq(XMMRegister dst, XMMRegister nds, XMMRegister src, int mask) {
6978   assert(VM_Version::supports_avx() && VM_Version::supports_clmul(), "");
6979   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
6980   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6981   emit_int8(0x44);
6982   emit_int8((unsigned char)(0xC0 | encode));
6983   emit_int8((unsigned char)mask);
6984 }
6985 
6986 void Assembler::evpclmulqdq(XMMRegister dst, XMMRegister nds, XMMRegister src, int mask, int vector_len) {
6987   assert(VM_Version::supports_vpclmulqdq(), "Requires vector carryless multiplication support");
6988   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6989   attributes.set_is_evex_instruction();
6990   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6991   emit_int8(0x44);
6992   emit_int8((unsigned char)(0xC0 | encode));
6993   emit_int8((unsigned char)mask);
6994 }
6995 
6996 void Assembler::vzeroupper() {
6997   if (VM_Version::supports_vzeroupper()) {
6998     InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
6999     (void)vex_prefix_and_encode(0, 0, 0, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
7000     emit_int8(0x77);
7001   }
7002 }
7003 
7004 #ifndef _LP64
7005 // 32bit only pieces of the assembler
7006 
7007 void Assembler::cmp_literal32(Register src1, int32_t imm32, RelocationHolder const& rspec) {
7008   // NO PREFIX AS NEVER 64BIT
7009   InstructionMark im(this);
7010   emit_int8((unsigned char)0x81);
7011   emit_int8((unsigned char)(0xF8 | src1->encoding()));
7012   emit_data(imm32, rspec, 0);
7013 }
7014 
7015 void Assembler::cmp_literal32(Address src1, int32_t imm32, RelocationHolder const& rspec) {
7016   // NO PREFIX AS NEVER 64BIT (not even 32bit versions of 64bit regs
7017   InstructionMark im(this);
7018   emit_int8((unsigned char)0x81);
7019   emit_operand(rdi, src1);
7020   emit_data(imm32, rspec, 0);
7021 }
7022 
7023 // The 64-bit (32bit platform) cmpxchg compares the value at adr with the contents of rdx:rax,
7024 // and stores rcx:rbx into adr if so; otherwise, the value at adr is loaded
7025 // into rdx:rax.  The ZF is set if the compared values were equal, and cleared otherwise.
7026 void Assembler::cmpxchg8(Address adr) {
7027   InstructionMark im(this);
7028   emit_int8(0x0F);
7029   emit_int8((unsigned char)0xC7);
7030   emit_operand(rcx, adr);
7031 }
7032 
7033 void Assembler::decl(Register dst) {
7034   // Don't use it directly. Use MacroAssembler::decrementl() instead.
7035  emit_int8(0x48 | dst->encoding());
7036 }
7037 
7038 #endif // _LP64
7039 
7040 // 64bit typically doesn't use the x87 but needs to for the trig funcs
7041 
7042 void Assembler::fabs() {
7043   emit_int8((unsigned char)0xD9);
7044   emit_int8((unsigned char)0xE1);
7045 }
7046 
7047 void Assembler::fadd(int i) {
7048   emit_farith(0xD8, 0xC0, i);
7049 }
7050 
7051 void Assembler::fadd_d(Address src) {
7052   InstructionMark im(this);
7053   emit_int8((unsigned char)0xDC);
7054   emit_operand32(rax, src);
7055 }
7056 
7057 void Assembler::fadd_s(Address src) {
7058   InstructionMark im(this);
7059   emit_int8((unsigned char)0xD8);
7060   emit_operand32(rax, src);
7061 }
7062 
7063 void Assembler::fadda(int i) {
7064   emit_farith(0xDC, 0xC0, i);
7065 }
7066 
7067 void Assembler::faddp(int i) {
7068   emit_farith(0xDE, 0xC0, i);
7069 }
7070 
7071 void Assembler::fchs() {
7072   emit_int8((unsigned char)0xD9);
7073   emit_int8((unsigned char)0xE0);
7074 }
7075 
7076 void Assembler::fcom(int i) {
7077   emit_farith(0xD8, 0xD0, i);
7078 }
7079 
7080 void Assembler::fcomp(int i) {
7081   emit_farith(0xD8, 0xD8, i);
7082 }
7083 
7084 void Assembler::fcomp_d(Address src) {
7085   InstructionMark im(this);
7086   emit_int8((unsigned char)0xDC);
7087   emit_operand32(rbx, src);
7088 }
7089 
7090 void Assembler::fcomp_s(Address src) {
7091   InstructionMark im(this);
7092   emit_int8((unsigned char)0xD8);
7093   emit_operand32(rbx, src);
7094 }
7095 
7096 void Assembler::fcompp() {
7097   emit_int8((unsigned char)0xDE);
7098   emit_int8((unsigned char)0xD9);
7099 }
7100 
7101 void Assembler::fcos() {
7102   emit_int8((unsigned char)0xD9);
7103   emit_int8((unsigned char)0xFF);
7104 }
7105 
7106 void Assembler::fdecstp() {
7107   emit_int8((unsigned char)0xD9);
7108   emit_int8((unsigned char)0xF6);
7109 }
7110 
7111 void Assembler::fdiv(int i) {
7112   emit_farith(0xD8, 0xF0, i);
7113 }
7114 
7115 void Assembler::fdiv_d(Address src) {
7116   InstructionMark im(this);
7117   emit_int8((unsigned char)0xDC);
7118   emit_operand32(rsi, src);
7119 }
7120 
7121 void Assembler::fdiv_s(Address src) {
7122   InstructionMark im(this);
7123   emit_int8((unsigned char)0xD8);
7124   emit_operand32(rsi, src);
7125 }
7126 
7127 void Assembler::fdiva(int i) {
7128   emit_farith(0xDC, 0xF8, i);
7129 }
7130 
7131 // Note: The Intel manual (Pentium Processor User's Manual, Vol.3, 1994)
7132 //       is erroneous for some of the floating-point instructions below.
7133 
7134 void Assembler::fdivp(int i) {
7135   emit_farith(0xDE, 0xF8, i);                    // ST(0) <- ST(0) / ST(1) and pop (Intel manual wrong)
7136 }
7137 
7138 void Assembler::fdivr(int i) {
7139   emit_farith(0xD8, 0xF8, i);
7140 }
7141 
7142 void Assembler::fdivr_d(Address src) {
7143   InstructionMark im(this);
7144   emit_int8((unsigned char)0xDC);
7145   emit_operand32(rdi, src);
7146 }
7147 
7148 void Assembler::fdivr_s(Address src) {
7149   InstructionMark im(this);
7150   emit_int8((unsigned char)0xD8);
7151   emit_operand32(rdi, src);
7152 }
7153 
7154 void Assembler::fdivra(int i) {
7155   emit_farith(0xDC, 0xF0, i);
7156 }
7157 
7158 void Assembler::fdivrp(int i) {
7159   emit_farith(0xDE, 0xF0, i);                    // ST(0) <- ST(1) / ST(0) and pop (Intel manual wrong)
7160 }
7161 
7162 void Assembler::ffree(int i) {
7163   emit_farith(0xDD, 0xC0, i);
7164 }
7165 
7166 void Assembler::fild_d(Address adr) {
7167   InstructionMark im(this);
7168   emit_int8((unsigned char)0xDF);
7169   emit_operand32(rbp, adr);
7170 }
7171 
7172 void Assembler::fild_s(Address adr) {
7173   InstructionMark im(this);
7174   emit_int8((unsigned char)0xDB);
7175   emit_operand32(rax, adr);
7176 }
7177 
7178 void Assembler::fincstp() {
7179   emit_int8((unsigned char)0xD9);
7180   emit_int8((unsigned char)0xF7);
7181 }
7182 
7183 void Assembler::finit() {
7184   emit_int8((unsigned char)0x9B);
7185   emit_int8((unsigned char)0xDB);
7186   emit_int8((unsigned char)0xE3);
7187 }
7188 
7189 void Assembler::fist_s(Address adr) {
7190   InstructionMark im(this);
7191   emit_int8((unsigned char)0xDB);
7192   emit_operand32(rdx, adr);
7193 }
7194 
7195 void Assembler::fistp_d(Address adr) {
7196   InstructionMark im(this);
7197   emit_int8((unsigned char)0xDF);
7198   emit_operand32(rdi, adr);
7199 }
7200 
7201 void Assembler::fistp_s(Address adr) {
7202   InstructionMark im(this);
7203   emit_int8((unsigned char)0xDB);
7204   emit_operand32(rbx, adr);
7205 }
7206 
7207 void Assembler::fld1() {
7208   emit_int8((unsigned char)0xD9);
7209   emit_int8((unsigned char)0xE8);
7210 }
7211 
7212 void Assembler::fld_d(Address adr) {
7213   InstructionMark im(this);
7214   emit_int8((unsigned char)0xDD);
7215   emit_operand32(rax, adr);
7216 }
7217 
7218 void Assembler::fld_s(Address adr) {
7219   InstructionMark im(this);
7220   emit_int8((unsigned char)0xD9);
7221   emit_operand32(rax, adr);
7222 }
7223 
7224 
7225 void Assembler::fld_s(int index) {
7226   emit_farith(0xD9, 0xC0, index);
7227 }
7228 
7229 void Assembler::fld_x(Address adr) {
7230   InstructionMark im(this);
7231   emit_int8((unsigned char)0xDB);
7232   emit_operand32(rbp, adr);
7233 }
7234 
7235 void Assembler::fldcw(Address src) {
7236   InstructionMark im(this);
7237   emit_int8((unsigned char)0xD9);
7238   emit_operand32(rbp, src);
7239 }
7240 
7241 void Assembler::fldenv(Address src) {
7242   InstructionMark im(this);
7243   emit_int8((unsigned char)0xD9);
7244   emit_operand32(rsp, src);
7245 }
7246 
7247 void Assembler::fldlg2() {
7248   emit_int8((unsigned char)0xD9);
7249   emit_int8((unsigned char)0xEC);
7250 }
7251 
7252 void Assembler::fldln2() {
7253   emit_int8((unsigned char)0xD9);
7254   emit_int8((unsigned char)0xED);
7255 }
7256 
7257 void Assembler::fldz() {
7258   emit_int8((unsigned char)0xD9);
7259   emit_int8((unsigned char)0xEE);
7260 }
7261 
7262 void Assembler::flog() {
7263   fldln2();
7264   fxch();
7265   fyl2x();
7266 }
7267 
7268 void Assembler::flog10() {
7269   fldlg2();
7270   fxch();
7271   fyl2x();
7272 }
7273 
7274 void Assembler::fmul(int i) {
7275   emit_farith(0xD8, 0xC8, i);
7276 }
7277 
7278 void Assembler::fmul_d(Address src) {
7279   InstructionMark im(this);
7280   emit_int8((unsigned char)0xDC);
7281   emit_operand32(rcx, src);
7282 }
7283 
7284 void Assembler::fmul_s(Address src) {
7285   InstructionMark im(this);
7286   emit_int8((unsigned char)0xD8);
7287   emit_operand32(rcx, src);
7288 }
7289 
7290 void Assembler::fmula(int i) {
7291   emit_farith(0xDC, 0xC8, i);
7292 }
7293 
7294 void Assembler::fmulp(int i) {
7295   emit_farith(0xDE, 0xC8, i);
7296 }
7297 
7298 void Assembler::fnsave(Address dst) {
7299   InstructionMark im(this);
7300   emit_int8((unsigned char)0xDD);
7301   emit_operand32(rsi, dst);
7302 }
7303 
7304 void Assembler::fnstcw(Address src) {
7305   InstructionMark im(this);
7306   emit_int8((unsigned char)0x9B);
7307   emit_int8((unsigned char)0xD9);
7308   emit_operand32(rdi, src);
7309 }
7310 
7311 void Assembler::fnstsw_ax() {
7312   emit_int8((unsigned char)0xDF);
7313   emit_int8((unsigned char)0xE0);
7314 }
7315 
7316 void Assembler::fprem() {
7317   emit_int8((unsigned char)0xD9);
7318   emit_int8((unsigned char)0xF8);
7319 }
7320 
7321 void Assembler::fprem1() {
7322   emit_int8((unsigned char)0xD9);
7323   emit_int8((unsigned char)0xF5);
7324 }
7325 
7326 void Assembler::frstor(Address src) {
7327   InstructionMark im(this);
7328   emit_int8((unsigned char)0xDD);
7329   emit_operand32(rsp, src);
7330 }
7331 
7332 void Assembler::fsin() {
7333   emit_int8((unsigned char)0xD9);
7334   emit_int8((unsigned char)0xFE);
7335 }
7336 
7337 void Assembler::fsqrt() {
7338   emit_int8((unsigned char)0xD9);
7339   emit_int8((unsigned char)0xFA);
7340 }
7341 
7342 void Assembler::fst_d(Address adr) {
7343   InstructionMark im(this);
7344   emit_int8((unsigned char)0xDD);
7345   emit_operand32(rdx, adr);
7346 }
7347 
7348 void Assembler::fst_s(Address adr) {
7349   InstructionMark im(this);
7350   emit_int8((unsigned char)0xD9);
7351   emit_operand32(rdx, adr);
7352 }
7353 
7354 void Assembler::fstp_d(Address adr) {
7355   InstructionMark im(this);
7356   emit_int8((unsigned char)0xDD);
7357   emit_operand32(rbx, adr);
7358 }
7359 
7360 void Assembler::fstp_d(int index) {
7361   emit_farith(0xDD, 0xD8, index);
7362 }
7363 
7364 void Assembler::fstp_s(Address adr) {
7365   InstructionMark im(this);
7366   emit_int8((unsigned char)0xD9);
7367   emit_operand32(rbx, adr);
7368 }
7369 
7370 void Assembler::fstp_x(Address adr) {
7371   InstructionMark im(this);
7372   emit_int8((unsigned char)0xDB);
7373   emit_operand32(rdi, adr);
7374 }
7375 
7376 void Assembler::fsub(int i) {
7377   emit_farith(0xD8, 0xE0, i);
7378 }
7379 
7380 void Assembler::fsub_d(Address src) {
7381   InstructionMark im(this);
7382   emit_int8((unsigned char)0xDC);
7383   emit_operand32(rsp, src);
7384 }
7385 
7386 void Assembler::fsub_s(Address src) {
7387   InstructionMark im(this);
7388   emit_int8((unsigned char)0xD8);
7389   emit_operand32(rsp, src);
7390 }
7391 
7392 void Assembler::fsuba(int i) {
7393   emit_farith(0xDC, 0xE8, i);
7394 }
7395 
7396 void Assembler::fsubp(int i) {
7397   emit_farith(0xDE, 0xE8, i);                    // ST(0) <- ST(0) - ST(1) and pop (Intel manual wrong)
7398 }
7399 
7400 void Assembler::fsubr(int i) {
7401   emit_farith(0xD8, 0xE8, i);
7402 }
7403 
7404 void Assembler::fsubr_d(Address src) {
7405   InstructionMark im(this);
7406   emit_int8((unsigned char)0xDC);
7407   emit_operand32(rbp, src);
7408 }
7409 
7410 void Assembler::fsubr_s(Address src) {
7411   InstructionMark im(this);
7412   emit_int8((unsigned char)0xD8);
7413   emit_operand32(rbp, src);
7414 }
7415 
7416 void Assembler::fsubra(int i) {
7417   emit_farith(0xDC, 0xE0, i);
7418 }
7419 
7420 void Assembler::fsubrp(int i) {
7421   emit_farith(0xDE, 0xE0, i);                    // ST(0) <- ST(1) - ST(0) and pop (Intel manual wrong)
7422 }
7423 
7424 void Assembler::ftan() {
7425   emit_int8((unsigned char)0xD9);
7426   emit_int8((unsigned char)0xF2);
7427   emit_int8((unsigned char)0xDD);
7428   emit_int8((unsigned char)0xD8);
7429 }
7430 
7431 void Assembler::ftst() {
7432   emit_int8((unsigned char)0xD9);
7433   emit_int8((unsigned char)0xE4);
7434 }
7435 
7436 void Assembler::fucomi(int i) {
7437   // make sure the instruction is supported (introduced for P6, together with cmov)
7438   guarantee(VM_Version::supports_cmov(), "illegal instruction");
7439   emit_farith(0xDB, 0xE8, i);
7440 }
7441 
7442 void Assembler::fucomip(int i) {
7443   // make sure the instruction is supported (introduced for P6, together with cmov)
7444   guarantee(VM_Version::supports_cmov(), "illegal instruction");
7445   emit_farith(0xDF, 0xE8, i);
7446 }
7447 
7448 void Assembler::fwait() {
7449   emit_int8((unsigned char)0x9B);
7450 }
7451 
7452 void Assembler::fxch(int i) {
7453   emit_farith(0xD9, 0xC8, i);
7454 }
7455 
7456 void Assembler::fyl2x() {
7457   emit_int8((unsigned char)0xD9);
7458   emit_int8((unsigned char)0xF1);
7459 }
7460 
7461 void Assembler::frndint() {
7462   emit_int8((unsigned char)0xD9);
7463   emit_int8((unsigned char)0xFC);
7464 }
7465 
7466 void Assembler::f2xm1() {
7467   emit_int8((unsigned char)0xD9);
7468   emit_int8((unsigned char)0xF0);
7469 }
7470 
7471 void Assembler::fldl2e() {
7472   emit_int8((unsigned char)0xD9);
7473   emit_int8((unsigned char)0xEA);
7474 }
7475 
7476 // SSE SIMD prefix byte values corresponding to VexSimdPrefix encoding.
7477 static int simd_pre[4] = { 0, 0x66, 0xF3, 0xF2 };
7478 // SSE opcode second byte values (first is 0x0F) corresponding to VexOpcode encoding.
7479 static int simd_opc[4] = { 0,    0, 0x38, 0x3A };
7480 
7481 // Generate SSE legacy REX prefix and SIMD opcode based on VEX encoding.
7482 void Assembler::rex_prefix(Address adr, XMMRegister xreg, VexSimdPrefix pre, VexOpcode opc, bool rex_w) {
7483   if (pre > 0) {
7484     emit_int8(simd_pre[pre]);
7485   }
7486   if (rex_w) {
7487     prefixq(adr, xreg);
7488   } else {
7489     prefix(adr, xreg);
7490   }
7491   if (opc > 0) {
7492     emit_int8(0x0F);
7493     int opc2 = simd_opc[opc];
7494     if (opc2 > 0) {
7495       emit_int8(opc2);
7496     }
7497   }
7498 }
7499 
7500 int Assembler::rex_prefix_and_encode(int dst_enc, int src_enc, VexSimdPrefix pre, VexOpcode opc, bool rex_w) {
7501   if (pre > 0) {
7502     emit_int8(simd_pre[pre]);
7503   }
7504   int encode = (rex_w) ? prefixq_and_encode(dst_enc, src_enc) : prefix_and_encode(dst_enc, src_enc);
7505   if (opc > 0) {
7506     emit_int8(0x0F);
7507     int opc2 = simd_opc[opc];
7508     if (opc2 > 0) {
7509       emit_int8(opc2);
7510     }
7511   }
7512   return encode;
7513 }
7514 
7515 
7516 void Assembler::vex_prefix(bool vex_r, bool vex_b, bool vex_x, int nds_enc, VexSimdPrefix pre, VexOpcode opc) {
7517   int vector_len = _attributes->get_vector_len();
7518   bool vex_w = _attributes->is_rex_vex_w();
7519   if (vex_b || vex_x || vex_w || (opc == VEX_OPCODE_0F_38) || (opc == VEX_OPCODE_0F_3A)) {
7520     prefix(VEX_3bytes);
7521 
7522     int byte1 = (vex_r ? VEX_R : 0) | (vex_x ? VEX_X : 0) | (vex_b ? VEX_B : 0);
7523     byte1 = (~byte1) & 0xE0;
7524     byte1 |= opc;
7525     emit_int8(byte1);
7526 
7527     int byte2 = ((~nds_enc) & 0xf) << 3;
7528     byte2 |= (vex_w ? VEX_W : 0) | ((vector_len > 0) ? 4 : 0) | pre;
7529     emit_int8(byte2);
7530   } else {
7531     prefix(VEX_2bytes);
7532 
7533     int byte1 = vex_r ? VEX_R : 0;
7534     byte1 = (~byte1) & 0x80;
7535     byte1 |= ((~nds_enc) & 0xf) << 3;
7536     byte1 |= ((vector_len > 0 ) ? 4 : 0) | pre;
7537     emit_int8(byte1);
7538   }
7539 }
7540 
7541 // This is a 4 byte encoding
7542 void Assembler::evex_prefix(bool vex_r, bool vex_b, bool vex_x, bool evex_r, bool evex_v, int nds_enc, VexSimdPrefix pre, VexOpcode opc){
7543   // EVEX 0x62 prefix
7544   prefix(EVEX_4bytes);
7545   bool vex_w = _attributes->is_rex_vex_w();
7546   int evex_encoding = (vex_w ? VEX_W : 0);
7547   // EVEX.b is not currently used for broadcast of single element or data rounding modes
7548   _attributes->set_evex_encoding(evex_encoding);
7549 
7550   // P0: byte 2, initialized to RXBR`00mm
7551   // instead of not'd
7552   int byte2 = (vex_r ? VEX_R : 0) | (vex_x ? VEX_X : 0) | (vex_b ? VEX_B : 0) | (evex_r ? EVEX_Rb : 0);
7553   byte2 = (~byte2) & 0xF0;
7554   // confine opc opcode extensions in mm bits to lower two bits
7555   // of form {0F, 0F_38, 0F_3A}
7556   byte2 |= opc;
7557   emit_int8(byte2);
7558 
7559   // P1: byte 3 as Wvvvv1pp
7560   int byte3 = ((~nds_enc) & 0xf) << 3;
7561   // p[10] is always 1
7562   byte3 |= EVEX_F;
7563   byte3 |= (vex_w & 1) << 7;
7564   // confine pre opcode extensions in pp bits to lower two bits
7565   // of form {66, F3, F2}
7566   byte3 |= pre;
7567   emit_int8(byte3);
7568 
7569   // P2: byte 4 as zL'Lbv'aaa
7570   // kregs are implemented in the low 3 bits as aaa (hard code k1, it will be initialized for now)
7571   int byte4 = (_attributes->is_no_reg_mask()) ?
7572               0 :
7573               _attributes->get_embedded_opmask_register_specifier();
7574   // EVEX.v` for extending EVEX.vvvv or VIDX
7575   byte4 |= (evex_v ? 0: EVEX_V);
7576   // third EXEC.b for broadcast actions
7577   byte4 |= (_attributes->is_extended_context() ? EVEX_Rb : 0);
7578   // fourth EVEX.L'L for vector length : 0 is 128, 1 is 256, 2 is 512, currently we do not support 1024
7579   byte4 |= ((_attributes->get_vector_len())& 0x3) << 5;
7580   // last is EVEX.z for zero/merge actions
7581   if (_attributes->is_no_reg_mask() == false) {
7582     byte4 |= (_attributes->is_clear_context() ? EVEX_Z : 0);
7583   }
7584   emit_int8(byte4);
7585 }
7586 
7587 void Assembler::vex_prefix(Address adr, int nds_enc, int xreg_enc, VexSimdPrefix pre, VexOpcode opc, InstructionAttr *attributes) {
7588   bool vex_r = ((xreg_enc & 8) == 8) ? 1 : 0;
7589   bool vex_b = adr.base_needs_rex();
7590   bool vex_x;
7591   if (adr.isxmmindex()) {
7592     vex_x = adr.xmmindex_needs_rex();
7593   } else {
7594     vex_x = adr.index_needs_rex();
7595   }
7596   set_attributes(attributes);
7597   attributes->set_current_assembler(this);
7598 
7599   // if vector length is turned off, revert to AVX for vectors smaller than 512-bit
7600   if (UseAVX > 2 && _legacy_mode_vl && attributes->uses_vl()) {
7601     switch (attributes->get_vector_len()) {
7602     case AVX_128bit:
7603     case AVX_256bit:
7604       attributes->set_is_legacy_mode();
7605       break;
7606     }
7607   }
7608 
7609   // For pure EVEX check and see if this instruction
7610   // is allowed in legacy mode and has resources which will
7611   // fit in it.  Pure EVEX instructions will use set_is_evex_instruction in their definition,
7612   // else that field is set when we encode to EVEX
7613   if (UseAVX > 2 && !attributes->is_legacy_mode() &&
7614       !_is_managed && !attributes->is_evex_instruction()) {
7615     if (!_legacy_mode_vl && attributes->get_vector_len() != AVX_512bit) {
7616       bool check_register_bank = NOT_IA32(true) IA32_ONLY(false);
7617       if (check_register_bank) {
7618         // check nds_enc and xreg_enc for upper bank usage
7619         if (nds_enc < 16 && xreg_enc < 16) {
7620           attributes->set_is_legacy_mode();
7621         }
7622       } else {
7623         attributes->set_is_legacy_mode();
7624       }
7625     }
7626   }
7627 
7628   _is_managed = false;
7629   if (UseAVX > 2 && !attributes->is_legacy_mode())
7630   {
7631     bool evex_r = (xreg_enc >= 16);
7632     bool evex_v;
7633     // EVEX.V' is set to true when VSIB is used as we may need to use higher order XMM registers (16-31)
7634     if (adr.isxmmindex())  {
7635       evex_v = ((adr._xmmindex->encoding() > 15) ? true : false);
7636     } else {
7637       evex_v = (nds_enc >= 16);
7638     }
7639     attributes->set_is_evex_instruction();
7640     evex_prefix(vex_r, vex_b, vex_x, evex_r, evex_v, nds_enc, pre, opc);
7641   } else {
7642     if (UseAVX > 2 && attributes->is_rex_vex_w_reverted()) {
7643       attributes->set_rex_vex_w(false);
7644     }
7645     vex_prefix(vex_r, vex_b, vex_x, nds_enc, pre, opc);
7646   }
7647 }
7648 
7649 int Assembler::vex_prefix_and_encode(int dst_enc, int nds_enc, int src_enc, VexSimdPrefix pre, VexOpcode opc, InstructionAttr *attributes) {
7650   bool vex_r = ((dst_enc & 8) == 8) ? 1 : 0;
7651   bool vex_b = ((src_enc & 8) == 8) ? 1 : 0;
7652   bool vex_x = false;
7653   set_attributes(attributes);
7654   attributes->set_current_assembler(this);
7655   bool check_register_bank = NOT_IA32(true) IA32_ONLY(false);
7656 
7657   // if vector length is turned off, revert to AVX for vectors smaller than 512-bit
7658   if (UseAVX > 2 && _legacy_mode_vl && attributes->uses_vl()) {
7659     switch (attributes->get_vector_len()) {
7660     case AVX_128bit:
7661     case AVX_256bit:
7662       if (check_register_bank) {
7663         if (dst_enc >= 16 || nds_enc >= 16 || src_enc >= 16) {
7664           // up propagate arithmetic instructions to meet RA requirements
7665           attributes->set_vector_len(AVX_512bit);
7666         } else {
7667           attributes->set_is_legacy_mode();
7668         }
7669       } else {
7670         attributes->set_is_legacy_mode();
7671       }
7672       break;
7673     }
7674   }
7675 
7676   // For pure EVEX check and see if this instruction
7677   // is allowed in legacy mode and has resources which will
7678   // fit in it.  Pure EVEX instructions will use set_is_evex_instruction in their definition,
7679   // else that field is set when we encode to EVEX
7680   if (UseAVX > 2 && !attributes->is_legacy_mode() &&
7681       !_is_managed && !attributes->is_evex_instruction()) {
7682     if (!_legacy_mode_vl && attributes->get_vector_len() != AVX_512bit) {
7683       if (check_register_bank) {
7684         // check dst_enc, nds_enc and src_enc for upper bank usage
7685         if (dst_enc < 16 && nds_enc < 16 && src_enc < 16) {
7686           attributes->set_is_legacy_mode();
7687         }
7688       } else {
7689         attributes->set_is_legacy_mode();
7690       }
7691     }
7692   }
7693 
7694   _is_managed = false;
7695   if (UseAVX > 2 && !attributes->is_legacy_mode())
7696   {
7697     bool evex_r = (dst_enc >= 16);
7698     bool evex_v = (nds_enc >= 16);
7699     // can use vex_x as bank extender on rm encoding
7700     vex_x = (src_enc >= 16);
7701     attributes->set_is_evex_instruction();
7702     evex_prefix(vex_r, vex_b, vex_x, evex_r, evex_v, nds_enc, pre, opc);
7703   } else {
7704     if (UseAVX > 2 && attributes->is_rex_vex_w_reverted()) {
7705       attributes->set_rex_vex_w(false);
7706     }
7707     vex_prefix(vex_r, vex_b, vex_x, nds_enc, pre, opc);
7708   }
7709 
7710   // return modrm byte components for operands
7711   return (((dst_enc & 7) << 3) | (src_enc & 7));
7712 }
7713 
7714 
7715 void Assembler::simd_prefix(XMMRegister xreg, XMMRegister nds, Address adr, VexSimdPrefix pre,
7716                             VexOpcode opc, InstructionAttr *attributes) {
7717   if (UseAVX > 0) {
7718     int xreg_enc = xreg->encoding();
7719     int nds_enc = nds->is_valid() ? nds->encoding() : 0;
7720     vex_prefix(adr, nds_enc, xreg_enc, pre, opc, attributes);
7721   } else {
7722     assert((nds == xreg) || (nds == xnoreg), "wrong sse encoding");
7723     rex_prefix(adr, xreg, pre, opc, attributes->is_rex_vex_w());
7724   }
7725 }
7726 
7727 int Assembler::simd_prefix_and_encode(XMMRegister dst, XMMRegister nds, XMMRegister src, VexSimdPrefix pre,
7728                                       VexOpcode opc, InstructionAttr *attributes) {
7729   int dst_enc = dst->encoding();
7730   int src_enc = src->encoding();
7731   if (UseAVX > 0) {
7732     int nds_enc = nds->is_valid() ? nds->encoding() : 0;
7733     return vex_prefix_and_encode(dst_enc, nds_enc, src_enc, pre, opc, attributes);
7734   } else {
7735     assert((nds == dst) || (nds == src) || (nds == xnoreg), "wrong sse encoding");
7736     return rex_prefix_and_encode(dst_enc, src_enc, pre, opc, attributes->is_rex_vex_w());
7737   }
7738 }
7739 
7740 void Assembler::cmppd(XMMRegister dst, XMMRegister nds, XMMRegister src, int cop, int vector_len) {
7741   assert(VM_Version::supports_avx(), "");
7742   assert(!VM_Version::supports_evex(), "");
7743   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
7744   int encode = simd_prefix_and_encode(dst, nds, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7745   emit_int8((unsigned char)0xC2);
7746   emit_int8((unsigned char)(0xC0 | encode));
7747   emit_int8((unsigned char)(0xF & cop));
7748 }
7749 
7750 void Assembler::blendvpd(XMMRegister dst, XMMRegister nds, XMMRegister src1, XMMRegister src2, int vector_len) {
7751   assert(VM_Version::supports_avx(), "");
7752   assert(!VM_Version::supports_evex(), "");
7753   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
7754   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src1->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
7755   emit_int8((unsigned char)0x4B);
7756   emit_int8((unsigned char)(0xC0 | encode));
7757   int src2_enc = src2->encoding();
7758   emit_int8((unsigned char)(0xF0 & src2_enc<<4));
7759 }
7760 
7761 void Assembler::cmpps(XMMRegister dst, XMMRegister nds, XMMRegister src, int cop, int vector_len) {
7762   assert(VM_Version::supports_avx(), "");
7763   assert(!VM_Version::supports_evex(), "");
7764   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
7765   int encode = simd_prefix_and_encode(dst, nds, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
7766   emit_int8((unsigned char)0xC2);
7767   emit_int8((unsigned char)(0xC0 | encode));
7768   emit_int8((unsigned char)(0xF & cop));
7769 }
7770 
7771 void Assembler::blendvps(XMMRegister dst, XMMRegister nds, XMMRegister src1, XMMRegister src2, int vector_len) {
7772   assert(VM_Version::supports_avx(), "");
7773   assert(!VM_Version::supports_evex(), "");
7774   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
7775   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src1->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
7776   emit_int8((unsigned char)0x4A);
7777   emit_int8((unsigned char)(0xC0 | encode));
7778   int src2_enc = src2->encoding();
7779   emit_int8((unsigned char)(0xF0 & src2_enc<<4));
7780 }
7781 
7782 void Assembler::vpblendd(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8, int vector_len) {
7783   assert(VM_Version::supports_avx2(), "");
7784   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
7785   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
7786   emit_int8((unsigned char)0x02);
7787   emit_int8((unsigned char)(0xC0 | encode));
7788   emit_int8((unsigned char)imm8);
7789 }
7790 
7791 void Assembler::shlxl(Register dst, Register src1, Register src2) {
7792   assert(VM_Version::supports_bmi2(), "");
7793   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
7794   int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src1->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
7795   emit_int8((unsigned char)0xF7);
7796   emit_int8((unsigned char)(0xC0 | encode));
7797 }
7798 
7799 void Assembler::shlxq(Register dst, Register src1, Register src2) {
7800   assert(VM_Version::supports_bmi2(), "");
7801   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
7802   int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src1->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
7803   emit_int8((unsigned char)0xF7);
7804   emit_int8((unsigned char)(0xC0 | encode));
7805 }
7806 
7807 #ifndef _LP64
7808 
7809 void Assembler::incl(Register dst) {
7810   // Don't use it directly. Use MacroAssembler::incrementl() instead.
7811   emit_int8(0x40 | dst->encoding());
7812 }
7813 
7814 void Assembler::lea(Register dst, Address src) {
7815   leal(dst, src);
7816 }
7817 
7818 void Assembler::mov_literal32(Address dst, int32_t imm32, RelocationHolder const& rspec) {
7819   InstructionMark im(this);
7820   emit_int8((unsigned char)0xC7);
7821   emit_operand(rax, dst);
7822   emit_data((int)imm32, rspec, 0);
7823 }
7824 
7825 void Assembler::mov_literal32(Register dst, int32_t imm32, RelocationHolder const& rspec) {
7826   InstructionMark im(this);
7827   int encode = prefix_and_encode(dst->encoding());
7828   emit_int8((unsigned char)(0xB8 | encode));
7829   emit_data((int)imm32, rspec, 0);
7830 }
7831 
7832 void Assembler::popa() { // 32bit
7833   emit_int8(0x61);
7834 }
7835 
7836 void Assembler::push_literal32(int32_t imm32, RelocationHolder const& rspec) {
7837   InstructionMark im(this);
7838   emit_int8(0x68);
7839   emit_data(imm32, rspec, 0);
7840 }
7841 
7842 void Assembler::pusha() { // 32bit
7843   emit_int8(0x60);
7844 }
7845 
7846 void Assembler::set_byte_if_not_zero(Register dst) {
7847   emit_int8(0x0F);
7848   emit_int8((unsigned char)0x95);
7849   emit_int8((unsigned char)(0xE0 | dst->encoding()));
7850 }
7851 
7852 void Assembler::shldl(Register dst, Register src) {
7853   emit_int8(0x0F);
7854   emit_int8((unsigned char)0xA5);
7855   emit_int8((unsigned char)(0xC0 | src->encoding() << 3 | dst->encoding()));
7856 }
7857 
7858 // 0F A4 / r ib
7859 void Assembler::shldl(Register dst, Register src, int8_t imm8) {
7860   emit_int8(0x0F);
7861   emit_int8((unsigned char)0xA4);
7862   emit_int8((unsigned char)(0xC0 | src->encoding() << 3 | dst->encoding()));
7863   emit_int8(imm8);
7864 }
7865 
7866 void Assembler::shrdl(Register dst, Register src) {
7867   emit_int8(0x0F);
7868   emit_int8((unsigned char)0xAD);
7869   emit_int8((unsigned char)(0xC0 | src->encoding() << 3 | dst->encoding()));
7870 }
7871 
7872 #else // LP64
7873 
7874 void Assembler::set_byte_if_not_zero(Register dst) {
7875   int enc = prefix_and_encode(dst->encoding(), true);
7876   emit_int8(0x0F);
7877   emit_int8((unsigned char)0x95);
7878   emit_int8((unsigned char)(0xE0 | enc));
7879 }
7880 
7881 // 64bit only pieces of the assembler
7882 // This should only be used by 64bit instructions that can use rip-relative
7883 // it cannot be used by instructions that want an immediate value.
7884 
7885 bool Assembler::reachable(AddressLiteral adr) {
7886   int64_t disp;
7887   // None will force a 64bit literal to the code stream. Likely a placeholder
7888   // for something that will be patched later and we need to certain it will
7889   // always be reachable.
7890   if (adr.reloc() == relocInfo::none) {
7891     return false;
7892   }
7893   if (adr.reloc() == relocInfo::internal_word_type) {
7894     // This should be rip relative and easily reachable.
7895     return true;
7896   }
7897   if (adr.reloc() == relocInfo::virtual_call_type ||
7898       adr.reloc() == relocInfo::opt_virtual_call_type ||
7899       adr.reloc() == relocInfo::static_call_type ||
7900       adr.reloc() == relocInfo::static_stub_type ) {
7901     // This should be rip relative within the code cache and easily
7902     // reachable until we get huge code caches. (At which point
7903     // ic code is going to have issues).
7904     return true;
7905   }
7906   if (adr.reloc() != relocInfo::external_word_type &&
7907       adr.reloc() != relocInfo::poll_return_type &&  // these are really external_word but need special
7908       adr.reloc() != relocInfo::poll_type &&         // relocs to identify them
7909       adr.reloc() != relocInfo::runtime_call_type ) {
7910     return false;
7911   }
7912 
7913   // Stress the correction code
7914   if (ForceUnreachable) {
7915     // Must be runtimecall reloc, see if it is in the codecache
7916     // Flipping stuff in the codecache to be unreachable causes issues
7917     // with things like inline caches where the additional instructions
7918     // are not handled.
7919     if (CodeCache::find_blob(adr._target) == NULL) {
7920       return false;
7921     }
7922   }
7923   // For external_word_type/runtime_call_type if it is reachable from where we
7924   // are now (possibly a temp buffer) and where we might end up
7925   // anywhere in the codeCache then we are always reachable.
7926   // This would have to change if we ever save/restore shared code
7927   // to be more pessimistic.
7928   disp = (int64_t)adr._target - ((int64_t)CodeCache::low_bound() + sizeof(int));
7929   if (!is_simm32(disp)) return false;
7930   disp = (int64_t)adr._target - ((int64_t)CodeCache::high_bound() + sizeof(int));
7931   if (!is_simm32(disp)) return false;
7932 
7933   disp = (int64_t)adr._target - ((int64_t)pc() + sizeof(int));
7934 
7935   // Because rip relative is a disp + address_of_next_instruction and we
7936   // don't know the value of address_of_next_instruction we apply a fudge factor
7937   // to make sure we will be ok no matter the size of the instruction we get placed into.
7938   // We don't have to fudge the checks above here because they are already worst case.
7939 
7940   // 12 == override/rex byte, opcode byte, rm byte, sib byte, a 4-byte disp , 4-byte literal
7941   // + 4 because better safe than sorry.
7942   const int fudge = 12 + 4;
7943   if (disp < 0) {
7944     disp -= fudge;
7945   } else {
7946     disp += fudge;
7947   }
7948   return is_simm32(disp);
7949 }
7950 
7951 // Check if the polling page is not reachable from the code cache using rip-relative
7952 // addressing.
7953 bool Assembler::is_polling_page_far() {
7954   intptr_t addr = (intptr_t)os::get_polling_page();
7955   return ForceUnreachable ||
7956          !is_simm32(addr - (intptr_t)CodeCache::low_bound()) ||
7957          !is_simm32(addr - (intptr_t)CodeCache::high_bound());
7958 }
7959 
7960 void Assembler::emit_data64(jlong data,
7961                             relocInfo::relocType rtype,
7962                             int format) {
7963   if (rtype == relocInfo::none) {
7964     emit_int64(data);
7965   } else {
7966     emit_data64(data, Relocation::spec_simple(rtype), format);
7967   }
7968 }
7969 
7970 void Assembler::emit_data64(jlong data,
7971                             RelocationHolder const& rspec,
7972                             int format) {
7973   assert(imm_operand == 0, "default format must be immediate in this file");
7974   assert(imm_operand == format, "must be immediate");
7975   assert(inst_mark() != NULL, "must be inside InstructionMark");
7976   // Do not use AbstractAssembler::relocate, which is not intended for
7977   // embedded words.  Instead, relocate to the enclosing instruction.
7978   code_section()->relocate(inst_mark(), rspec, format);
7979 #ifdef ASSERT
7980   check_relocation(rspec, format);
7981 #endif
7982   emit_int64(data);
7983 }
7984 
7985 int Assembler::prefix_and_encode(int reg_enc, bool byteinst) {
7986   if (reg_enc >= 8) {
7987     prefix(REX_B);
7988     reg_enc -= 8;
7989   } else if (byteinst && reg_enc >= 4) {
7990     prefix(REX);
7991   }
7992   return reg_enc;
7993 }
7994 
7995 int Assembler::prefixq_and_encode(int reg_enc) {
7996   if (reg_enc < 8) {
7997     prefix(REX_W);
7998   } else {
7999     prefix(REX_WB);
8000     reg_enc -= 8;
8001   }
8002   return reg_enc;
8003 }
8004 
8005 int Assembler::prefix_and_encode(int dst_enc, bool dst_is_byte, int src_enc, bool src_is_byte) {
8006   if (dst_enc < 8) {
8007     if (src_enc >= 8) {
8008       prefix(REX_B);
8009       src_enc -= 8;
8010     } else if ((src_is_byte && src_enc >= 4) || (dst_is_byte && dst_enc >= 4)) {
8011       prefix(REX);
8012     }
8013   } else {
8014     if (src_enc < 8) {
8015       prefix(REX_R);
8016     } else {
8017       prefix(REX_RB);
8018       src_enc -= 8;
8019     }
8020     dst_enc -= 8;
8021   }
8022   return dst_enc << 3 | src_enc;
8023 }
8024 
8025 int Assembler::prefixq_and_encode(int dst_enc, int src_enc) {
8026   if (dst_enc < 8) {
8027     if (src_enc < 8) {
8028       prefix(REX_W);
8029     } else {
8030       prefix(REX_WB);
8031       src_enc -= 8;
8032     }
8033   } else {
8034     if (src_enc < 8) {
8035       prefix(REX_WR);
8036     } else {
8037       prefix(REX_WRB);
8038       src_enc -= 8;
8039     }
8040     dst_enc -= 8;
8041   }
8042   return dst_enc << 3 | src_enc;
8043 }
8044 
8045 void Assembler::prefix(Register reg) {
8046   if (reg->encoding() >= 8) {
8047     prefix(REX_B);
8048   }
8049 }
8050 
8051 void Assembler::prefix(Register dst, Register src, Prefix p) {
8052   if (src->encoding() >= 8) {
8053     p = (Prefix)(p | REX_B);
8054   }
8055   if (dst->encoding() >= 8) {
8056     p = (Prefix)( p | REX_R);
8057   }
8058   if (p != Prefix_EMPTY) {
8059     // do not generate an empty prefix
8060     prefix(p);
8061   }
8062 }
8063 
8064 void Assembler::prefix(Register dst, Address adr, Prefix p) {
8065   if (adr.base_needs_rex()) {
8066     if (adr.index_needs_rex()) {
8067       assert(false, "prefix(Register dst, Address adr, Prefix p) does not support handling of an X");
8068     } else {
8069       prefix(REX_B);
8070     }
8071   } else {
8072     if (adr.index_needs_rex()) {
8073       assert(false, "prefix(Register dst, Address adr, Prefix p) does not support handling of an X");
8074     }
8075   }
8076   if (dst->encoding() >= 8) {
8077     p = (Prefix)(p | REX_R);
8078   }
8079   if (p != Prefix_EMPTY) {
8080     // do not generate an empty prefix
8081     prefix(p);
8082   }
8083 }
8084 
8085 void Assembler::prefix(Address adr) {
8086   if (adr.base_needs_rex()) {
8087     if (adr.index_needs_rex()) {
8088       prefix(REX_XB);
8089     } else {
8090       prefix(REX_B);
8091     }
8092   } else {
8093     if (adr.index_needs_rex()) {
8094       prefix(REX_X);
8095     }
8096   }
8097 }
8098 
8099 void Assembler::prefixq(Address adr) {
8100   if (adr.base_needs_rex()) {
8101     if (adr.index_needs_rex()) {
8102       prefix(REX_WXB);
8103     } else {
8104       prefix(REX_WB);
8105     }
8106   } else {
8107     if (adr.index_needs_rex()) {
8108       prefix(REX_WX);
8109     } else {
8110       prefix(REX_W);
8111     }
8112   }
8113 }
8114 
8115 
8116 void Assembler::prefix(Address adr, Register reg, bool byteinst) {
8117   if (reg->encoding() < 8) {
8118     if (adr.base_needs_rex()) {
8119       if (adr.index_needs_rex()) {
8120         prefix(REX_XB);
8121       } else {
8122         prefix(REX_B);
8123       }
8124     } else {
8125       if (adr.index_needs_rex()) {
8126         prefix(REX_X);
8127       } else if (byteinst && reg->encoding() >= 4 ) {
8128         prefix(REX);
8129       }
8130     }
8131   } else {
8132     if (adr.base_needs_rex()) {
8133       if (adr.index_needs_rex()) {
8134         prefix(REX_RXB);
8135       } else {
8136         prefix(REX_RB);
8137       }
8138     } else {
8139       if (adr.index_needs_rex()) {
8140         prefix(REX_RX);
8141       } else {
8142         prefix(REX_R);
8143       }
8144     }
8145   }
8146 }
8147 
8148 void Assembler::prefixq(Address adr, Register src) {
8149   if (src->encoding() < 8) {
8150     if (adr.base_needs_rex()) {
8151       if (adr.index_needs_rex()) {
8152         prefix(REX_WXB);
8153       } else {
8154         prefix(REX_WB);
8155       }
8156     } else {
8157       if (adr.index_needs_rex()) {
8158         prefix(REX_WX);
8159       } else {
8160         prefix(REX_W);
8161       }
8162     }
8163   } else {
8164     if (adr.base_needs_rex()) {
8165       if (adr.index_needs_rex()) {
8166         prefix(REX_WRXB);
8167       } else {
8168         prefix(REX_WRB);
8169       }
8170     } else {
8171       if (adr.index_needs_rex()) {
8172         prefix(REX_WRX);
8173       } else {
8174         prefix(REX_WR);
8175       }
8176     }
8177   }
8178 }
8179 
8180 void Assembler::prefix(Address adr, XMMRegister reg) {
8181   if (reg->encoding() < 8) {
8182     if (adr.base_needs_rex()) {
8183       if (adr.index_needs_rex()) {
8184         prefix(REX_XB);
8185       } else {
8186         prefix(REX_B);
8187       }
8188     } else {
8189       if (adr.index_needs_rex()) {
8190         prefix(REX_X);
8191       }
8192     }
8193   } else {
8194     if (adr.base_needs_rex()) {
8195       if (adr.index_needs_rex()) {
8196         prefix(REX_RXB);
8197       } else {
8198         prefix(REX_RB);
8199       }
8200     } else {
8201       if (adr.index_needs_rex()) {
8202         prefix(REX_RX);
8203       } else {
8204         prefix(REX_R);
8205       }
8206     }
8207   }
8208 }
8209 
8210 void Assembler::prefixq(Address adr, XMMRegister src) {
8211   if (src->encoding() < 8) {
8212     if (adr.base_needs_rex()) {
8213       if (adr.index_needs_rex()) {
8214         prefix(REX_WXB);
8215       } else {
8216         prefix(REX_WB);
8217       }
8218     } else {
8219       if (adr.index_needs_rex()) {
8220         prefix(REX_WX);
8221       } else {
8222         prefix(REX_W);
8223       }
8224     }
8225   } else {
8226     if (adr.base_needs_rex()) {
8227       if (adr.index_needs_rex()) {
8228         prefix(REX_WRXB);
8229       } else {
8230         prefix(REX_WRB);
8231       }
8232     } else {
8233       if (adr.index_needs_rex()) {
8234         prefix(REX_WRX);
8235       } else {
8236         prefix(REX_WR);
8237       }
8238     }
8239   }
8240 }
8241 
8242 void Assembler::adcq(Register dst, int32_t imm32) {
8243   (void) prefixq_and_encode(dst->encoding());
8244   emit_arith(0x81, 0xD0, dst, imm32);
8245 }
8246 
8247 void Assembler::adcq(Register dst, Address src) {
8248   InstructionMark im(this);
8249   prefixq(src, dst);
8250   emit_int8(0x13);
8251   emit_operand(dst, src);
8252 }
8253 
8254 void Assembler::adcq(Register dst, Register src) {
8255   (void) prefixq_and_encode(dst->encoding(), src->encoding());
8256   emit_arith(0x13, 0xC0, dst, src);
8257 }
8258 
8259 void Assembler::addq(Address dst, int32_t imm32) {
8260   InstructionMark im(this);
8261   prefixq(dst);
8262   emit_arith_operand(0x81, rax, dst,imm32);
8263 }
8264 
8265 void Assembler::addq(Address dst, Register src) {
8266   InstructionMark im(this);
8267   prefixq(dst, src);
8268   emit_int8(0x01);
8269   emit_operand(src, dst);
8270 }
8271 
8272 void Assembler::addq(Register dst, int32_t imm32) {
8273   (void) prefixq_and_encode(dst->encoding());
8274   emit_arith(0x81, 0xC0, dst, imm32);
8275 }
8276 
8277 void Assembler::addq(Register dst, Address src) {
8278   InstructionMark im(this);
8279   prefixq(src, dst);
8280   emit_int8(0x03);
8281   emit_operand(dst, src);
8282 }
8283 
8284 void Assembler::addq(Register dst, Register src) {
8285   (void) prefixq_and_encode(dst->encoding(), src->encoding());
8286   emit_arith(0x03, 0xC0, dst, src);
8287 }
8288 
8289 void Assembler::adcxq(Register dst, Register src) {
8290   //assert(VM_Version::supports_adx(), "adx instructions not supported");
8291   emit_int8((unsigned char)0x66);
8292   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
8293   emit_int8(0x0F);
8294   emit_int8(0x38);
8295   emit_int8((unsigned char)0xF6);
8296   emit_int8((unsigned char)(0xC0 | encode));
8297 }
8298 
8299 void Assembler::adoxq(Register dst, Register src) {
8300   //assert(VM_Version::supports_adx(), "adx instructions not supported");
8301   emit_int8((unsigned char)0xF3);
8302   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
8303   emit_int8(0x0F);
8304   emit_int8(0x38);
8305   emit_int8((unsigned char)0xF6);
8306   emit_int8((unsigned char)(0xC0 | encode));
8307 }
8308 
8309 void Assembler::andq(Address dst, int32_t imm32) {
8310   InstructionMark im(this);
8311   prefixq(dst);
8312   emit_int8((unsigned char)0x81);
8313   emit_operand(rsp, dst, 4);
8314   emit_int32(imm32);
8315 }
8316 
8317 void Assembler::andq(Register dst, int32_t imm32) {
8318   (void) prefixq_and_encode(dst->encoding());
8319   emit_arith(0x81, 0xE0, dst, imm32);
8320 }
8321 
8322 void Assembler::andq(Register dst, Address src) {
8323   InstructionMark im(this);
8324   prefixq(src, dst);
8325   emit_int8(0x23);
8326   emit_operand(dst, src);
8327 }
8328 
8329 void Assembler::andq(Register dst, Register src) {
8330   (void) prefixq_and_encode(dst->encoding(), src->encoding());
8331   emit_arith(0x23, 0xC0, dst, src);
8332 }
8333 
8334 void Assembler::andnq(Register dst, Register src1, Register src2) {
8335   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
8336   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
8337   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
8338   emit_int8((unsigned char)0xF2);
8339   emit_int8((unsigned char)(0xC0 | encode));
8340 }
8341 
8342 void Assembler::andnq(Register dst, Register src1, Address src2) {
8343   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
8344   InstructionMark im(this);
8345   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
8346   vex_prefix(src2, src1->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
8347   emit_int8((unsigned char)0xF2);
8348   emit_operand(dst, src2);
8349 }
8350 
8351 void Assembler::bsfq(Register dst, Register src) {
8352   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
8353   emit_int8(0x0F);
8354   emit_int8((unsigned char)0xBC);
8355   emit_int8((unsigned char)(0xC0 | encode));
8356 }
8357 
8358 void Assembler::bsrq(Register dst, Register src) {
8359   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
8360   emit_int8(0x0F);
8361   emit_int8((unsigned char)0xBD);
8362   emit_int8((unsigned char)(0xC0 | encode));
8363 }
8364 
8365 void Assembler::bswapq(Register reg) {
8366   int encode = prefixq_and_encode(reg->encoding());
8367   emit_int8(0x0F);
8368   emit_int8((unsigned char)(0xC8 | encode));
8369 }
8370 
8371 void Assembler::blsiq(Register dst, Register src) {
8372   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
8373   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
8374   int encode = vex_prefix_and_encode(rbx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
8375   emit_int8((unsigned char)0xF3);
8376   emit_int8((unsigned char)(0xC0 | encode));
8377 }
8378 
8379 void Assembler::blsiq(Register dst, Address src) {
8380   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
8381   InstructionMark im(this);
8382   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
8383   vex_prefix(src, dst->encoding(), rbx->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
8384   emit_int8((unsigned char)0xF3);
8385   emit_operand(rbx, src);
8386 }
8387 
8388 void Assembler::blsmskq(Register dst, Register src) {
8389   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
8390   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
8391   int encode = vex_prefix_and_encode(rdx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
8392   emit_int8((unsigned char)0xF3);
8393   emit_int8((unsigned char)(0xC0 | encode));
8394 }
8395 
8396 void Assembler::blsmskq(Register dst, Address src) {
8397   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
8398   InstructionMark im(this);
8399   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
8400   vex_prefix(src, dst->encoding(), rdx->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
8401   emit_int8((unsigned char)0xF3);
8402   emit_operand(rdx, src);
8403 }
8404 
8405 void Assembler::blsrq(Register dst, Register src) {
8406   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
8407   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
8408   int encode = vex_prefix_and_encode(rcx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
8409   emit_int8((unsigned char)0xF3);
8410   emit_int8((unsigned char)(0xC0 | encode));
8411 }
8412 
8413 void Assembler::blsrq(Register dst, Address src) {
8414   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
8415   InstructionMark im(this);
8416   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
8417   vex_prefix(src, dst->encoding(), rcx->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
8418   emit_int8((unsigned char)0xF3);
8419   emit_operand(rcx, src);
8420 }
8421 
8422 void Assembler::cdqq() {
8423   prefix(REX_W);
8424   emit_int8((unsigned char)0x99);
8425 }
8426 
8427 void Assembler::clflush(Address adr) {
8428   prefix(adr);
8429   emit_int8(0x0F);
8430   emit_int8((unsigned char)0xAE);
8431   emit_operand(rdi, adr);
8432 }
8433 
8434 void Assembler::cmovq(Condition cc, Register dst, Register src) {
8435   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
8436   emit_int8(0x0F);
8437   emit_int8(0x40 | cc);
8438   emit_int8((unsigned char)(0xC0 | encode));
8439 }
8440 
8441 void Assembler::cmovq(Condition cc, Register dst, Address src) {
8442   InstructionMark im(this);
8443   prefixq(src, dst);
8444   emit_int8(0x0F);
8445   emit_int8(0x40 | cc);
8446   emit_operand(dst, src);
8447 }
8448 
8449 void Assembler::cmpq(Address dst, int32_t imm32) {
8450   InstructionMark im(this);
8451   prefixq(dst);
8452   emit_int8((unsigned char)0x81);
8453   emit_operand(rdi, dst, 4);
8454   emit_int32(imm32);
8455 }
8456 
8457 void Assembler::cmpq(Register dst, int32_t imm32) {
8458   (void) prefixq_and_encode(dst->encoding());
8459   emit_arith(0x81, 0xF8, dst, imm32);
8460 }
8461 
8462 void Assembler::cmpq(Address dst, Register src) {
8463   InstructionMark im(this);
8464   prefixq(dst, src);
8465   emit_int8(0x3B);
8466   emit_operand(src, dst);
8467 }
8468 
8469 void Assembler::cmpq(Register dst, Register src) {
8470   (void) prefixq_and_encode(dst->encoding(), src->encoding());
8471   emit_arith(0x3B, 0xC0, dst, src);
8472 }
8473 
8474 void Assembler::cmpq(Register dst, Address  src) {
8475   InstructionMark im(this);
8476   prefixq(src, dst);
8477   emit_int8(0x3B);
8478   emit_operand(dst, src);
8479 }
8480 
8481 void Assembler::cmpxchgq(Register reg, Address adr) {
8482   InstructionMark im(this);
8483   prefixq(adr, reg);
8484   emit_int8(0x0F);
8485   emit_int8((unsigned char)0xB1);
8486   emit_operand(reg, adr);
8487 }
8488 
8489 void Assembler::cvtsi2sdq(XMMRegister dst, Register src) {
8490   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
8491   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
8492   int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
8493   emit_int8(0x2A);
8494   emit_int8((unsigned char)(0xC0 | encode));
8495 }
8496 
8497 void Assembler::cvtsi2sdq(XMMRegister dst, Address src) {
8498   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
8499   InstructionMark im(this);
8500   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
8501   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
8502   simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
8503   emit_int8(0x2A);
8504   emit_operand(dst, src);
8505 }
8506 
8507 void Assembler::cvtsi2ssq(XMMRegister dst, Address src) {
8508   NOT_LP64(assert(VM_Version::supports_sse(), ""));
8509   InstructionMark im(this);
8510   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
8511   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
8512   simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
8513   emit_int8(0x2A);
8514   emit_operand(dst, src);
8515 }
8516 
8517 void Assembler::cvttsd2siq(Register dst, XMMRegister src) {
8518   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
8519   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
8520   int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
8521   emit_int8(0x2C);
8522   emit_int8((unsigned char)(0xC0 | encode));
8523 }
8524 
8525 void Assembler::cvttss2siq(Register dst, XMMRegister src) {
8526   NOT_LP64(assert(VM_Version::supports_sse(), ""));
8527   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
8528   int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
8529   emit_int8(0x2C);
8530   emit_int8((unsigned char)(0xC0 | encode));
8531 }
8532 
8533 void Assembler::decl(Register dst) {
8534   // Don't use it directly. Use MacroAssembler::decrementl() instead.
8535   // Use two-byte form (one-byte form is a REX prefix in 64-bit mode)
8536   int encode = prefix_and_encode(dst->encoding());
8537   emit_int8((unsigned char)0xFF);
8538   emit_int8((unsigned char)(0xC8 | encode));
8539 }
8540 
8541 void Assembler::decq(Register dst) {
8542   // Don't use it directly. Use MacroAssembler::decrementq() instead.
8543   // Use two-byte form (one-byte from is a REX prefix in 64-bit mode)
8544   int encode = prefixq_and_encode(dst->encoding());
8545   emit_int8((unsigned char)0xFF);
8546   emit_int8(0xC8 | encode);
8547 }
8548 
8549 void Assembler::decq(Address dst) {
8550   // Don't use it directly. Use MacroAssembler::decrementq() instead.
8551   InstructionMark im(this);
8552   prefixq(dst);
8553   emit_int8((unsigned char)0xFF);
8554   emit_operand(rcx, dst);
8555 }
8556 
8557 void Assembler::fxrstor(Address src) {
8558   prefixq(src);
8559   emit_int8(0x0F);
8560   emit_int8((unsigned char)0xAE);
8561   emit_operand(as_Register(1), src);
8562 }
8563 
8564 void Assembler::xrstor(Address src) {
8565   prefixq(src);
8566   emit_int8(0x0F);
8567   emit_int8((unsigned char)0xAE);
8568   emit_operand(as_Register(5), src);
8569 }
8570 
8571 void Assembler::fxsave(Address dst) {
8572   prefixq(dst);
8573   emit_int8(0x0F);
8574   emit_int8((unsigned char)0xAE);
8575   emit_operand(as_Register(0), dst);
8576 }
8577 
8578 void Assembler::xsave(Address dst) {
8579   prefixq(dst);
8580   emit_int8(0x0F);
8581   emit_int8((unsigned char)0xAE);
8582   emit_operand(as_Register(4), dst);
8583 }
8584 
8585 void Assembler::idivq(Register src) {
8586   int encode = prefixq_and_encode(src->encoding());
8587   emit_int8((unsigned char)0xF7);
8588   emit_int8((unsigned char)(0xF8 | encode));
8589 }
8590 
8591 void Assembler::imulq(Register dst, Register src) {
8592   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
8593   emit_int8(0x0F);
8594   emit_int8((unsigned char)0xAF);
8595   emit_int8((unsigned char)(0xC0 | encode));
8596 }
8597 
8598 void Assembler::imulq(Register dst, Register src, int value) {
8599   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
8600   if (is8bit(value)) {
8601     emit_int8(0x6B);
8602     emit_int8((unsigned char)(0xC0 | encode));
8603     emit_int8(value & 0xFF);
8604   } else {
8605     emit_int8(0x69);
8606     emit_int8((unsigned char)(0xC0 | encode));
8607     emit_int32(value);
8608   }
8609 }
8610 
8611 void Assembler::imulq(Register dst, Address src) {
8612   InstructionMark im(this);
8613   prefixq(src, dst);
8614   emit_int8(0x0F);
8615   emit_int8((unsigned char) 0xAF);
8616   emit_operand(dst, src);
8617 }
8618 
8619 void Assembler::incl(Register dst) {
8620   // Don't use it directly. Use MacroAssembler::incrementl() instead.
8621   // Use two-byte form (one-byte from is a REX prefix in 64-bit mode)
8622   int encode = prefix_and_encode(dst->encoding());
8623   emit_int8((unsigned char)0xFF);
8624   emit_int8((unsigned char)(0xC0 | encode));
8625 }
8626 
8627 void Assembler::incq(Register dst) {
8628   // Don't use it directly. Use MacroAssembler::incrementq() instead.
8629   // Use two-byte form (one-byte from is a REX prefix in 64-bit mode)
8630   int encode = prefixq_and_encode(dst->encoding());
8631   emit_int8((unsigned char)0xFF);
8632   emit_int8((unsigned char)(0xC0 | encode));
8633 }
8634 
8635 void Assembler::incq(Address dst) {
8636   // Don't use it directly. Use MacroAssembler::incrementq() instead.
8637   InstructionMark im(this);
8638   prefixq(dst);
8639   emit_int8((unsigned char)0xFF);
8640   emit_operand(rax, dst);
8641 }
8642 
8643 void Assembler::lea(Register dst, Address src) {
8644   leaq(dst, src);
8645 }
8646 
8647 void Assembler::leaq(Register dst, Address src) {
8648   InstructionMark im(this);
8649   prefixq(src, dst);
8650   emit_int8((unsigned char)0x8D);
8651   emit_operand(dst, src);
8652 }
8653 
8654 void Assembler::mov64(Register dst, int64_t imm64) {
8655   InstructionMark im(this);
8656   int encode = prefixq_and_encode(dst->encoding());
8657   emit_int8((unsigned char)(0xB8 | encode));
8658   emit_int64(imm64);
8659 }
8660 
8661 void Assembler::mov_literal64(Register dst, intptr_t imm64, RelocationHolder const& rspec) {
8662   InstructionMark im(this);
8663   int encode = prefixq_and_encode(dst->encoding());
8664   emit_int8(0xB8 | encode);
8665   emit_data64(imm64, rspec);
8666 }
8667 
8668 void Assembler::mov_narrow_oop(Register dst, int32_t imm32, RelocationHolder const& rspec) {
8669   InstructionMark im(this);
8670   int encode = prefix_and_encode(dst->encoding());
8671   emit_int8((unsigned char)(0xB8 | encode));
8672   emit_data((int)imm32, rspec, narrow_oop_operand);
8673 }
8674 
8675 void Assembler::mov_narrow_oop(Address dst, int32_t imm32,  RelocationHolder const& rspec) {
8676   InstructionMark im(this);
8677   prefix(dst);
8678   emit_int8((unsigned char)0xC7);
8679   emit_operand(rax, dst, 4);
8680   emit_data((int)imm32, rspec, narrow_oop_operand);
8681 }
8682 
8683 void Assembler::cmp_narrow_oop(Register src1, int32_t imm32, RelocationHolder const& rspec) {
8684   InstructionMark im(this);
8685   int encode = prefix_and_encode(src1->encoding());
8686   emit_int8((unsigned char)0x81);
8687   emit_int8((unsigned char)(0xF8 | encode));
8688   emit_data((int)imm32, rspec, narrow_oop_operand);
8689 }
8690 
8691 void Assembler::cmp_narrow_oop(Address src1, int32_t imm32, RelocationHolder const& rspec) {
8692   InstructionMark im(this);
8693   prefix(src1);
8694   emit_int8((unsigned char)0x81);
8695   emit_operand(rax, src1, 4);
8696   emit_data((int)imm32, rspec, narrow_oop_operand);
8697 }
8698 
8699 void Assembler::lzcntq(Register dst, Register src) {
8700   assert(VM_Version::supports_lzcnt(), "encoding is treated as BSR");
8701   emit_int8((unsigned char)0xF3);
8702   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
8703   emit_int8(0x0F);
8704   emit_int8((unsigned char)0xBD);
8705   emit_int8((unsigned char)(0xC0 | encode));
8706 }
8707 
8708 void Assembler::movdq(XMMRegister dst, Register src) {
8709   // table D-1 says MMX/SSE2
8710   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
8711   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
8712   int encode = simd_prefix_and_encode(dst, xnoreg, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
8713   emit_int8(0x6E);
8714   emit_int8((unsigned char)(0xC0 | encode));
8715 }
8716 
8717 void Assembler::movdq(Register dst, XMMRegister src) {
8718   // table D-1 says MMX/SSE2
8719   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
8720   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
8721   // swap src/dst to get correct prefix
8722   int encode = simd_prefix_and_encode(src, xnoreg, as_XMMRegister(dst->encoding()), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
8723   emit_int8(0x7E);
8724   emit_int8((unsigned char)(0xC0 | encode));
8725 }
8726 
8727 void Assembler::movq(Register dst, Register src) {
8728   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
8729   emit_int8((unsigned char)0x8B);
8730   emit_int8((unsigned char)(0xC0 | encode));
8731 }
8732 
8733 void Assembler::movq(Register dst, Address src) {
8734   InstructionMark im(this);
8735   prefixq(src, dst);
8736   emit_int8((unsigned char)0x8B);
8737   emit_operand(dst, src);
8738 }
8739 
8740 void Assembler::movq(Address dst, Register src) {
8741   InstructionMark im(this);
8742   prefixq(dst, src);
8743   emit_int8((unsigned char)0x89);
8744   emit_operand(src, dst);
8745 }
8746 
8747 void Assembler::movsbq(Register dst, Address src) {
8748   InstructionMark im(this);
8749   prefixq(src, dst);
8750   emit_int8(0x0F);
8751   emit_int8((unsigned char)0xBE);
8752   emit_operand(dst, src);
8753 }
8754 
8755 void Assembler::movsbq(Register dst, Register src) {
8756   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
8757   emit_int8(0x0F);
8758   emit_int8((unsigned char)0xBE);
8759   emit_int8((unsigned char)(0xC0 | encode));
8760 }
8761 
8762 void Assembler::movslq(Register dst, int32_t imm32) {
8763   // dbx shows movslq(rcx, 3) as movq     $0x0000000049000000,(%rbx)
8764   // and movslq(r8, 3); as movl     $0x0000000048000000,(%rbx)
8765   // as a result we shouldn't use until tested at runtime...
8766   ShouldNotReachHere();
8767   InstructionMark im(this);
8768   int encode = prefixq_and_encode(dst->encoding());
8769   emit_int8((unsigned char)(0xC7 | encode));
8770   emit_int32(imm32);
8771 }
8772 
8773 void Assembler::movslq(Address dst, int32_t imm32) {
8774   assert(is_simm32(imm32), "lost bits");
8775   InstructionMark im(this);
8776   prefixq(dst);
8777   emit_int8((unsigned char)0xC7);
8778   emit_operand(rax, dst, 4);
8779   emit_int32(imm32);
8780 }
8781 
8782 void Assembler::movslq(Register dst, Address src) {
8783   InstructionMark im(this);
8784   prefixq(src, dst);
8785   emit_int8(0x63);
8786   emit_operand(dst, src);
8787 }
8788 
8789 void Assembler::movslq(Register dst, Register src) {
8790   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
8791   emit_int8(0x63);
8792   emit_int8((unsigned char)(0xC0 | encode));
8793 }
8794 
8795 void Assembler::movswq(Register dst, Address src) {
8796   InstructionMark im(this);
8797   prefixq(src, dst);
8798   emit_int8(0x0F);
8799   emit_int8((unsigned char)0xBF);
8800   emit_operand(dst, src);
8801 }
8802 
8803 void Assembler::movswq(Register dst, Register src) {
8804   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
8805   emit_int8((unsigned char)0x0F);
8806   emit_int8((unsigned char)0xBF);
8807   emit_int8((unsigned char)(0xC0 | encode));
8808 }
8809 
8810 void Assembler::movzbq(Register dst, Address src) {
8811   InstructionMark im(this);
8812   prefixq(src, dst);
8813   emit_int8((unsigned char)0x0F);
8814   emit_int8((unsigned char)0xB6);
8815   emit_operand(dst, src);
8816 }
8817 
8818 void Assembler::movzbq(Register dst, Register src) {
8819   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
8820   emit_int8(0x0F);
8821   emit_int8((unsigned char)0xB6);
8822   emit_int8(0xC0 | encode);
8823 }
8824 
8825 void Assembler::movzwq(Register dst, Address src) {
8826   InstructionMark im(this);
8827   prefixq(src, dst);
8828   emit_int8((unsigned char)0x0F);
8829   emit_int8((unsigned char)0xB7);
8830   emit_operand(dst, src);
8831 }
8832 
8833 void Assembler::movzwq(Register dst, Register src) {
8834   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
8835   emit_int8((unsigned char)0x0F);
8836   emit_int8((unsigned char)0xB7);
8837   emit_int8((unsigned char)(0xC0 | encode));
8838 }
8839 
8840 void Assembler::mulq(Address src) {
8841   InstructionMark im(this);
8842   prefixq(src);
8843   emit_int8((unsigned char)0xF7);
8844   emit_operand(rsp, src);
8845 }
8846 
8847 void Assembler::mulq(Register src) {
8848   int encode = prefixq_and_encode(src->encoding());
8849   emit_int8((unsigned char)0xF7);
8850   emit_int8((unsigned char)(0xE0 | encode));
8851 }
8852 
8853 void Assembler::mulxq(Register dst1, Register dst2, Register src) {
8854   assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
8855   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
8856   int encode = vex_prefix_and_encode(dst1->encoding(), dst2->encoding(), src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_38, &attributes);
8857   emit_int8((unsigned char)0xF6);
8858   emit_int8((unsigned char)(0xC0 | encode));
8859 }
8860 
8861 void Assembler::negq(Register dst) {
8862   int encode = prefixq_and_encode(dst->encoding());
8863   emit_int8((unsigned char)0xF7);
8864   emit_int8((unsigned char)(0xD8 | encode));
8865 }
8866 
8867 void Assembler::notq(Register dst) {
8868   int encode = prefixq_and_encode(dst->encoding());
8869   emit_int8((unsigned char)0xF7);
8870   emit_int8((unsigned char)(0xD0 | encode));
8871 }
8872 
8873 void Assembler::orq(Address dst, int32_t imm32) {
8874   InstructionMark im(this);
8875   prefixq(dst);
8876   emit_int8((unsigned char)0x81);
8877   emit_operand(rcx, dst, 4);
8878   emit_int32(imm32);
8879 }
8880 
8881 void Assembler::orq(Register dst, int32_t imm32) {
8882   (void) prefixq_and_encode(dst->encoding());
8883   emit_arith(0x81, 0xC8, dst, imm32);
8884 }
8885 
8886 void Assembler::orq(Register dst, Address src) {
8887   InstructionMark im(this);
8888   prefixq(src, dst);
8889   emit_int8(0x0B);
8890   emit_operand(dst, src);
8891 }
8892 
8893 void Assembler::orq(Register dst, Register src) {
8894   (void) prefixq_and_encode(dst->encoding(), src->encoding());
8895   emit_arith(0x0B, 0xC0, dst, src);
8896 }
8897 
8898 void Assembler::popa() { // 64bit
8899   movq(r15, Address(rsp, 0));
8900   movq(r14, Address(rsp, wordSize));
8901   movq(r13, Address(rsp, 2 * wordSize));
8902   movq(r12, Address(rsp, 3 * wordSize));
8903   movq(r11, Address(rsp, 4 * wordSize));
8904   movq(r10, Address(rsp, 5 * wordSize));
8905   movq(r9,  Address(rsp, 6 * wordSize));
8906   movq(r8,  Address(rsp, 7 * wordSize));
8907   movq(rdi, Address(rsp, 8 * wordSize));
8908   movq(rsi, Address(rsp, 9 * wordSize));
8909   movq(rbp, Address(rsp, 10 * wordSize));
8910   // skip rsp
8911   movq(rbx, Address(rsp, 12 * wordSize));
8912   movq(rdx, Address(rsp, 13 * wordSize));
8913   movq(rcx, Address(rsp, 14 * wordSize));
8914   movq(rax, Address(rsp, 15 * wordSize));
8915 
8916   addq(rsp, 16 * wordSize);
8917 }
8918 
8919 void Assembler::popcntq(Register dst, Address src) {
8920   assert(VM_Version::supports_popcnt(), "must support");
8921   InstructionMark im(this);
8922   emit_int8((unsigned char)0xF3);
8923   prefixq(src, dst);
8924   emit_int8((unsigned char)0x0F);
8925   emit_int8((unsigned char)0xB8);
8926   emit_operand(dst, src);
8927 }
8928 
8929 void Assembler::popcntq(Register dst, Register src) {
8930   assert(VM_Version::supports_popcnt(), "must support");
8931   emit_int8((unsigned char)0xF3);
8932   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
8933   emit_int8((unsigned char)0x0F);
8934   emit_int8((unsigned char)0xB8);
8935   emit_int8((unsigned char)(0xC0 | encode));
8936 }
8937 
8938 void Assembler::popq(Address dst) {
8939   InstructionMark im(this);
8940   prefixq(dst);
8941   emit_int8((unsigned char)0x8F);
8942   emit_operand(rax, dst);
8943 }
8944 
8945 void Assembler::pusha() { // 64bit
8946   // we have to store original rsp.  ABI says that 128 bytes
8947   // below rsp are local scratch.
8948   movq(Address(rsp, -5 * wordSize), rsp);
8949 
8950   subq(rsp, 16 * wordSize);
8951 
8952   movq(Address(rsp, 15 * wordSize), rax);
8953   movq(Address(rsp, 14 * wordSize), rcx);
8954   movq(Address(rsp, 13 * wordSize), rdx);
8955   movq(Address(rsp, 12 * wordSize), rbx);
8956   // skip rsp
8957   movq(Address(rsp, 10 * wordSize), rbp);
8958   movq(Address(rsp, 9 * wordSize), rsi);
8959   movq(Address(rsp, 8 * wordSize), rdi);
8960   movq(Address(rsp, 7 * wordSize), r8);
8961   movq(Address(rsp, 6 * wordSize), r9);
8962   movq(Address(rsp, 5 * wordSize), r10);
8963   movq(Address(rsp, 4 * wordSize), r11);
8964   movq(Address(rsp, 3 * wordSize), r12);
8965   movq(Address(rsp, 2 * wordSize), r13);
8966   movq(Address(rsp, wordSize), r14);
8967   movq(Address(rsp, 0), r15);
8968 }
8969 
8970 void Assembler::pushq(Address src) {
8971   InstructionMark im(this);
8972   prefixq(src);
8973   emit_int8((unsigned char)0xFF);
8974   emit_operand(rsi, src);
8975 }
8976 
8977 void Assembler::rclq(Register dst, int imm8) {
8978   assert(isShiftCount(imm8 >> 1), "illegal shift count");
8979   int encode = prefixq_and_encode(dst->encoding());
8980   if (imm8 == 1) {
8981     emit_int8((unsigned char)0xD1);
8982     emit_int8((unsigned char)(0xD0 | encode));
8983   } else {
8984     emit_int8((unsigned char)0xC1);
8985     emit_int8((unsigned char)(0xD0 | encode));
8986     emit_int8(imm8);
8987   }
8988 }
8989 
8990 void Assembler::rcrq(Register dst, int imm8) {
8991   assert(isShiftCount(imm8 >> 1), "illegal shift count");
8992   int encode = prefixq_and_encode(dst->encoding());
8993   if (imm8 == 1) {
8994     emit_int8((unsigned char)0xD1);
8995     emit_int8((unsigned char)(0xD8 | encode));
8996   } else {
8997     emit_int8((unsigned char)0xC1);
8998     emit_int8((unsigned char)(0xD8 | encode));
8999     emit_int8(imm8);
9000   }
9001 }
9002 
9003 void Assembler::rorq(Register dst, int imm8) {
9004   assert(isShiftCount(imm8 >> 1), "illegal shift count");
9005   int encode = prefixq_and_encode(dst->encoding());
9006   if (imm8 == 1) {
9007     emit_int8((unsigned char)0xD1);
9008     emit_int8((unsigned char)(0xC8 | encode));
9009   } else {
9010     emit_int8((unsigned char)0xC1);
9011     emit_int8((unsigned char)(0xc8 | encode));
9012     emit_int8(imm8);
9013   }
9014 }
9015 
9016 void Assembler::rorxq(Register dst, Register src, int imm8) {
9017   assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
9018   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
9019   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_3A, &attributes);
9020   emit_int8((unsigned char)0xF0);
9021   emit_int8((unsigned char)(0xC0 | encode));
9022   emit_int8(imm8);
9023 }
9024 
9025 void Assembler::rorxd(Register dst, Register src, int imm8) {
9026   assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
9027   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
9028   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_3A, &attributes);
9029   emit_int8((unsigned char)0xF0);
9030   emit_int8((unsigned char)(0xC0 | encode));
9031   emit_int8(imm8);
9032 }
9033 
9034 void Assembler::sarq(Register dst, int imm8) {
9035   assert(isShiftCount(imm8 >> 1), "illegal shift count");
9036   int encode = prefixq_and_encode(dst->encoding());
9037   if (imm8 == 1) {
9038     emit_int8((unsigned char)0xD1);
9039     emit_int8((unsigned char)(0xF8 | encode));
9040   } else {
9041     emit_int8((unsigned char)0xC1);
9042     emit_int8((unsigned char)(0xF8 | encode));
9043     emit_int8(imm8);
9044   }
9045 }
9046 
9047 void Assembler::sarq(Register dst) {
9048   int encode = prefixq_and_encode(dst->encoding());
9049   emit_int8((unsigned char)0xD3);
9050   emit_int8((unsigned char)(0xF8 | encode));
9051 }
9052 
9053 void Assembler::sbbq(Address dst, int32_t imm32) {
9054   InstructionMark im(this);
9055   prefixq(dst);
9056   emit_arith_operand(0x81, rbx, dst, imm32);
9057 }
9058 
9059 void Assembler::sbbq(Register dst, int32_t imm32) {
9060   (void) prefixq_and_encode(dst->encoding());
9061   emit_arith(0x81, 0xD8, dst, imm32);
9062 }
9063 
9064 void Assembler::sbbq(Register dst, Address src) {
9065   InstructionMark im(this);
9066   prefixq(src, dst);
9067   emit_int8(0x1B);
9068   emit_operand(dst, src);
9069 }
9070 
9071 void Assembler::sbbq(Register dst, Register src) {
9072   (void) prefixq_and_encode(dst->encoding(), src->encoding());
9073   emit_arith(0x1B, 0xC0, dst, src);
9074 }
9075 
9076 void Assembler::shlq(Register dst, int imm8) {
9077   assert(isShiftCount(imm8 >> 1), "illegal shift count");
9078   int encode = prefixq_and_encode(dst->encoding());
9079   if (imm8 == 1) {
9080     emit_int8((unsigned char)0xD1);
9081     emit_int8((unsigned char)(0xE0 | encode));
9082   } else {
9083     emit_int8((unsigned char)0xC1);
9084     emit_int8((unsigned char)(0xE0 | encode));
9085     emit_int8(imm8);
9086   }
9087 }
9088 
9089 void Assembler::shlq(Register dst) {
9090   int encode = prefixq_and_encode(dst->encoding());
9091   emit_int8((unsigned char)0xD3);
9092   emit_int8((unsigned char)(0xE0 | encode));
9093 }
9094 
9095 void Assembler::shrq(Register dst, int imm8) {
9096   assert(isShiftCount(imm8 >> 1), "illegal shift count");
9097   int encode = prefixq_and_encode(dst->encoding());
9098   emit_int8((unsigned char)0xC1);
9099   emit_int8((unsigned char)(0xE8 | encode));
9100   emit_int8(imm8);
9101 }
9102 
9103 void Assembler::shrq(Register dst) {
9104   int encode = prefixq_and_encode(dst->encoding());
9105   emit_int8((unsigned char)0xD3);
9106   emit_int8(0xE8 | encode);
9107 }
9108 
9109 void Assembler::subq(Address dst, int32_t imm32) {
9110   InstructionMark im(this);
9111   prefixq(dst);
9112   emit_arith_operand(0x81, rbp, dst, imm32);
9113 }
9114 
9115 void Assembler::subq(Address dst, Register src) {
9116   InstructionMark im(this);
9117   prefixq(dst, src);
9118   emit_int8(0x29);
9119   emit_operand(src, dst);
9120 }
9121 
9122 void Assembler::subq(Register dst, int32_t imm32) {
9123   (void) prefixq_and_encode(dst->encoding());
9124   emit_arith(0x81, 0xE8, dst, imm32);
9125 }
9126 
9127 // Force generation of a 4 byte immediate value even if it fits into 8bit
9128 void Assembler::subq_imm32(Register dst, int32_t imm32) {
9129   (void) prefixq_and_encode(dst->encoding());
9130   emit_arith_imm32(0x81, 0xE8, dst, imm32);
9131 }
9132 
9133 void Assembler::subq(Register dst, Address src) {
9134   InstructionMark im(this);
9135   prefixq(src, dst);
9136   emit_int8(0x2B);
9137   emit_operand(dst, src);
9138 }
9139 
9140 void Assembler::subq(Register dst, Register src) {
9141   (void) prefixq_and_encode(dst->encoding(), src->encoding());
9142   emit_arith(0x2B, 0xC0, dst, src);
9143 }
9144 
9145 void Assembler::testq(Register dst, int32_t imm32) {
9146   // not using emit_arith because test
9147   // doesn't support sign-extension of
9148   // 8bit operands
9149   int encode = dst->encoding();
9150   if (encode == 0) {
9151     prefix(REX_W);
9152     emit_int8((unsigned char)0xA9);
9153   } else {
9154     encode = prefixq_and_encode(encode);
9155     emit_int8((unsigned char)0xF7);
9156     emit_int8((unsigned char)(0xC0 | encode));
9157   }
9158   emit_int32(imm32);
9159 }
9160 
9161 void Assembler::testq(Register dst, Register src) {
9162   (void) prefixq_and_encode(dst->encoding(), src->encoding());
9163   emit_arith(0x85, 0xC0, dst, src);
9164 }
9165 
9166 void Assembler::testq(Register dst, Address src) {
9167   InstructionMark im(this);
9168   prefixq(src, dst);
9169   emit_int8((unsigned char)0x85);
9170   emit_operand(dst, src);
9171 }
9172 
9173 void Assembler::xaddq(Address dst, Register src) {
9174   InstructionMark im(this);
9175   prefixq(dst, src);
9176   emit_int8(0x0F);
9177   emit_int8((unsigned char)0xC1);
9178   emit_operand(src, dst);
9179 }
9180 
9181 void Assembler::xchgq(Register dst, Address src) {
9182   InstructionMark im(this);
9183   prefixq(src, dst);
9184   emit_int8((unsigned char)0x87);
9185   emit_operand(dst, src);
9186 }
9187 
9188 void Assembler::xchgq(Register dst, Register src) {
9189   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
9190   emit_int8((unsigned char)0x87);
9191   emit_int8((unsigned char)(0xc0 | encode));
9192 }
9193 
9194 void Assembler::xorq(Register dst, Register src) {
9195   (void) prefixq_and_encode(dst->encoding(), src->encoding());
9196   emit_arith(0x33, 0xC0, dst, src);
9197 }
9198 
9199 void Assembler::xorq(Register dst, Address src) {
9200   InstructionMark im(this);
9201   prefixq(src, dst);
9202   emit_int8(0x33);
9203   emit_operand(dst, src);
9204 }
9205 
9206 #endif // !LP64