1 /*
   2  * Copyright (c) 1997, 2016, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #include "precompiled.hpp"
  26 #include "asm/assembler.hpp"
  27 #include "asm/assembler.inline.hpp"
  28 #include "gc/shared/cardTableModRefBS.hpp"
  29 #include "gc/shared/collectedHeap.inline.hpp"
  30 #include "interpreter/interpreter.hpp"
  31 #include "memory/resourceArea.hpp"
  32 #include "prims/methodHandles.hpp"
  33 #include "runtime/biasedLocking.hpp"
  34 #include "runtime/interfaceSupport.hpp"
  35 #include "runtime/objectMonitor.hpp"
  36 #include "runtime/os.hpp"
  37 #include "runtime/sharedRuntime.hpp"
  38 #include "runtime/stubRoutines.hpp"
  39 #include "utilities/macros.hpp"
  40 #if INCLUDE_ALL_GCS
  41 #include "gc/g1/g1CollectedHeap.inline.hpp"
  42 #include "gc/g1/g1SATBCardTableModRefBS.hpp"
  43 #include "gc/g1/heapRegion.hpp"
  44 #endif // INCLUDE_ALL_GCS
  45 
  46 #ifdef PRODUCT
  47 #define BLOCK_COMMENT(str) /* nothing */
  48 #define STOP(error) stop(error)
  49 #else
  50 #define BLOCK_COMMENT(str) block_comment(str)
  51 #define STOP(error) block_comment(error); stop(error)
  52 #endif
  53 
  54 #define BIND(label) bind(label); BLOCK_COMMENT(#label ":")
  55 // Implementation of AddressLiteral
  56 
  57 // A 2-D table for managing compressed displacement(disp8) on EVEX enabled platforms.
  58 unsigned char tuple_table[Assembler::EVEX_ETUP + 1][Assembler::AVX_512bit + 1] = {
  59   // -----------------Table 4.5 -------------------- //
  60   16, 32, 64,  // EVEX_FV(0)
  61   4,  4,  4,   // EVEX_FV(1) - with Evex.b
  62   16, 32, 64,  // EVEX_FV(2) - with Evex.w
  63   8,  8,  8,   // EVEX_FV(3) - with Evex.w and Evex.b
  64   8,  16, 32,  // EVEX_HV(0)
  65   4,  4,  4,   // EVEX_HV(1) - with Evex.b
  66   // -----------------Table 4.6 -------------------- //
  67   16, 32, 64,  // EVEX_FVM(0)
  68   1,  1,  1,   // EVEX_T1S(0)
  69   2,  2,  2,   // EVEX_T1S(1)
  70   4,  4,  4,   // EVEX_T1S(2)
  71   8,  8,  8,   // EVEX_T1S(3)
  72   4,  4,  4,   // EVEX_T1F(0)
  73   8,  8,  8,   // EVEX_T1F(1)
  74   8,  8,  8,   // EVEX_T2(0)
  75   0,  16, 16,  // EVEX_T2(1)
  76   0,  16, 16,  // EVEX_T4(0)
  77   0,  0,  32,  // EVEX_T4(1)
  78   0,  0,  32,  // EVEX_T8(0)
  79   8,  16, 32,  // EVEX_HVM(0)
  80   4,  8,  16,  // EVEX_QVM(0)
  81   2,  4,  8,   // EVEX_OVM(0)
  82   16, 16, 16,  // EVEX_M128(0)
  83   8,  32, 64,  // EVEX_DUP(0)
  84   0,  0,  0    // EVEX_NTUP
  85 };
  86 
  87 AddressLiteral::AddressLiteral(address target, relocInfo::relocType rtype) {
  88   _is_lval = false;
  89   _target = target;
  90   switch (rtype) {
  91   case relocInfo::oop_type:
  92   case relocInfo::metadata_type:
  93     // Oops are a special case. Normally they would be their own section
  94     // but in cases like icBuffer they are literals in the code stream that
  95     // we don't have a section for. We use none so that we get a literal address
  96     // which is always patchable.
  97     break;
  98   case relocInfo::external_word_type:
  99     _rspec = external_word_Relocation::spec(target);
 100     break;
 101   case relocInfo::internal_word_type:
 102     _rspec = internal_word_Relocation::spec(target);
 103     break;
 104   case relocInfo::opt_virtual_call_type:
 105     _rspec = opt_virtual_call_Relocation::spec();
 106     break;
 107   case relocInfo::static_call_type:
 108     _rspec = static_call_Relocation::spec();
 109     break;
 110   case relocInfo::runtime_call_type:
 111     _rspec = runtime_call_Relocation::spec();
 112     break;
 113   case relocInfo::poll_type:
 114   case relocInfo::poll_return_type:
 115     _rspec = Relocation::spec_simple(rtype);
 116     break;
 117   case relocInfo::none:
 118     break;
 119   default:
 120     ShouldNotReachHere();
 121     break;
 122   }
 123 }
 124 
 125 // Implementation of Address
 126 
 127 #ifdef _LP64
 128 
 129 Address Address::make_array(ArrayAddress adr) {
 130   // Not implementable on 64bit machines
 131   // Should have been handled higher up the call chain.
 132   ShouldNotReachHere();
 133   return Address();
 134 }
 135 
 136 // exceedingly dangerous constructor
 137 Address::Address(int disp, address loc, relocInfo::relocType rtype) {
 138   _base  = noreg;
 139   _index = noreg;
 140   _scale = no_scale;
 141   _disp  = disp;
 142   switch (rtype) {
 143     case relocInfo::external_word_type:
 144       _rspec = external_word_Relocation::spec(loc);
 145       break;
 146     case relocInfo::internal_word_type:
 147       _rspec = internal_word_Relocation::spec(loc);
 148       break;
 149     case relocInfo::runtime_call_type:
 150       // HMM
 151       _rspec = runtime_call_Relocation::spec();
 152       break;
 153     case relocInfo::poll_type:
 154     case relocInfo::poll_return_type:
 155       _rspec = Relocation::spec_simple(rtype);
 156       break;
 157     case relocInfo::none:
 158       break;
 159     default:
 160       ShouldNotReachHere();
 161   }
 162 }
 163 #else // LP64
 164 
 165 Address Address::make_array(ArrayAddress adr) {
 166   AddressLiteral base = adr.base();
 167   Address index = adr.index();
 168   assert(index._disp == 0, "must not have disp"); // maybe it can?
 169   Address array(index._base, index._index, index._scale, (intptr_t) base.target());
 170   array._rspec = base._rspec;
 171   return array;
 172 }
 173 
 174 // exceedingly dangerous constructor
 175 Address::Address(address loc, RelocationHolder spec) {
 176   _base  = noreg;
 177   _index = noreg;
 178   _scale = no_scale;
 179   _disp  = (intptr_t) loc;
 180   _rspec = spec;
 181 }
 182 
 183 #endif // _LP64
 184 
 185 
 186 
 187 // Convert the raw encoding form into the form expected by the constructor for
 188 // Address.  An index of 4 (rsp) corresponds to having no index, so convert
 189 // that to noreg for the Address constructor.
 190 Address Address::make_raw(int base, int index, int scale, int disp, relocInfo::relocType disp_reloc) {
 191   RelocationHolder rspec = (disp_reloc == relocInfo::none)
 192                                      ? RelocationHolder::none
 193                                      : Relocation::spec_simple(disp_reloc);
 194   bool valid_index = index != rsp->encoding();
 195   if (valid_index) {
 196     Address madr(as_Register(base), as_Register(index), (Address::ScaleFactor)scale, in_ByteSize(disp));
 197     madr._rspec = rspec;
 198     return madr;
 199   } else {
 200     Address madr(as_Register(base), noreg, Address::no_scale, in_ByteSize(disp));
 201     madr._rspec = rspec;
 202     return madr;
 203   }
 204 }
 205 
 206 // Implementation of Assembler
 207 
 208 int AbstractAssembler::code_fill_byte() {
 209   return (u_char)'\xF4'; // hlt
 210 }
 211 
 212 // make this go away someday
 213 void Assembler::emit_data(jint data, relocInfo::relocType rtype, int format) {
 214   if (rtype == relocInfo::none)
 215     emit_int32(data);
 216   else
 217     emit_data(data, Relocation::spec_simple(rtype), format);
 218 }
 219 
 220 void Assembler::emit_data(jint data, RelocationHolder const& rspec, int format) {
 221   assert(imm_operand == 0, "default format must be immediate in this file");
 222   assert(inst_mark() != NULL, "must be inside InstructionMark");
 223   if (rspec.type() !=  relocInfo::none) {
 224     #ifdef ASSERT
 225       check_relocation(rspec, format);
 226     #endif
 227     // Do not use AbstractAssembler::relocate, which is not intended for
 228     // embedded words.  Instead, relocate to the enclosing instruction.
 229 
 230     // hack. call32 is too wide for mask so use disp32
 231     if (format == call32_operand)
 232       code_section()->relocate(inst_mark(), rspec, disp32_operand);
 233     else
 234       code_section()->relocate(inst_mark(), rspec, format);
 235   }
 236   emit_int32(data);
 237 }
 238 
 239 static int encode(Register r) {
 240   int enc = r->encoding();
 241   if (enc >= 8) {
 242     enc -= 8;
 243   }
 244   return enc;
 245 }
 246 
 247 void Assembler::emit_arith_b(int op1, int op2, Register dst, int imm8) {
 248   assert(dst->has_byte_register(), "must have byte register");
 249   assert(isByte(op1) && isByte(op2), "wrong opcode");
 250   assert(isByte(imm8), "not a byte");
 251   assert((op1 & 0x01) == 0, "should be 8bit operation");
 252   emit_int8(op1);
 253   emit_int8(op2 | encode(dst));
 254   emit_int8(imm8);
 255 }
 256 
 257 
 258 void Assembler::emit_arith(int op1, int op2, Register dst, int32_t imm32) {
 259   assert(isByte(op1) && isByte(op2), "wrong opcode");
 260   assert((op1 & 0x01) == 1, "should be 32bit operation");
 261   assert((op1 & 0x02) == 0, "sign-extension bit should not be set");
 262   if (is8bit(imm32)) {
 263     emit_int8(op1 | 0x02); // set sign bit
 264     emit_int8(op2 | encode(dst));
 265     emit_int8(imm32 & 0xFF);
 266   } else {
 267     emit_int8(op1);
 268     emit_int8(op2 | encode(dst));
 269     emit_int32(imm32);
 270   }
 271 }
 272 
 273 // Force generation of a 4 byte immediate value even if it fits into 8bit
 274 void Assembler::emit_arith_imm32(int op1, int op2, Register dst, int32_t imm32) {
 275   assert(isByte(op1) && isByte(op2), "wrong opcode");
 276   assert((op1 & 0x01) == 1, "should be 32bit operation");
 277   assert((op1 & 0x02) == 0, "sign-extension bit should not be set");
 278   emit_int8(op1);
 279   emit_int8(op2 | encode(dst));
 280   emit_int32(imm32);
 281 }
 282 
 283 // immediate-to-memory forms
 284 void Assembler::emit_arith_operand(int op1, Register rm, Address adr, int32_t imm32) {
 285   assert((op1 & 0x01) == 1, "should be 32bit operation");
 286   assert((op1 & 0x02) == 0, "sign-extension bit should not be set");
 287   if (is8bit(imm32)) {
 288     emit_int8(op1 | 0x02); // set sign bit
 289     emit_operand(rm, adr, 1);
 290     emit_int8(imm32 & 0xFF);
 291   } else {
 292     emit_int8(op1);
 293     emit_operand(rm, adr, 4);
 294     emit_int32(imm32);
 295   }
 296 }
 297 
 298 
 299 void Assembler::emit_arith(int op1, int op2, Register dst, Register src) {
 300   assert(isByte(op1) && isByte(op2), "wrong opcode");
 301   emit_int8(op1);
 302   emit_int8(op2 | encode(dst) << 3 | encode(src));
 303 }
 304 
 305 
 306 bool Assembler::query_compressed_disp_byte(int disp, bool is_evex_inst, int vector_len,
 307                                            int cur_tuple_type, int in_size_in_bits, int cur_encoding) {
 308   int mod_idx = 0;
 309   // We will test if the displacement fits the compressed format and if so
 310   // apply the compression to the displacment iff the result is8bit.
 311   if (VM_Version::supports_evex() && is_evex_inst) {
 312     switch (cur_tuple_type) {
 313     case EVEX_FV:
 314       if ((cur_encoding & VEX_W) == VEX_W) {
 315         mod_idx = ((cur_encoding & EVEX_Rb) == EVEX_Rb) ? 3 : 2;
 316       } else {
 317         mod_idx = ((cur_encoding & EVEX_Rb) == EVEX_Rb) ? 1 : 0;
 318       }
 319       break;
 320 
 321     case EVEX_HV:
 322       mod_idx = ((cur_encoding & EVEX_Rb) == EVEX_Rb) ? 1 : 0;
 323       break;
 324 
 325     case EVEX_FVM:
 326       break;
 327 
 328     case EVEX_T1S:
 329       switch (in_size_in_bits) {
 330       case EVEX_8bit:
 331         break;
 332 
 333       case EVEX_16bit:
 334         mod_idx = 1;
 335         break;
 336 
 337       case EVEX_32bit:
 338         mod_idx = 2;
 339         break;
 340 
 341       case EVEX_64bit:
 342         mod_idx = 3;
 343         break;
 344       }
 345       break;
 346 
 347     case EVEX_T1F:
 348     case EVEX_T2:
 349     case EVEX_T4:
 350       mod_idx = (in_size_in_bits == EVEX_64bit) ? 1 : 0;
 351       break;
 352 
 353     case EVEX_T8:
 354       break;
 355 
 356     case EVEX_HVM:
 357       break;
 358 
 359     case EVEX_QVM:
 360       break;
 361 
 362     case EVEX_OVM:
 363       break;
 364 
 365     case EVEX_M128:
 366       break;
 367 
 368     case EVEX_DUP:
 369       break;
 370 
 371     default:
 372       assert(0, "no valid evex tuple_table entry");
 373       break;
 374     }
 375 
 376     if (vector_len >= AVX_128bit && vector_len <= AVX_512bit) {
 377       int disp_factor = tuple_table[cur_tuple_type + mod_idx][vector_len];
 378       if ((disp % disp_factor) == 0) {
 379         int new_disp = disp / disp_factor;
 380         if ((-0x80 <= new_disp && new_disp < 0x80)) {
 381           disp = new_disp;
 382         }
 383       } else {
 384         return false;
 385       }
 386     }
 387   }
 388   return (-0x80 <= disp && disp < 0x80);
 389 }
 390 
 391 
 392 bool Assembler::emit_compressed_disp_byte(int &disp) {
 393   int mod_idx = 0;
 394   // We will test if the displacement fits the compressed format and if so
 395   // apply the compression to the displacment iff the result is8bit.
 396   if (VM_Version::supports_evex() && _attributes && _attributes->is_evex_instruction()) {
 397     int evex_encoding = _attributes->get_evex_encoding();
 398     int tuple_type = _attributes->get_tuple_type();
 399     switch (tuple_type) {
 400     case EVEX_FV:
 401       if ((evex_encoding & VEX_W) == VEX_W) {
 402         mod_idx = ((evex_encoding & EVEX_Rb) == EVEX_Rb) ? 3 : 2;
 403       } else {
 404         mod_idx = ((evex_encoding & EVEX_Rb) == EVEX_Rb) ? 1 : 0;
 405       }
 406       break;
 407 
 408     case EVEX_HV:
 409       mod_idx = ((evex_encoding & EVEX_Rb) == EVEX_Rb) ? 1 : 0;
 410       break;
 411 
 412     case EVEX_FVM:
 413       break;
 414 
 415     case EVEX_T1S:
 416       switch (_attributes->get_input_size()) {
 417       case EVEX_8bit:
 418         break;
 419 
 420       case EVEX_16bit:
 421         mod_idx = 1;
 422         break;
 423 
 424       case EVEX_32bit:
 425         mod_idx = 2;
 426         break;
 427 
 428       case EVEX_64bit:
 429         mod_idx = 3;
 430         break;
 431       }
 432       break;
 433 
 434     case EVEX_T1F:
 435     case EVEX_T2:
 436     case EVEX_T4:
 437       mod_idx = (_attributes->get_input_size() == EVEX_64bit) ? 1 : 0;
 438       break;
 439 
 440     case EVEX_T8:
 441       break;
 442 
 443     case EVEX_HVM:
 444       break;
 445 
 446     case EVEX_QVM:
 447       break;
 448 
 449     case EVEX_OVM:
 450       break;
 451 
 452     case EVEX_M128:
 453       break;
 454 
 455     case EVEX_DUP:
 456       break;
 457 
 458     default:
 459       assert(0, "no valid evex tuple_table entry");
 460       break;
 461     }
 462 
 463     int vector_len = _attributes->get_vector_len();
 464     if (vector_len >= AVX_128bit && vector_len <= AVX_512bit) {
 465       int disp_factor = tuple_table[tuple_type + mod_idx][vector_len];
 466       if ((disp % disp_factor) == 0) {
 467         int new_disp = disp / disp_factor;
 468         if (is8bit(new_disp)) {
 469           disp = new_disp;
 470         }
 471       } else {
 472         return false;
 473       }
 474     }
 475   }
 476   return is8bit(disp);
 477 }
 478 
 479 
 480 void Assembler::emit_operand(Register reg, Register base, Register index,
 481                              Address::ScaleFactor scale, int disp,
 482                              RelocationHolder const& rspec,
 483                              int rip_relative_correction) {
 484   relocInfo::relocType rtype = (relocInfo::relocType) rspec.type();
 485 
 486   // Encode the registers as needed in the fields they are used in
 487 
 488   int regenc = encode(reg) << 3;
 489   int indexenc = index->is_valid() ? encode(index) << 3 : 0;
 490   int baseenc = base->is_valid() ? encode(base) : 0;
 491 
 492   if (base->is_valid()) {
 493     if (index->is_valid()) {
 494       assert(scale != Address::no_scale, "inconsistent address");
 495       // [base + index*scale + disp]
 496       if (disp == 0 && rtype == relocInfo::none  &&
 497           base != rbp LP64_ONLY(&& base != r13)) {
 498         // [base + index*scale]
 499         // [00 reg 100][ss index base]
 500         assert(index != rsp, "illegal addressing mode");
 501         emit_int8(0x04 | regenc);
 502         emit_int8(scale << 6 | indexenc | baseenc);
 503       } else if (emit_compressed_disp_byte(disp) && rtype == relocInfo::none) {
 504         // [base + index*scale + imm8]
 505         // [01 reg 100][ss index base] imm8
 506         assert(index != rsp, "illegal addressing mode");
 507         emit_int8(0x44 | regenc);
 508         emit_int8(scale << 6 | indexenc | baseenc);
 509         emit_int8(disp & 0xFF);
 510       } else {
 511         // [base + index*scale + disp32]
 512         // [10 reg 100][ss index base] disp32
 513         assert(index != rsp, "illegal addressing mode");
 514         emit_int8(0x84 | regenc);
 515         emit_int8(scale << 6 | indexenc | baseenc);
 516         emit_data(disp, rspec, disp32_operand);
 517       }
 518     } else if (base == rsp LP64_ONLY(|| base == r12)) {
 519       // [rsp + disp]
 520       if (disp == 0 && rtype == relocInfo::none) {
 521         // [rsp]
 522         // [00 reg 100][00 100 100]
 523         emit_int8(0x04 | regenc);
 524         emit_int8(0x24);
 525       } else if (emit_compressed_disp_byte(disp) && rtype == relocInfo::none) {
 526         // [rsp + imm8]
 527         // [01 reg 100][00 100 100] disp8
 528         emit_int8(0x44 | regenc);
 529         emit_int8(0x24);
 530         emit_int8(disp & 0xFF);
 531       } else {
 532         // [rsp + imm32]
 533         // [10 reg 100][00 100 100] disp32
 534         emit_int8(0x84 | regenc);
 535         emit_int8(0x24);
 536         emit_data(disp, rspec, disp32_operand);
 537       }
 538     } else {
 539       // [base + disp]
 540       assert(base != rsp LP64_ONLY(&& base != r12), "illegal addressing mode");
 541       if (disp == 0 && rtype == relocInfo::none &&
 542           base != rbp LP64_ONLY(&& base != r13)) {
 543         // [base]
 544         // [00 reg base]
 545         emit_int8(0x00 | regenc | baseenc);
 546       } else if (emit_compressed_disp_byte(disp) && rtype == relocInfo::none) {
 547         // [base + disp8]
 548         // [01 reg base] disp8
 549         emit_int8(0x40 | regenc | baseenc);
 550         emit_int8(disp & 0xFF);
 551       } else {
 552         // [base + disp32]
 553         // [10 reg base] disp32
 554         emit_int8(0x80 | regenc | baseenc);
 555         emit_data(disp, rspec, disp32_operand);
 556       }
 557     }
 558   } else {
 559     if (index->is_valid()) {
 560       assert(scale != Address::no_scale, "inconsistent address");
 561       // [index*scale + disp]
 562       // [00 reg 100][ss index 101] disp32
 563       assert(index != rsp, "illegal addressing mode");
 564       emit_int8(0x04 | regenc);
 565       emit_int8(scale << 6 | indexenc | 0x05);
 566       emit_data(disp, rspec, disp32_operand);
 567     } else if (rtype != relocInfo::none ) {
 568       // [disp] (64bit) RIP-RELATIVE (32bit) abs
 569       // [00 000 101] disp32
 570 
 571       emit_int8(0x05 | regenc);
 572       // Note that the RIP-rel. correction applies to the generated
 573       // disp field, but _not_ to the target address in the rspec.
 574 
 575       // disp was created by converting the target address minus the pc
 576       // at the start of the instruction. That needs more correction here.
 577       // intptr_t disp = target - next_ip;
 578       assert(inst_mark() != NULL, "must be inside InstructionMark");
 579       address next_ip = pc() + sizeof(int32_t) + rip_relative_correction;
 580       int64_t adjusted = disp;
 581       // Do rip-rel adjustment for 64bit
 582       LP64_ONLY(adjusted -=  (next_ip - inst_mark()));
 583       assert(is_simm32(adjusted),
 584              "must be 32bit offset (RIP relative address)");
 585       emit_data((int32_t) adjusted, rspec, disp32_operand);
 586 
 587     } else {
 588       // 32bit never did this, did everything as the rip-rel/disp code above
 589       // [disp] ABSOLUTE
 590       // [00 reg 100][00 100 101] disp32
 591       emit_int8(0x04 | regenc);
 592       emit_int8(0x25);
 593       emit_data(disp, rspec, disp32_operand);
 594     }
 595   }
 596 }
 597 
 598 void Assembler::emit_operand(XMMRegister reg, Register base, Register index,
 599                              Address::ScaleFactor scale, int disp,
 600                              RelocationHolder const& rspec) {
 601   if (UseAVX > 2) {
 602     int xreg_enc = reg->encoding();
 603     if (xreg_enc > 15) {
 604       XMMRegister new_reg = as_XMMRegister(xreg_enc & 0xf);
 605       emit_operand((Register)new_reg, base, index, scale, disp, rspec);
 606       return;
 607     }
 608   }
 609   emit_operand((Register)reg, base, index, scale, disp, rspec);
 610 }
 611 
 612 // Secret local extension to Assembler::WhichOperand:
 613 #define end_pc_operand (_WhichOperand_limit)
 614 
 615 address Assembler::locate_operand(address inst, WhichOperand which) {
 616   // Decode the given instruction, and return the address of
 617   // an embedded 32-bit operand word.
 618 
 619   // If "which" is disp32_operand, selects the displacement portion
 620   // of an effective address specifier.
 621   // If "which" is imm64_operand, selects the trailing immediate constant.
 622   // If "which" is call32_operand, selects the displacement of a call or jump.
 623   // Caller is responsible for ensuring that there is such an operand,
 624   // and that it is 32/64 bits wide.
 625 
 626   // If "which" is end_pc_operand, find the end of the instruction.
 627 
 628   address ip = inst;
 629   bool is_64bit = false;
 630 
 631   debug_only(bool has_disp32 = false);
 632   int tail_size = 0; // other random bytes (#32, #16, etc.) at end of insn
 633 
 634   again_after_prefix:
 635   switch (0xFF & *ip++) {
 636 
 637   // These convenience macros generate groups of "case" labels for the switch.
 638 #define REP4(x) (x)+0: case (x)+1: case (x)+2: case (x)+3
 639 #define REP8(x) (x)+0: case (x)+1: case (x)+2: case (x)+3: \
 640              case (x)+4: case (x)+5: case (x)+6: case (x)+7
 641 #define REP16(x) REP8((x)+0): \
 642               case REP8((x)+8)
 643 
 644   case CS_segment:
 645   case SS_segment:
 646   case DS_segment:
 647   case ES_segment:
 648   case FS_segment:
 649   case GS_segment:
 650     // Seems dubious
 651     LP64_ONLY(assert(false, "shouldn't have that prefix"));
 652     assert(ip == inst+1, "only one prefix allowed");
 653     goto again_after_prefix;
 654 
 655   case 0x67:
 656   case REX:
 657   case REX_B:
 658   case REX_X:
 659   case REX_XB:
 660   case REX_R:
 661   case REX_RB:
 662   case REX_RX:
 663   case REX_RXB:
 664     NOT_LP64(assert(false, "64bit prefixes"));
 665     goto again_after_prefix;
 666 
 667   case REX_W:
 668   case REX_WB:
 669   case REX_WX:
 670   case REX_WXB:
 671   case REX_WR:
 672   case REX_WRB:
 673   case REX_WRX:
 674   case REX_WRXB:
 675     NOT_LP64(assert(false, "64bit prefixes"));
 676     is_64bit = true;
 677     goto again_after_prefix;
 678 
 679   case 0xFF: // pushq a; decl a; incl a; call a; jmp a
 680   case 0x88: // movb a, r
 681   case 0x89: // movl a, r
 682   case 0x8A: // movb r, a
 683   case 0x8B: // movl r, a
 684   case 0x8F: // popl a
 685     debug_only(has_disp32 = true);
 686     break;
 687 
 688   case 0x68: // pushq #32
 689     if (which == end_pc_operand) {
 690       return ip + 4;
 691     }
 692     assert(which == imm_operand && !is_64bit, "pushl has no disp32 or 64bit immediate");
 693     return ip;                  // not produced by emit_operand
 694 
 695   case 0x66: // movw ... (size prefix)
 696     again_after_size_prefix2:
 697     switch (0xFF & *ip++) {
 698     case REX:
 699     case REX_B:
 700     case REX_X:
 701     case REX_XB:
 702     case REX_R:
 703     case REX_RB:
 704     case REX_RX:
 705     case REX_RXB:
 706     case REX_W:
 707     case REX_WB:
 708     case REX_WX:
 709     case REX_WXB:
 710     case REX_WR:
 711     case REX_WRB:
 712     case REX_WRX:
 713     case REX_WRXB:
 714       NOT_LP64(assert(false, "64bit prefix found"));
 715       goto again_after_size_prefix2;
 716     case 0x8B: // movw r, a
 717     case 0x89: // movw a, r
 718       debug_only(has_disp32 = true);
 719       break;
 720     case 0xC7: // movw a, #16
 721       debug_only(has_disp32 = true);
 722       tail_size = 2;  // the imm16
 723       break;
 724     case 0x0F: // several SSE/SSE2 variants
 725       ip--;    // reparse the 0x0F
 726       goto again_after_prefix;
 727     default:
 728       ShouldNotReachHere();
 729     }
 730     break;
 731 
 732   case REP8(0xB8): // movl/q r, #32/#64(oop?)
 733     if (which == end_pc_operand)  return ip + (is_64bit ? 8 : 4);
 734     // these asserts are somewhat nonsensical
 735 #ifndef _LP64
 736     assert(which == imm_operand || which == disp32_operand,
 737            "which %d is_64_bit %d ip " INTPTR_FORMAT, which, is_64bit, p2i(ip));
 738 #else
 739     assert((which == call32_operand || which == imm_operand) && is_64bit ||
 740            which == narrow_oop_operand && !is_64bit,
 741            "which %d is_64_bit %d ip " INTPTR_FORMAT, which, is_64bit, p2i(ip));
 742 #endif // _LP64
 743     return ip;
 744 
 745   case 0x69: // imul r, a, #32
 746   case 0xC7: // movl a, #32(oop?)
 747     tail_size = 4;
 748     debug_only(has_disp32 = true); // has both kinds of operands!
 749     break;
 750 
 751   case 0x0F: // movx..., etc.
 752     switch (0xFF & *ip++) {
 753     case 0x3A: // pcmpestri
 754       tail_size = 1;
 755     case 0x38: // ptest, pmovzxbw
 756       ip++; // skip opcode
 757       debug_only(has_disp32 = true); // has both kinds of operands!
 758       break;
 759 
 760     case 0x70: // pshufd r, r/a, #8
 761       debug_only(has_disp32 = true); // has both kinds of operands!
 762     case 0x73: // psrldq r, #8
 763       tail_size = 1;
 764       break;
 765 
 766     case 0x12: // movlps
 767     case 0x28: // movaps
 768     case 0x2E: // ucomiss
 769     case 0x2F: // comiss
 770     case 0x54: // andps
 771     case 0x55: // andnps
 772     case 0x56: // orps
 773     case 0x57: // xorps
 774     case 0x58: // addpd
 775     case 0x59: // mulpd
 776     case 0x6E: // movd
 777     case 0x7E: // movd
 778     case 0xAE: // ldmxcsr, stmxcsr, fxrstor, fxsave, clflush
 779     case 0xFE: // paddd
 780       debug_only(has_disp32 = true);
 781       break;
 782 
 783     case 0xAD: // shrd r, a, %cl
 784     case 0xAF: // imul r, a
 785     case 0xBE: // movsbl r, a (movsxb)
 786     case 0xBF: // movswl r, a (movsxw)
 787     case 0xB6: // movzbl r, a (movzxb)
 788     case 0xB7: // movzwl r, a (movzxw)
 789     case REP16(0x40): // cmovl cc, r, a
 790     case 0xB0: // cmpxchgb
 791     case 0xB1: // cmpxchg
 792     case 0xC1: // xaddl
 793     case 0xC7: // cmpxchg8
 794     case REP16(0x90): // setcc a
 795       debug_only(has_disp32 = true);
 796       // fall out of the switch to decode the address
 797       break;
 798 
 799     case 0xC4: // pinsrw r, a, #8
 800       debug_only(has_disp32 = true);
 801     case 0xC5: // pextrw r, r, #8
 802       tail_size = 1;  // the imm8
 803       break;
 804 
 805     case 0xAC: // shrd r, a, #8
 806       debug_only(has_disp32 = true);
 807       tail_size = 1;  // the imm8
 808       break;
 809 
 810     case REP16(0x80): // jcc rdisp32
 811       if (which == end_pc_operand)  return ip + 4;
 812       assert(which == call32_operand, "jcc has no disp32 or imm");
 813       return ip;
 814     default:
 815       ShouldNotReachHere();
 816     }
 817     break;
 818 
 819   case 0x81: // addl a, #32; addl r, #32
 820     // also: orl, adcl, sbbl, andl, subl, xorl, cmpl
 821     // on 32bit in the case of cmpl, the imm might be an oop
 822     tail_size = 4;
 823     debug_only(has_disp32 = true); // has both kinds of operands!
 824     break;
 825 
 826   case 0x83: // addl a, #8; addl r, #8
 827     // also: orl, adcl, sbbl, andl, subl, xorl, cmpl
 828     debug_only(has_disp32 = true); // has both kinds of operands!
 829     tail_size = 1;
 830     break;
 831 
 832   case 0x9B:
 833     switch (0xFF & *ip++) {
 834     case 0xD9: // fnstcw a
 835       debug_only(has_disp32 = true);
 836       break;
 837     default:
 838       ShouldNotReachHere();
 839     }
 840     break;
 841 
 842   case REP4(0x00): // addb a, r; addl a, r; addb r, a; addl r, a
 843   case REP4(0x10): // adc...
 844   case REP4(0x20): // and...
 845   case REP4(0x30): // xor...
 846   case REP4(0x08): // or...
 847   case REP4(0x18): // sbb...
 848   case REP4(0x28): // sub...
 849   case 0xF7: // mull a
 850   case 0x8D: // lea r, a
 851   case 0x87: // xchg r, a
 852   case REP4(0x38): // cmp...
 853   case 0x85: // test r, a
 854     debug_only(has_disp32 = true); // has both kinds of operands!
 855     break;
 856 
 857   case 0xC1: // sal a, #8; sar a, #8; shl a, #8; shr a, #8
 858   case 0xC6: // movb a, #8
 859   case 0x80: // cmpb a, #8
 860   case 0x6B: // imul r, a, #8
 861     debug_only(has_disp32 = true); // has both kinds of operands!
 862     tail_size = 1; // the imm8
 863     break;
 864 
 865   case 0xC4: // VEX_3bytes
 866   case 0xC5: // VEX_2bytes
 867     assert((UseAVX > 0), "shouldn't have VEX prefix");
 868     assert(ip == inst+1, "no prefixes allowed");
 869     // C4 and C5 are also used as opcodes for PINSRW and PEXTRW instructions
 870     // but they have prefix 0x0F and processed when 0x0F processed above.
 871     //
 872     // In 32-bit mode the VEX first byte C4 and C5 alias onto LDS and LES
 873     // instructions (these instructions are not supported in 64-bit mode).
 874     // To distinguish them bits [7:6] are set in the VEX second byte since
 875     // ModRM byte can not be of the form 11xxxxxx in 32-bit mode. To set
 876     // those VEX bits REX and vvvv bits are inverted.
 877     //
 878     // Fortunately C2 doesn't generate these instructions so we don't need
 879     // to check for them in product version.
 880 
 881     // Check second byte
 882     NOT_LP64(assert((0xC0 & *ip) == 0xC0, "shouldn't have LDS and LES instructions"));
 883 
 884     int vex_opcode;
 885     // First byte
 886     if ((0xFF & *inst) == VEX_3bytes) {
 887       vex_opcode = VEX_OPCODE_MASK & *ip;
 888       ip++; // third byte
 889       is_64bit = ((VEX_W & *ip) == VEX_W);
 890     } else {
 891       vex_opcode = VEX_OPCODE_0F;
 892     }
 893     ip++; // opcode
 894     // To find the end of instruction (which == end_pc_operand).
 895     switch (vex_opcode) {
 896       case VEX_OPCODE_0F:
 897         switch (0xFF & *ip) {
 898         case 0x70: // pshufd r, r/a, #8
 899         case 0x71: // ps[rl|ra|ll]w r, #8
 900         case 0x72: // ps[rl|ra|ll]d r, #8
 901         case 0x73: // ps[rl|ra|ll]q r, #8
 902         case 0xC2: // cmp[ps|pd|ss|sd] r, r, r/a, #8
 903         case 0xC4: // pinsrw r, r, r/a, #8
 904         case 0xC5: // pextrw r/a, r, #8
 905         case 0xC6: // shufp[s|d] r, r, r/a, #8
 906           tail_size = 1;  // the imm8
 907           break;
 908         }
 909         break;
 910       case VEX_OPCODE_0F_3A:
 911         tail_size = 1;
 912         break;
 913     }
 914     ip++; // skip opcode
 915     debug_only(has_disp32 = true); // has both kinds of operands!
 916     break;
 917 
 918   case 0x62: // EVEX_4bytes
 919     assert((UseAVX > 0), "shouldn't have EVEX prefix");
 920     assert(ip == inst+1, "no prefixes allowed");
 921     // no EVEX collisions, all instructions that have 0x62 opcodes
 922     // have EVEX versions and are subopcodes of 0x66
 923     ip++; // skip P0 and exmaine W in P1
 924     is_64bit = ((VEX_W & *ip) == VEX_W);
 925     ip++; // move to P2
 926     ip++; // skip P2, move to opcode
 927     // To find the end of instruction (which == end_pc_operand).
 928     switch (0xFF & *ip) {
 929     case 0x22: // pinsrd r, r/a, #8
 930     case 0x61: // pcmpestri r, r/a, #8
 931     case 0x70: // pshufd r, r/a, #8
 932     case 0x73: // psrldq r, #8
 933       tail_size = 1;  // the imm8
 934       break;
 935     default:
 936       break;
 937     }
 938     ip++; // skip opcode
 939     debug_only(has_disp32 = true); // has both kinds of operands!
 940     break;
 941 
 942   case 0xD1: // sal a, 1; sar a, 1; shl a, 1; shr a, 1
 943   case 0xD3: // sal a, %cl; sar a, %cl; shl a, %cl; shr a, %cl
 944   case 0xD9: // fld_s a; fst_s a; fstp_s a; fldcw a
 945   case 0xDD: // fld_d a; fst_d a; fstp_d a
 946   case 0xDB: // fild_s a; fistp_s a; fld_x a; fstp_x a
 947   case 0xDF: // fild_d a; fistp_d a
 948   case 0xD8: // fadd_s a; fsubr_s a; fmul_s a; fdivr_s a; fcomp_s a
 949   case 0xDC: // fadd_d a; fsubr_d a; fmul_d a; fdivr_d a; fcomp_d a
 950   case 0xDE: // faddp_d a; fsubrp_d a; fmulp_d a; fdivrp_d a; fcompp_d a
 951     debug_only(has_disp32 = true);
 952     break;
 953 
 954   case 0xE8: // call rdisp32
 955   case 0xE9: // jmp  rdisp32
 956     if (which == end_pc_operand)  return ip + 4;
 957     assert(which == call32_operand, "call has no disp32 or imm");
 958     return ip;
 959 
 960   case 0xF0:                    // Lock
 961     assert(os::is_MP(), "only on MP");
 962     goto again_after_prefix;
 963 
 964   case 0xF3:                    // For SSE
 965   case 0xF2:                    // For SSE2
 966     switch (0xFF & *ip++) {
 967     case REX:
 968     case REX_B:
 969     case REX_X:
 970     case REX_XB:
 971     case REX_R:
 972     case REX_RB:
 973     case REX_RX:
 974     case REX_RXB:
 975     case REX_W:
 976     case REX_WB:
 977     case REX_WX:
 978     case REX_WXB:
 979     case REX_WR:
 980     case REX_WRB:
 981     case REX_WRX:
 982     case REX_WRXB:
 983       NOT_LP64(assert(false, "found 64bit prefix"));
 984       ip++;
 985     default:
 986       ip++;
 987     }
 988     debug_only(has_disp32 = true); // has both kinds of operands!
 989     break;
 990 
 991   default:
 992     ShouldNotReachHere();
 993 
 994 #undef REP8
 995 #undef REP16
 996   }
 997 
 998   assert(which != call32_operand, "instruction is not a call, jmp, or jcc");
 999 #ifdef _LP64
1000   assert(which != imm_operand, "instruction is not a movq reg, imm64");
1001 #else
1002   // assert(which != imm_operand || has_imm32, "instruction has no imm32 field");
1003   assert(which != imm_operand || has_disp32, "instruction has no imm32 field");
1004 #endif // LP64
1005   assert(which != disp32_operand || has_disp32, "instruction has no disp32 field");
1006 
1007   // parse the output of emit_operand
1008   int op2 = 0xFF & *ip++;
1009   int base = op2 & 0x07;
1010   int op3 = -1;
1011   const int b100 = 4;
1012   const int b101 = 5;
1013   if (base == b100 && (op2 >> 6) != 3) {
1014     op3 = 0xFF & *ip++;
1015     base = op3 & 0x07;   // refetch the base
1016   }
1017   // now ip points at the disp (if any)
1018 
1019   switch (op2 >> 6) {
1020   case 0:
1021     // [00 reg  100][ss index base]
1022     // [00 reg  100][00   100  esp]
1023     // [00 reg base]
1024     // [00 reg  100][ss index  101][disp32]
1025     // [00 reg  101]               [disp32]
1026 
1027     if (base == b101) {
1028       if (which == disp32_operand)
1029         return ip;              // caller wants the disp32
1030       ip += 4;                  // skip the disp32
1031     }
1032     break;
1033 
1034   case 1:
1035     // [01 reg  100][ss index base][disp8]
1036     // [01 reg  100][00   100  esp][disp8]
1037     // [01 reg base]               [disp8]
1038     ip += 1;                    // skip the disp8
1039     break;
1040 
1041   case 2:
1042     // [10 reg  100][ss index base][disp32]
1043     // [10 reg  100][00   100  esp][disp32]
1044     // [10 reg base]               [disp32]
1045     if (which == disp32_operand)
1046       return ip;                // caller wants the disp32
1047     ip += 4;                    // skip the disp32
1048     break;
1049 
1050   case 3:
1051     // [11 reg base]  (not a memory addressing mode)
1052     break;
1053   }
1054 
1055   if (which == end_pc_operand) {
1056     return ip + tail_size;
1057   }
1058 
1059 #ifdef _LP64
1060   assert(which == narrow_oop_operand && !is_64bit, "instruction is not a movl adr, imm32");
1061 #else
1062   assert(which == imm_operand, "instruction has only an imm field");
1063 #endif // LP64
1064   return ip;
1065 }
1066 
1067 address Assembler::locate_next_instruction(address inst) {
1068   // Secretly share code with locate_operand:
1069   return locate_operand(inst, end_pc_operand);
1070 }
1071 
1072 
1073 #ifdef ASSERT
1074 void Assembler::check_relocation(RelocationHolder const& rspec, int format) {
1075   address inst = inst_mark();
1076   assert(inst != NULL && inst < pc(), "must point to beginning of instruction");
1077   address opnd;
1078 
1079   Relocation* r = rspec.reloc();
1080   if (r->type() == relocInfo::none) {
1081     return;
1082   } else if (r->is_call() || format == call32_operand) {
1083     // assert(format == imm32_operand, "cannot specify a nonzero format");
1084     opnd = locate_operand(inst, call32_operand);
1085   } else if (r->is_data()) {
1086     assert(format == imm_operand || format == disp32_operand
1087            LP64_ONLY(|| format == narrow_oop_operand), "format ok");
1088     opnd = locate_operand(inst, (WhichOperand)format);
1089   } else {
1090     assert(format == imm_operand, "cannot specify a format");
1091     return;
1092   }
1093   assert(opnd == pc(), "must put operand where relocs can find it");
1094 }
1095 #endif // ASSERT
1096 
1097 void Assembler::emit_operand32(Register reg, Address adr) {
1098   assert(reg->encoding() < 8, "no extended registers");
1099   assert(!adr.base_needs_rex() && !adr.index_needs_rex(), "no extended registers");
1100   emit_operand(reg, adr._base, adr._index, adr._scale, adr._disp,
1101                adr._rspec);
1102 }
1103 
1104 void Assembler::emit_operand(Register reg, Address adr,
1105                              int rip_relative_correction) {
1106   emit_operand(reg, adr._base, adr._index, adr._scale, adr._disp,
1107                adr._rspec,
1108                rip_relative_correction);
1109 }
1110 
1111 void Assembler::emit_operand(XMMRegister reg, Address adr) {
1112   emit_operand(reg, adr._base, adr._index, adr._scale, adr._disp,
1113                adr._rspec);
1114 }
1115 
1116 // MMX operations
1117 void Assembler::emit_operand(MMXRegister reg, Address adr) {
1118   assert(!adr.base_needs_rex() && !adr.index_needs_rex(), "no extended registers");
1119   emit_operand((Register)reg, adr._base, adr._index, adr._scale, adr._disp, adr._rspec);
1120 }
1121 
1122 // work around gcc (3.2.1-7a) bug
1123 void Assembler::emit_operand(Address adr, MMXRegister reg) {
1124   assert(!adr.base_needs_rex() && !adr.index_needs_rex(), "no extended registers");
1125   emit_operand((Register)reg, adr._base, adr._index, adr._scale, adr._disp, adr._rspec);
1126 }
1127 
1128 
1129 void Assembler::emit_farith(int b1, int b2, int i) {
1130   assert(isByte(b1) && isByte(b2), "wrong opcode");
1131   assert(0 <= i &&  i < 8, "illegal stack offset");
1132   emit_int8(b1);
1133   emit_int8(b2 + i);
1134 }
1135 
1136 
1137 // Now the Assembler instructions (identical for 32/64 bits)
1138 
1139 void Assembler::adcl(Address dst, int32_t imm32) {
1140   InstructionMark im(this);
1141   prefix(dst);
1142   emit_arith_operand(0x81, rdx, dst, imm32);
1143 }
1144 
1145 void Assembler::adcl(Address dst, Register src) {
1146   InstructionMark im(this);
1147   prefix(dst, src);
1148   emit_int8(0x11);
1149   emit_operand(src, dst);
1150 }
1151 
1152 void Assembler::adcl(Register dst, int32_t imm32) {
1153   prefix(dst);
1154   emit_arith(0x81, 0xD0, dst, imm32);
1155 }
1156 
1157 void Assembler::adcl(Register dst, Address src) {
1158   InstructionMark im(this);
1159   prefix(src, dst);
1160   emit_int8(0x13);
1161   emit_operand(dst, src);
1162 }
1163 
1164 void Assembler::adcl(Register dst, Register src) {
1165   (void) prefix_and_encode(dst->encoding(), src->encoding());
1166   emit_arith(0x13, 0xC0, dst, src);
1167 }
1168 
1169 void Assembler::addl(Address dst, int32_t imm32) {
1170   InstructionMark im(this);
1171   prefix(dst);
1172   emit_arith_operand(0x81, rax, dst, imm32);
1173 }
1174 
1175 void Assembler::addb(Address dst, int imm8) {
1176   InstructionMark im(this);
1177   prefix(dst);
1178   emit_int8((unsigned char)0x80);
1179   emit_operand(rax, dst, 1);
1180   emit_int8(imm8);
1181 }
1182 
1183 void Assembler::addw(Address dst, int imm16) {
1184   InstructionMark im(this);
1185   emit_int8(0x66);
1186   prefix(dst);
1187   emit_int8((unsigned char)0x81);
1188   emit_operand(rax, dst, 2);
1189   emit_int16(imm16);
1190 }
1191 
1192 void Assembler::addl(Address dst, Register src) {
1193   InstructionMark im(this);
1194   prefix(dst, src);
1195   emit_int8(0x01);
1196   emit_operand(src, dst);
1197 }
1198 
1199 void Assembler::addl(Register dst, int32_t imm32) {
1200   prefix(dst);
1201   emit_arith(0x81, 0xC0, dst, imm32);
1202 }
1203 
1204 void Assembler::addl(Register dst, Address src) {
1205   InstructionMark im(this);
1206   prefix(src, dst);
1207   emit_int8(0x03);
1208   emit_operand(dst, src);
1209 }
1210 
1211 void Assembler::addl(Register dst, Register src) {
1212   (void) prefix_and_encode(dst->encoding(), src->encoding());
1213   emit_arith(0x03, 0xC0, dst, src);
1214 }
1215 
1216 void Assembler::addr_nop_4() {
1217   assert(UseAddressNop, "no CPU support");
1218   // 4 bytes: NOP DWORD PTR [EAX+0]
1219   emit_int8(0x0F);
1220   emit_int8(0x1F);
1221   emit_int8(0x40); // emit_rm(cbuf, 0x1, EAX_enc, EAX_enc);
1222   emit_int8(0);    // 8-bits offset (1 byte)
1223 }
1224 
1225 void Assembler::addr_nop_5() {
1226   assert(UseAddressNop, "no CPU support");
1227   // 5 bytes: NOP DWORD PTR [EAX+EAX*0+0] 8-bits offset
1228   emit_int8(0x0F);
1229   emit_int8(0x1F);
1230   emit_int8(0x44); // emit_rm(cbuf, 0x1, EAX_enc, 0x4);
1231   emit_int8(0x00); // emit_rm(cbuf, 0x0, EAX_enc, EAX_enc);
1232   emit_int8(0);    // 8-bits offset (1 byte)
1233 }
1234 
1235 void Assembler::addr_nop_7() {
1236   assert(UseAddressNop, "no CPU support");
1237   // 7 bytes: NOP DWORD PTR [EAX+0] 32-bits offset
1238   emit_int8(0x0F);
1239   emit_int8(0x1F);
1240   emit_int8((unsigned char)0x80);
1241                    // emit_rm(cbuf, 0x2, EAX_enc, EAX_enc);
1242   emit_int32(0);   // 32-bits offset (4 bytes)
1243 }
1244 
1245 void Assembler::addr_nop_8() {
1246   assert(UseAddressNop, "no CPU support");
1247   // 8 bytes: NOP DWORD PTR [EAX+EAX*0+0] 32-bits offset
1248   emit_int8(0x0F);
1249   emit_int8(0x1F);
1250   emit_int8((unsigned char)0x84);
1251                    // emit_rm(cbuf, 0x2, EAX_enc, 0x4);
1252   emit_int8(0x00); // emit_rm(cbuf, 0x0, EAX_enc, EAX_enc);
1253   emit_int32(0);   // 32-bits offset (4 bytes)
1254 }
1255 
1256 void Assembler::addsd(XMMRegister dst, XMMRegister src) {
1257   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1258   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
1259   attributes.set_rex_vex_w_reverted();
1260   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
1261   emit_int8(0x58);
1262   emit_int8((unsigned char)(0xC0 | encode));
1263 }
1264 
1265 void Assembler::addsd(XMMRegister dst, Address src) {
1266   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1267   InstructionMark im(this);
1268   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
1269   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
1270   attributes.set_rex_vex_w_reverted();
1271   simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
1272   emit_int8(0x58);
1273   emit_operand(dst, src);
1274 }
1275 
1276 void Assembler::addss(XMMRegister dst, XMMRegister src) {
1277   NOT_LP64(assert(VM_Version::supports_sse(), ""));
1278   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
1279   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
1280   emit_int8(0x58);
1281   emit_int8((unsigned char)(0xC0 | encode));
1282 }
1283 
1284 void Assembler::addss(XMMRegister dst, Address src) {
1285   NOT_LP64(assert(VM_Version::supports_sse(), ""));
1286   InstructionMark im(this);
1287   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
1288   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
1289   simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
1290   emit_int8(0x58);
1291   emit_operand(dst, src);
1292 }
1293 
1294 void Assembler::aesdec(XMMRegister dst, Address src) {
1295   assert(VM_Version::supports_aes(), "");
1296   InstructionMark im(this);
1297   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
1298   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
1299   emit_int8((unsigned char)0xDE);
1300   emit_operand(dst, src);
1301 }
1302 
1303 void Assembler::aesdec(XMMRegister dst, XMMRegister src) {
1304   assert(VM_Version::supports_aes(), "");
1305   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
1306   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
1307   emit_int8((unsigned char)0xDE);
1308   emit_int8(0xC0 | encode);
1309 }
1310 
1311 void Assembler::aesdeclast(XMMRegister dst, Address src) {
1312   assert(VM_Version::supports_aes(), "");
1313   InstructionMark im(this);
1314   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
1315   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
1316   emit_int8((unsigned char)0xDF);
1317   emit_operand(dst, src);
1318 }
1319 
1320 void Assembler::aesdeclast(XMMRegister dst, XMMRegister src) {
1321   assert(VM_Version::supports_aes(), "");
1322   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
1323   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
1324   emit_int8((unsigned char)0xDF);
1325   emit_int8((unsigned char)(0xC0 | encode));
1326 }
1327 
1328 void Assembler::aesenc(XMMRegister dst, Address src) {
1329   assert(VM_Version::supports_aes(), "");
1330   InstructionMark im(this);
1331   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
1332   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
1333   emit_int8((unsigned char)0xDC);
1334   emit_operand(dst, src);
1335 }
1336 
1337 void Assembler::aesenc(XMMRegister dst, XMMRegister src) {
1338   assert(VM_Version::supports_aes(), "");
1339   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
1340   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
1341   emit_int8((unsigned char)0xDC);
1342   emit_int8(0xC0 | encode);
1343 }
1344 
1345 void Assembler::aesenclast(XMMRegister dst, Address src) {
1346   assert(VM_Version::supports_aes(), "");
1347   InstructionMark im(this);
1348   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
1349   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
1350   emit_int8((unsigned char)0xDD);
1351   emit_operand(dst, src);
1352 }
1353 
1354 void Assembler::aesenclast(XMMRegister dst, XMMRegister src) {
1355   assert(VM_Version::supports_aes(), "");
1356   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
1357   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
1358   emit_int8((unsigned char)0xDD);
1359   emit_int8((unsigned char)(0xC0 | encode));
1360 }
1361 
1362 void Assembler::andl(Address dst, int32_t imm32) {
1363   InstructionMark im(this);
1364   prefix(dst);
1365   emit_int8((unsigned char)0x81);
1366   emit_operand(rsp, dst, 4);
1367   emit_int32(imm32);
1368 }
1369 
1370 void Assembler::andl(Register dst, int32_t imm32) {
1371   prefix(dst);
1372   emit_arith(0x81, 0xE0, dst, imm32);
1373 }
1374 
1375 void Assembler::andl(Register dst, Address src) {
1376   InstructionMark im(this);
1377   prefix(src, dst);
1378   emit_int8(0x23);
1379   emit_operand(dst, src);
1380 }
1381 
1382 void Assembler::andl(Register dst, Register src) {
1383   (void) prefix_and_encode(dst->encoding(), src->encoding());
1384   emit_arith(0x23, 0xC0, dst, src);
1385 }
1386 
1387 void Assembler::andnl(Register dst, Register src1, Register src2) {
1388   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1389   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
1390   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
1391   emit_int8((unsigned char)0xF2);
1392   emit_int8((unsigned char)(0xC0 | encode));
1393 }
1394 
1395 void Assembler::andnl(Register dst, Register src1, Address src2) {
1396   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1397   InstructionMark im(this);
1398   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
1399   vex_prefix(src2, src1->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
1400   emit_int8((unsigned char)0xF2);
1401   emit_operand(dst, src2);
1402 }
1403 
1404 void Assembler::bsfl(Register dst, Register src) {
1405   int encode = prefix_and_encode(dst->encoding(), src->encoding());
1406   emit_int8(0x0F);
1407   emit_int8((unsigned char)0xBC);
1408   emit_int8((unsigned char)(0xC0 | encode));
1409 }
1410 
1411 void Assembler::bsrl(Register dst, Register src) {
1412   int encode = prefix_and_encode(dst->encoding(), src->encoding());
1413   emit_int8(0x0F);
1414   emit_int8((unsigned char)0xBD);
1415   emit_int8((unsigned char)(0xC0 | encode));
1416 }
1417 
1418 void Assembler::bswapl(Register reg) { // bswap
1419   int encode = prefix_and_encode(reg->encoding());
1420   emit_int8(0x0F);
1421   emit_int8((unsigned char)(0xC8 | encode));
1422 }
1423 
1424 void Assembler::blsil(Register dst, Register src) {
1425   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1426   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
1427   int encode = vex_prefix_and_encode(rbx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
1428   emit_int8((unsigned char)0xF3);
1429   emit_int8((unsigned char)(0xC0 | encode));
1430 }
1431 
1432 void Assembler::blsil(Register dst, Address src) {
1433   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1434   InstructionMark im(this);
1435   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
1436   vex_prefix(src, dst->encoding(), rbx->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
1437   emit_int8((unsigned char)0xF3);
1438   emit_operand(rbx, src);
1439 }
1440 
1441 void Assembler::blsmskl(Register dst, Register src) {
1442   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1443   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
1444   int encode = vex_prefix_and_encode(rdx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
1445   emit_int8((unsigned char)0xF3);
1446   emit_int8((unsigned char)(0xC0 | encode));
1447 }
1448 
1449 void Assembler::blsmskl(Register dst, Address src) {
1450   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1451   InstructionMark im(this);
1452   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
1453   vex_prefix(src, dst->encoding(), rdx->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
1454   emit_int8((unsigned char)0xF3);
1455   emit_operand(rdx, src);
1456 }
1457 
1458 void Assembler::blsrl(Register dst, Register src) {
1459   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1460   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
1461   int encode = vex_prefix_and_encode(rcx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
1462   emit_int8((unsigned char)0xF3);
1463   emit_int8((unsigned char)(0xC0 | encode));
1464 }
1465 
1466 void Assembler::blsrl(Register dst, Address src) {
1467   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1468   InstructionMark im(this);
1469   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
1470   vex_prefix(src, dst->encoding(), rcx->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
1471   emit_int8((unsigned char)0xF3);
1472   emit_operand(rcx, src);
1473 }
1474 
1475 void Assembler::call(Label& L, relocInfo::relocType rtype) {
1476   // suspect disp32 is always good
1477   int operand = LP64_ONLY(disp32_operand) NOT_LP64(imm_operand);
1478 
1479   if (L.is_bound()) {
1480     const int long_size = 5;
1481     int offs = (int)( target(L) - pc() );
1482     assert(offs <= 0, "assembler error");
1483     InstructionMark im(this);
1484     // 1110 1000 #32-bit disp
1485     emit_int8((unsigned char)0xE8);
1486     emit_data(offs - long_size, rtype, operand);
1487   } else {
1488     InstructionMark im(this);
1489     // 1110 1000 #32-bit disp
1490     L.add_patch_at(code(), locator());
1491 
1492     emit_int8((unsigned char)0xE8);
1493     emit_data(int(0), rtype, operand);
1494   }
1495 }
1496 
1497 void Assembler::call(Register dst) {
1498   int encode = prefix_and_encode(dst->encoding());
1499   emit_int8((unsigned char)0xFF);
1500   emit_int8((unsigned char)(0xD0 | encode));
1501 }
1502 
1503 
1504 void Assembler::call(Address adr) {
1505   InstructionMark im(this);
1506   prefix(adr);
1507   emit_int8((unsigned char)0xFF);
1508   emit_operand(rdx, adr);
1509 }
1510 
1511 void Assembler::call_literal(address entry, RelocationHolder const& rspec) {
1512   assert(entry != NULL, "call most probably wrong");
1513   InstructionMark im(this);
1514   emit_int8((unsigned char)0xE8);
1515   intptr_t disp = entry - (pc() + sizeof(int32_t));
1516   assert(is_simm32(disp), "must be 32bit offset (call2)");
1517   // Technically, should use call32_operand, but this format is
1518   // implied by the fact that we're emitting a call instruction.
1519 
1520   int operand = LP64_ONLY(disp32_operand) NOT_LP64(call32_operand);
1521   emit_data((int) disp, rspec, operand);
1522 }
1523 
1524 void Assembler::cdql() {
1525   emit_int8((unsigned char)0x99);
1526 }
1527 
1528 void Assembler::cld() {
1529   emit_int8((unsigned char)0xFC);
1530 }
1531 
1532 void Assembler::cmovl(Condition cc, Register dst, Register src) {
1533   NOT_LP64(guarantee(VM_Version::supports_cmov(), "illegal instruction"));
1534   int encode = prefix_and_encode(dst->encoding(), src->encoding());
1535   emit_int8(0x0F);
1536   emit_int8(0x40 | cc);
1537   emit_int8((unsigned char)(0xC0 | encode));
1538 }
1539 
1540 
1541 void Assembler::cmovl(Condition cc, Register dst, Address src) {
1542   NOT_LP64(guarantee(VM_Version::supports_cmov(), "illegal instruction"));
1543   prefix(src, dst);
1544   emit_int8(0x0F);
1545   emit_int8(0x40 | cc);
1546   emit_operand(dst, src);
1547 }
1548 
1549 void Assembler::cmpb(Address dst, int imm8) {
1550   InstructionMark im(this);
1551   prefix(dst);
1552   emit_int8((unsigned char)0x80);
1553   emit_operand(rdi, dst, 1);
1554   emit_int8(imm8);
1555 }
1556 
1557 void Assembler::cmpl(Address dst, int32_t imm32) {
1558   InstructionMark im(this);
1559   prefix(dst);
1560   emit_int8((unsigned char)0x81);
1561   emit_operand(rdi, dst, 4);
1562   emit_int32(imm32);
1563 }
1564 
1565 void Assembler::cmpl(Register dst, int32_t imm32) {
1566   prefix(dst);
1567   emit_arith(0x81, 0xF8, dst, imm32);
1568 }
1569 
1570 void Assembler::cmpl(Register dst, Register src) {
1571   (void) prefix_and_encode(dst->encoding(), src->encoding());
1572   emit_arith(0x3B, 0xC0, dst, src);
1573 }
1574 
1575 void Assembler::cmpl(Register dst, Address  src) {
1576   InstructionMark im(this);
1577   prefix(src, dst);
1578   emit_int8((unsigned char)0x3B);
1579   emit_operand(dst, src);
1580 }
1581 
1582 void Assembler::cmpw(Address dst, int imm16) {
1583   InstructionMark im(this);
1584   assert(!dst.base_needs_rex() && !dst.index_needs_rex(), "no extended registers");
1585   emit_int8(0x66);
1586   emit_int8((unsigned char)0x81);
1587   emit_operand(rdi, dst, 2);
1588   emit_int16(imm16);
1589 }
1590 
1591 // The 32-bit cmpxchg compares the value at adr with the contents of rax,
1592 // and stores reg into adr if so; otherwise, the value at adr is loaded into rax,.
1593 // The ZF is set if the compared values were equal, and cleared otherwise.
1594 void Assembler::cmpxchgl(Register reg, Address adr) { // cmpxchg
1595   InstructionMark im(this);
1596   prefix(adr, reg);
1597   emit_int8(0x0F);
1598   emit_int8((unsigned char)0xB1);
1599   emit_operand(reg, adr);
1600 }
1601 
1602 // The 8-bit cmpxchg compares the value at adr with the contents of rax,
1603 // and stores reg into adr if so; otherwise, the value at adr is loaded into rax,.
1604 // The ZF is set if the compared values were equal, and cleared otherwise.
1605 void Assembler::cmpxchgb(Register reg, Address adr) { // cmpxchg
1606   InstructionMark im(this);
1607   prefix(adr, reg, true);
1608   emit_int8(0x0F);
1609   emit_int8((unsigned char)0xB0);
1610   emit_operand(reg, adr);
1611 }
1612 
1613 void Assembler::comisd(XMMRegister dst, Address src) {
1614   // NOTE: dbx seems to decode this as comiss even though the
1615   // 0x66 is there. Strangly ucomisd comes out correct
1616   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1617   InstructionMark im(this);
1618   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);;
1619   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
1620   attributes.set_rex_vex_w_reverted();
1621   simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
1622   emit_int8(0x2F);
1623   emit_operand(dst, src);
1624 }
1625 
1626 void Assembler::comisd(XMMRegister dst, XMMRegister src) {
1627   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1628   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1629   attributes.set_rex_vex_w_reverted();
1630   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
1631   emit_int8(0x2F);
1632   emit_int8((unsigned char)(0xC0 | encode));
1633 }
1634 
1635 void Assembler::comiss(XMMRegister dst, Address src) {
1636   NOT_LP64(assert(VM_Version::supports_sse(), ""));
1637   InstructionMark im(this);
1638   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1639   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
1640   simd_prefix(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
1641   emit_int8(0x2F);
1642   emit_operand(dst, src);
1643 }
1644 
1645 void Assembler::comiss(XMMRegister dst, XMMRegister src) {
1646   NOT_LP64(assert(VM_Version::supports_sse(), ""));
1647   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1648   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
1649   emit_int8(0x2F);
1650   emit_int8((unsigned char)(0xC0 | encode));
1651 }
1652 
1653 void Assembler::cpuid() {
1654   emit_int8(0x0F);
1655   emit_int8((unsigned char)0xA2);
1656 }
1657 
1658 // Opcode / Instruction                      Op /  En  64 - Bit Mode     Compat / Leg Mode Description                  Implemented
1659 // F2 0F 38 F0 / r       CRC32 r32, r / m8   RM        Valid             Valid             Accumulate CRC32 on r / m8.  v
1660 // F2 REX 0F 38 F0 / r   CRC32 r32, r / m8*  RM        Valid             N.E.              Accumulate CRC32 on r / m8.  -
1661 // F2 REX.W 0F 38 F0 / r CRC32 r64, r / m8   RM        Valid             N.E.              Accumulate CRC32 on r / m8.  -
1662 //
1663 // F2 0F 38 F1 / r       CRC32 r32, r / m16  RM        Valid             Valid             Accumulate CRC32 on r / m16. v
1664 //
1665 // F2 0F 38 F1 / r       CRC32 r32, r / m32  RM        Valid             Valid             Accumulate CRC32 on r / m32. v
1666 //
1667 // F2 REX.W 0F 38 F1 / r CRC32 r64, r / m64  RM        Valid             N.E.              Accumulate CRC32 on r / m64. v
1668 void Assembler::crc32(Register crc, Register v, int8_t sizeInBytes) {
1669   assert(VM_Version::supports_sse4_2(), "");
1670   int8_t w = 0x01;
1671   Prefix p = Prefix_EMPTY;
1672 
1673   emit_int8((int8_t)0xF2);
1674   switch (sizeInBytes) {
1675   case 1:
1676     w = 0;
1677     break;
1678   case 2:
1679   case 4:
1680     break;
1681   LP64_ONLY(case 8:)
1682     // This instruction is not valid in 32 bits
1683     // Note:
1684     // http://www.intel.com/content/dam/www/public/us/en/documents/manuals/64-ia-32-architectures-software-developer-instruction-set-reference-manual-325383.pdf
1685     //
1686     // Page B - 72   Vol. 2C says
1687     // qwreg2 to qwreg            1111 0010 : 0100 1R0B : 0000 1111 : 0011 1000 : 1111 0000 : 11 qwreg1 qwreg2
1688     // mem64 to qwreg             1111 0010 : 0100 1R0B : 0000 1111 : 0011 1000 : 1111 0000 : mod qwreg r / m
1689     //                                                                            F0!!!
1690     // while 3 - 208 Vol. 2A
1691     // F2 REX.W 0F 38 F1 / r       CRC32 r64, r / m64             RM         Valid      N.E.Accumulate CRC32 on r / m64.
1692     //
1693     // the 0 on a last bit is reserved for a different flavor of this instruction :
1694     // F2 REX.W 0F 38 F0 / r       CRC32 r64, r / m8              RM         Valid      N.E.Accumulate CRC32 on r / m8.
1695     p = REX_W;
1696     break;
1697   default:
1698     assert(0, "Unsupported value for a sizeInBytes argument");
1699     break;
1700   }
1701   LP64_ONLY(prefix(crc, v, p);)
1702   emit_int8((int8_t)0x0F);
1703   emit_int8(0x38);
1704   emit_int8((int8_t)(0xF0 | w));
1705   emit_int8(0xC0 | ((crc->encoding() & 0x7) << 3) | (v->encoding() & 7));
1706 }
1707 
1708 void Assembler::crc32(Register crc, Address adr, int8_t sizeInBytes) {
1709   assert(VM_Version::supports_sse4_2(), "");
1710   InstructionMark im(this);
1711   int8_t w = 0x01;
1712   Prefix p = Prefix_EMPTY;
1713 
1714   emit_int8((int8_t)0xF2);
1715   switch (sizeInBytes) {
1716   case 1:
1717     w = 0;
1718     break;
1719   case 2:
1720   case 4:
1721     break;
1722   LP64_ONLY(case 8:)
1723     // This instruction is not valid in 32 bits
1724     p = REX_W;
1725     break;
1726   default:
1727     assert(0, "Unsupported value for a sizeInBytes argument");
1728     break;
1729   }
1730   LP64_ONLY(prefix(crc, adr, p);)
1731   emit_int8((int8_t)0x0F);
1732   emit_int8(0x38);
1733   emit_int8((int8_t)(0xF0 | w));
1734   emit_operand(crc, adr);
1735 }
1736 
1737 void Assembler::cvtdq2pd(XMMRegister dst, XMMRegister src) {
1738   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1739   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
1740   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
1741   emit_int8((unsigned char)0xE6);
1742   emit_int8((unsigned char)(0xC0 | encode));
1743 }
1744 
1745 void Assembler::cvtdq2ps(XMMRegister dst, XMMRegister src) {
1746   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1747   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
1748   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
1749   emit_int8(0x5B);
1750   emit_int8((unsigned char)(0xC0 | encode));
1751 }
1752 
1753 void Assembler::cvtsd2ss(XMMRegister dst, XMMRegister src) {
1754   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1755   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
1756   attributes.set_rex_vex_w_reverted();
1757   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
1758   emit_int8(0x5A);
1759   emit_int8((unsigned char)(0xC0 | encode));
1760 }
1761 
1762 void Assembler::cvtsd2ss(XMMRegister dst, Address src) {
1763   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1764   InstructionMark im(this);
1765   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
1766   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
1767   attributes.set_rex_vex_w_reverted();
1768   simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
1769   emit_int8(0x5A);
1770   emit_operand(dst, src);
1771 }
1772 
1773 void Assembler::cvtsi2sdl(XMMRegister dst, Register src) {
1774   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1775   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1776   int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
1777   emit_int8(0x2A);
1778   emit_int8((unsigned char)(0xC0 | encode));
1779 }
1780 
1781 void Assembler::cvtsi2sdl(XMMRegister dst, Address src) {
1782   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1783   InstructionMark im(this);
1784   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1785   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
1786   simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
1787   emit_int8(0x2A);
1788   emit_operand(dst, src);
1789 }
1790 
1791 void Assembler::cvtsi2ssl(XMMRegister dst, Register src) {
1792   NOT_LP64(assert(VM_Version::supports_sse(), ""));
1793   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1794   int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
1795   emit_int8(0x2A);
1796   emit_int8((unsigned char)(0xC0 | encode));
1797 }
1798 
1799 void Assembler::cvtsi2ssl(XMMRegister dst, Address src) {
1800   NOT_LP64(assert(VM_Version::supports_sse(), ""));
1801   InstructionMark im(this);
1802   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1803   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
1804   simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
1805   emit_int8(0x2A);
1806   emit_operand(dst, src);
1807 }
1808 
1809 void Assembler::cvtsi2ssq(XMMRegister dst, Register src) {
1810   NOT_LP64(assert(VM_Version::supports_sse(), ""));
1811   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1812   int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
1813   emit_int8(0x2A);
1814   emit_int8((unsigned char)(0xC0 | encode));
1815 }
1816 
1817 void Assembler::cvtss2sd(XMMRegister dst, XMMRegister src) {
1818   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1819   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
1820   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
1821   emit_int8(0x5A);
1822   emit_int8((unsigned char)(0xC0 | encode));
1823 }
1824 
1825 void Assembler::cvtss2sd(XMMRegister dst, Address src) {
1826   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1827   InstructionMark im(this);
1828   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
1829   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
1830   simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
1831   emit_int8(0x5A);
1832   emit_operand(dst, src);
1833 }
1834 
1835 
1836 void Assembler::cvttsd2sil(Register dst, XMMRegister src) {
1837   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1838   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1839   int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
1840   emit_int8(0x2C);
1841   emit_int8((unsigned char)(0xC0 | encode));
1842 }
1843 
1844 void Assembler::cvttss2sil(Register dst, XMMRegister src) {
1845   NOT_LP64(assert(VM_Version::supports_sse(), ""));
1846   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1847   int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
1848   emit_int8(0x2C);
1849   emit_int8((unsigned char)(0xC0 | encode));
1850 }
1851 
1852 void Assembler::cvttpd2dq(XMMRegister dst, XMMRegister src) {
1853   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1854   int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_128bit;
1855   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
1856   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
1857   emit_int8((unsigned char)0xE6);
1858   emit_int8((unsigned char)(0xC0 | encode));
1859 }
1860 
1861 void Assembler::decl(Address dst) {
1862   // Don't use it directly. Use MacroAssembler::decrement() instead.
1863   InstructionMark im(this);
1864   prefix(dst);
1865   emit_int8((unsigned char)0xFF);
1866   emit_operand(rcx, dst);
1867 }
1868 
1869 void Assembler::divsd(XMMRegister dst, Address src) {
1870   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1871   InstructionMark im(this);
1872   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
1873   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
1874   attributes.set_rex_vex_w_reverted();
1875   simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
1876   emit_int8(0x5E);
1877   emit_operand(dst, src);
1878 }
1879 
1880 void Assembler::divsd(XMMRegister dst, XMMRegister src) {
1881   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1882   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
1883   attributes.set_rex_vex_w_reverted();
1884   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
1885   emit_int8(0x5E);
1886   emit_int8((unsigned char)(0xC0 | encode));
1887 }
1888 
1889 void Assembler::divss(XMMRegister dst, Address src) {
1890   NOT_LP64(assert(VM_Version::supports_sse(), ""));
1891   InstructionMark im(this);
1892   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
1893   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
1894   simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
1895   emit_int8(0x5E);
1896   emit_operand(dst, src);
1897 }
1898 
1899 void Assembler::divss(XMMRegister dst, XMMRegister src) {
1900   NOT_LP64(assert(VM_Version::supports_sse(), ""));
1901   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
1902   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
1903   emit_int8(0x5E);
1904   emit_int8((unsigned char)(0xC0 | encode));
1905 }
1906 
1907 void Assembler::emms() {
1908   NOT_LP64(assert(VM_Version::supports_mmx(), ""));
1909   emit_int8(0x0F);
1910   emit_int8(0x77);
1911 }
1912 
1913 void Assembler::hlt() {
1914   emit_int8((unsigned char)0xF4);
1915 }
1916 
1917 void Assembler::idivl(Register src) {
1918   int encode = prefix_and_encode(src->encoding());
1919   emit_int8((unsigned char)0xF7);
1920   emit_int8((unsigned char)(0xF8 | encode));
1921 }
1922 
1923 void Assembler::divl(Register src) { // Unsigned
1924   int encode = prefix_and_encode(src->encoding());
1925   emit_int8((unsigned char)0xF7);
1926   emit_int8((unsigned char)(0xF0 | encode));
1927 }
1928 
1929 void Assembler::imull(Register src) {
1930   int encode = prefix_and_encode(src->encoding());
1931   emit_int8((unsigned char)0xF7);
1932   emit_int8((unsigned char)(0xE8 | encode));
1933 }
1934 
1935 void Assembler::imull(Register dst, Register src) {
1936   int encode = prefix_and_encode(dst->encoding(), src->encoding());
1937   emit_int8(0x0F);
1938   emit_int8((unsigned char)0xAF);
1939   emit_int8((unsigned char)(0xC0 | encode));
1940 }
1941 
1942 
1943 void Assembler::imull(Register dst, Register src, int value) {
1944   int encode = prefix_and_encode(dst->encoding(), src->encoding());
1945   if (is8bit(value)) {
1946     emit_int8(0x6B);
1947     emit_int8((unsigned char)(0xC0 | encode));
1948     emit_int8(value & 0xFF);
1949   } else {
1950     emit_int8(0x69);
1951     emit_int8((unsigned char)(0xC0 | encode));
1952     emit_int32(value);
1953   }
1954 }
1955 
1956 void Assembler::imull(Register dst, Address src) {
1957   InstructionMark im(this);
1958   prefix(src, dst);
1959   emit_int8(0x0F);
1960   emit_int8((unsigned char) 0xAF);
1961   emit_operand(dst, src);
1962 }
1963 
1964 
1965 void Assembler::incl(Address dst) {
1966   // Don't use it directly. Use MacroAssembler::increment() instead.
1967   InstructionMark im(this);
1968   prefix(dst);
1969   emit_int8((unsigned char)0xFF);
1970   emit_operand(rax, dst);
1971 }
1972 
1973 void Assembler::jcc(Condition cc, Label& L, bool maybe_short) {
1974   InstructionMark im(this);
1975   assert((0 <= cc) && (cc < 16), "illegal cc");
1976   if (L.is_bound()) {
1977     address dst = target(L);
1978     assert(dst != NULL, "jcc most probably wrong");
1979 
1980     const int short_size = 2;
1981     const int long_size = 6;
1982     intptr_t offs = (intptr_t)dst - (intptr_t)pc();
1983     if (maybe_short && is8bit(offs - short_size)) {
1984       // 0111 tttn #8-bit disp
1985       emit_int8(0x70 | cc);
1986       emit_int8((offs - short_size) & 0xFF);
1987     } else {
1988       // 0000 1111 1000 tttn #32-bit disp
1989       assert(is_simm32(offs - long_size),
1990              "must be 32bit offset (call4)");
1991       emit_int8(0x0F);
1992       emit_int8((unsigned char)(0x80 | cc));
1993       emit_int32(offs - long_size);
1994     }
1995   } else {
1996     // Note: could eliminate cond. jumps to this jump if condition
1997     //       is the same however, seems to be rather unlikely case.
1998     // Note: use jccb() if label to be bound is very close to get
1999     //       an 8-bit displacement
2000     L.add_patch_at(code(), locator());
2001     emit_int8(0x0F);
2002     emit_int8((unsigned char)(0x80 | cc));
2003     emit_int32(0);
2004   }
2005 }
2006 
2007 void Assembler::jccb(Condition cc, Label& L) {
2008   if (L.is_bound()) {
2009     const int short_size = 2;
2010     address entry = target(L);
2011 #ifdef ASSERT
2012     intptr_t dist = (intptr_t)entry - ((intptr_t)pc() + short_size);
2013     intptr_t delta = short_branch_delta();
2014     if (delta != 0) {
2015       dist += (dist < 0 ? (-delta) :delta);
2016     }
2017     assert(is8bit(dist), "Dispacement too large for a short jmp");
2018 #endif
2019     intptr_t offs = (intptr_t)entry - (intptr_t)pc();
2020     // 0111 tttn #8-bit disp
2021     emit_int8(0x70 | cc);
2022     emit_int8((offs - short_size) & 0xFF);
2023   } else {
2024     InstructionMark im(this);
2025     L.add_patch_at(code(), locator());
2026     emit_int8(0x70 | cc);
2027     emit_int8(0);
2028   }
2029 }
2030 
2031 void Assembler::jmp(Address adr) {
2032   InstructionMark im(this);
2033   prefix(adr);
2034   emit_int8((unsigned char)0xFF);
2035   emit_operand(rsp, adr);
2036 }
2037 
2038 void Assembler::jmp(Label& L, bool maybe_short) {
2039   if (L.is_bound()) {
2040     address entry = target(L);
2041     assert(entry != NULL, "jmp most probably wrong");
2042     InstructionMark im(this);
2043     const int short_size = 2;
2044     const int long_size = 5;
2045     intptr_t offs = entry - pc();
2046     if (maybe_short && is8bit(offs - short_size)) {
2047       emit_int8((unsigned char)0xEB);
2048       emit_int8((offs - short_size) & 0xFF);
2049     } else {
2050       emit_int8((unsigned char)0xE9);
2051       emit_int32(offs - long_size);
2052     }
2053   } else {
2054     // By default, forward jumps are always 32-bit displacements, since
2055     // we can't yet know where the label will be bound.  If you're sure that
2056     // the forward jump will not run beyond 256 bytes, use jmpb to
2057     // force an 8-bit displacement.
2058     InstructionMark im(this);
2059     L.add_patch_at(code(), locator());
2060     emit_int8((unsigned char)0xE9);
2061     emit_int32(0);
2062   }
2063 }
2064 
2065 void Assembler::jmp(Register entry) {
2066   int encode = prefix_and_encode(entry->encoding());
2067   emit_int8((unsigned char)0xFF);
2068   emit_int8((unsigned char)(0xE0 | encode));
2069 }
2070 
2071 void Assembler::jmp_literal(address dest, RelocationHolder const& rspec) {
2072   InstructionMark im(this);
2073   emit_int8((unsigned char)0xE9);
2074   assert(dest != NULL, "must have a target");
2075   intptr_t disp = dest - (pc() + sizeof(int32_t));
2076   assert(is_simm32(disp), "must be 32bit offset (jmp)");
2077   emit_data(disp, rspec.reloc(), call32_operand);
2078 }
2079 
2080 void Assembler::jmpb(Label& L) {
2081   if (L.is_bound()) {
2082     const int short_size = 2;
2083     address entry = target(L);
2084     assert(entry != NULL, "jmp most probably wrong");
2085 #ifdef ASSERT
2086     intptr_t dist = (intptr_t)entry - ((intptr_t)pc() + short_size);
2087     intptr_t delta = short_branch_delta();
2088     if (delta != 0) {
2089       dist += (dist < 0 ? (-delta) :delta);
2090     }
2091     assert(is8bit(dist), "Dispacement too large for a short jmp");
2092 #endif
2093     intptr_t offs = entry - pc();
2094     emit_int8((unsigned char)0xEB);
2095     emit_int8((offs - short_size) & 0xFF);
2096   } else {
2097     InstructionMark im(this);
2098     L.add_patch_at(code(), locator());
2099     emit_int8((unsigned char)0xEB);
2100     emit_int8(0);
2101   }
2102 }
2103 
2104 void Assembler::ldmxcsr( Address src) {
2105   NOT_LP64(assert(VM_Version::supports_sse(), ""));
2106   InstructionMark im(this);
2107   prefix(src);
2108   emit_int8(0x0F);
2109   emit_int8((unsigned char)0xAE);
2110   emit_operand(as_Register(2), src);
2111 }
2112 
2113 void Assembler::leal(Register dst, Address src) {
2114   InstructionMark im(this);
2115 #ifdef _LP64
2116   emit_int8(0x67); // addr32
2117   prefix(src, dst);
2118 #endif // LP64
2119   emit_int8((unsigned char)0x8D);
2120   emit_operand(dst, src);
2121 }
2122 
2123 void Assembler::lfence() {
2124   emit_int8(0x0F);
2125   emit_int8((unsigned char)0xAE);
2126   emit_int8((unsigned char)0xE8);
2127 }
2128 
2129 void Assembler::lock() {
2130   emit_int8((unsigned char)0xF0);
2131 }
2132 
2133 void Assembler::lzcntl(Register dst, Register src) {
2134   assert(VM_Version::supports_lzcnt(), "encoding is treated as BSR");
2135   emit_int8((unsigned char)0xF3);
2136   int encode = prefix_and_encode(dst->encoding(), src->encoding());
2137   emit_int8(0x0F);
2138   emit_int8((unsigned char)0xBD);
2139   emit_int8((unsigned char)(0xC0 | encode));
2140 }
2141 
2142 // Emit mfence instruction
2143 void Assembler::mfence() {
2144   NOT_LP64(assert(VM_Version::supports_sse2(), "unsupported");)
2145   emit_int8(0x0F);
2146   emit_int8((unsigned char)0xAE);
2147   emit_int8((unsigned char)0xF0);
2148 }
2149 
2150 void Assembler::mov(Register dst, Register src) {
2151   LP64_ONLY(movq(dst, src)) NOT_LP64(movl(dst, src));
2152 }
2153 
2154 void Assembler::movapd(XMMRegister dst, XMMRegister src) {
2155   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2156   int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_128bit;
2157   InstructionAttr attributes(vector_len, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
2158   attributes.set_rex_vex_w_reverted();
2159   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2160   emit_int8(0x28);
2161   emit_int8((unsigned char)(0xC0 | encode));
2162 }
2163 
2164 void Assembler::movaps(XMMRegister dst, XMMRegister src) {
2165   NOT_LP64(assert(VM_Version::supports_sse(), ""));
2166   int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_128bit;
2167   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
2168   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2169   emit_int8(0x28);
2170   emit_int8((unsigned char)(0xC0 | encode));
2171 }
2172 
2173 void Assembler::movlhps(XMMRegister dst, XMMRegister src) {
2174   NOT_LP64(assert(VM_Version::supports_sse(), ""));
2175   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2176   int encode = simd_prefix_and_encode(dst, src, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2177   emit_int8(0x16);
2178   emit_int8((unsigned char)(0xC0 | encode));
2179 }
2180 
2181 void Assembler::movb(Register dst, Address src) {
2182   NOT_LP64(assert(dst->has_byte_register(), "must have byte register"));
2183   InstructionMark im(this);
2184   prefix(src, dst, true);
2185   emit_int8((unsigned char)0x8A);
2186   emit_operand(dst, src);
2187 }
2188 
2189 void Assembler::movddup(XMMRegister dst, XMMRegister src) {
2190   NOT_LP64(assert(VM_Version::supports_sse3(), ""));
2191   int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_128bit;
2192   InstructionAttr attributes(vector_len, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2193   attributes.set_rex_vex_w_reverted();
2194   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2195   emit_int8(0x12);
2196   emit_int8(0xC0 | encode);
2197 }
2198 
2199 void Assembler::kmovbl(KRegister dst, Register src) {
2200   assert(VM_Version::supports_avx512dq(), "");
2201   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2202   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2203   emit_int8((unsigned char)0x92);
2204   emit_int8((unsigned char)(0xC0 | encode));
2205 }
2206 
2207 void Assembler::kmovbl(Register dst, KRegister src) {
2208   assert(VM_Version::supports_avx512dq(), "");
2209   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2210   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2211   emit_int8((unsigned char)0x93);
2212   emit_int8((unsigned char)(0xC0 | encode));
2213 }
2214 
2215 void Assembler::kmovwl(KRegister dst, Register src) {
2216   assert(VM_Version::supports_evex(), "");
2217   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2218   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2219   emit_int8((unsigned char)0x92);
2220   emit_int8((unsigned char)(0xC0 | encode));
2221 }
2222 
2223 void Assembler::kmovwl(Register dst, KRegister src) {
2224   assert(VM_Version::supports_evex(), "");
2225   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2226   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2227   emit_int8((unsigned char)0x93);
2228   emit_int8((unsigned char)(0xC0 | encode));
2229 }
2230 
2231 void Assembler::kmovwl(KRegister dst, Address src) {
2232   assert(VM_Version::supports_evex(), "");
2233   InstructionMark im(this);
2234   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2235   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2236   emit_int8((unsigned char)0x90);
2237   emit_operand((Register)dst, src);
2238 }
2239 
2240 void Assembler::kmovdl(KRegister dst, Register src) {
2241   assert(VM_Version::supports_avx512bw(), "");
2242   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2243   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2244   emit_int8((unsigned char)0x92);
2245   emit_int8((unsigned char)(0xC0 | encode));
2246 }
2247 
2248 void Assembler::kmovdl(Register dst, KRegister src) {
2249   assert(VM_Version::supports_avx512bw(), "");
2250   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2251   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2252   emit_int8((unsigned char)0x93);
2253   emit_int8((unsigned char)(0xC0 | encode));
2254 }
2255 
2256 void Assembler::kmovql(KRegister dst, KRegister src) {
2257   assert(VM_Version::supports_avx512bw(), "");
2258   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2259   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2260   emit_int8((unsigned char)0x90);
2261   emit_int8((unsigned char)(0xC0 | encode));
2262 }
2263 
2264 void Assembler::kmovql(KRegister dst, Address src) {
2265   assert(VM_Version::supports_avx512bw(), "");
2266   InstructionMark im(this);
2267   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2268   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2269   emit_int8((unsigned char)0x90);
2270   emit_operand((Register)dst, src);
2271 }
2272 
2273 void Assembler::kmovql(Address dst, KRegister src) {
2274   assert(VM_Version::supports_avx512bw(), "");
2275   InstructionMark im(this);
2276   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2277   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2278   emit_int8((unsigned char)0x90);
2279   emit_operand((Register)src, dst);
2280 }
2281 
2282 void Assembler::kmovql(KRegister dst, Register src) {
2283   assert(VM_Version::supports_avx512bw(), "");
2284   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2285   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2286   emit_int8((unsigned char)0x92);
2287   emit_int8((unsigned char)(0xC0 | encode));
2288 }
2289 
2290 void Assembler::kmovql(Register dst, KRegister src) {
2291   assert(VM_Version::supports_avx512bw(), "");
2292   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2293   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2294   emit_int8((unsigned char)0x93);
2295   emit_int8((unsigned char)(0xC0 | encode));
2296 }
2297 
2298 void Assembler::knotwl(KRegister dst, KRegister src) {
2299   assert(VM_Version::supports_evex(), "");
2300   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2301   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2302   emit_int8((unsigned char)0x44);
2303   emit_int8((unsigned char)(0xC0 | encode));
2304 }
2305 
2306 // This instruction produces ZF or CF flags
2307 void Assembler::kortestbl(KRegister src1, KRegister src2) {
2308   assert(VM_Version::supports_avx512dq(), "");
2309   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2310   int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2311   emit_int8((unsigned char)0x98);
2312   emit_int8((unsigned char)(0xC0 | encode));
2313 }
2314 
2315 // This instruction produces ZF or CF flags
2316 void Assembler::kortestwl(KRegister src1, KRegister src2) {
2317   assert(VM_Version::supports_evex(), "");
2318   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2319   int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2320   emit_int8((unsigned char)0x98);
2321   emit_int8((unsigned char)(0xC0 | encode));
2322 }
2323 
2324 // This instruction produces ZF or CF flags
2325 void Assembler::kortestdl(KRegister src1, KRegister src2) {
2326   assert(VM_Version::supports_avx512bw(), "");
2327   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2328   int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2329   emit_int8((unsigned char)0x98);
2330   emit_int8((unsigned char)(0xC0 | encode));
2331 }
2332 
2333 // This instruction produces ZF or CF flags
2334 void Assembler::kortestql(KRegister src1, KRegister src2) {
2335   assert(VM_Version::supports_avx512bw(), "");
2336   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2337   int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2338   emit_int8((unsigned char)0x98);
2339   emit_int8((unsigned char)(0xC0 | encode));
2340 }
2341 
2342 // This instruction produces ZF or CF flags
2343 void Assembler::ktestql(KRegister src1, KRegister src2) {
2344   assert(VM_Version::supports_avx512bw(), "");
2345   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2346   int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2347   emit_int8((unsigned char)0x99);
2348   emit_int8((unsigned char)(0xC0 | encode));
2349 }
2350 
2351 void Assembler::ktestq(KRegister src1, KRegister src2) {
2352   assert(VM_Version::supports_avx512bw(), "");
2353   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2354   int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2355   emit_int8((unsigned char)0x99);
2356   emit_int8((unsigned char)(0xC0 | encode));
2357 }
2358 
2359 void Assembler::ktestd(KRegister src1, KRegister src2) {
2360   assert(VM_Version::supports_avx512bw(), "");
2361   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2362   int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2363   emit_int8((unsigned char)0x99);
2364   emit_int8((unsigned char)(0xC0 | encode));
2365 }
2366 
2367 void Assembler::movb(Address dst, int imm8) {
2368   InstructionMark im(this);
2369    prefix(dst);
2370   emit_int8((unsigned char)0xC6);
2371   emit_operand(rax, dst, 1);
2372   emit_int8(imm8);
2373 }
2374 
2375 
2376 void Assembler::movb(Address dst, Register src) {
2377   assert(src->has_byte_register(), "must have byte register");
2378   InstructionMark im(this);
2379   prefix(dst, src, true);
2380   emit_int8((unsigned char)0x88);
2381   emit_operand(src, dst);
2382 }
2383 
2384 void Assembler::movdl(XMMRegister dst, Register src) {
2385   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2386   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2387   int encode = simd_prefix_and_encode(dst, xnoreg, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2388   emit_int8(0x6E);
2389   emit_int8((unsigned char)(0xC0 | encode));
2390 }
2391 
2392 void Assembler::movdl(Register dst, XMMRegister src) {
2393   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2394   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2395   // swap src/dst to get correct prefix
2396   int encode = simd_prefix_and_encode(src, xnoreg, as_XMMRegister(dst->encoding()), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2397   emit_int8(0x7E);
2398   emit_int8((unsigned char)(0xC0 | encode));
2399 }
2400 
2401 void Assembler::movdl(XMMRegister dst, Address src) {
2402   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2403   InstructionMark im(this);
2404   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2405   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
2406   simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2407   emit_int8(0x6E);
2408   emit_operand(dst, src);
2409 }
2410 
2411 void Assembler::movdl(Address dst, XMMRegister src) {
2412   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2413   InstructionMark im(this);
2414   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2415   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
2416   simd_prefix(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2417   emit_int8(0x7E);
2418   emit_operand(src, dst);
2419 }
2420 
2421 void Assembler::movdqa(XMMRegister dst, XMMRegister src) {
2422   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2423   int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_128bit;
2424   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2425   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2426   emit_int8(0x6F);
2427   emit_int8((unsigned char)(0xC0 | encode));
2428 }
2429 
2430 void Assembler::movdqa(XMMRegister dst, Address src) {
2431   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2432   InstructionMark im(this);
2433   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2434   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2435   simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2436   emit_int8(0x6F);
2437   emit_operand(dst, src);
2438 }
2439 
2440 void Assembler::movdqu(XMMRegister dst, Address src) {
2441   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2442   InstructionMark im(this);
2443   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2444   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2445   simd_prefix(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2446   emit_int8(0x6F);
2447   emit_operand(dst, src);
2448 }
2449 
2450 void Assembler::movdqu(XMMRegister dst, XMMRegister src) {
2451   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2452   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2453   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2454   emit_int8(0x6F);
2455   emit_int8((unsigned char)(0xC0 | encode));
2456 }
2457 
2458 void Assembler::movdqu(Address dst, XMMRegister src) {
2459   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2460   InstructionMark im(this);
2461   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2462   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2463   simd_prefix(src, xnoreg, dst, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2464   emit_int8(0x7F);
2465   emit_operand(src, dst);
2466 }
2467 
2468 // Move Unaligned 256bit Vector
2469 void Assembler::vmovdqu(XMMRegister dst, XMMRegister src) {
2470   assert(UseAVX > 0, "");
2471   InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2472   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2473   emit_int8(0x6F);
2474   emit_int8((unsigned char)(0xC0 | encode));
2475 }
2476 
2477 void Assembler::vmovdqu(XMMRegister dst, Address src) {
2478   assert(UseAVX > 0, "");
2479   InstructionMark im(this);
2480   InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2481   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2482   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2483   emit_int8(0x6F);
2484   emit_operand(dst, src);
2485 }
2486 
2487 void Assembler::vmovdqu(Address dst, XMMRegister src) {
2488   assert(UseAVX > 0, "");
2489   InstructionMark im(this);
2490   InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2491   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2492   // swap src<->dst for encoding
2493   assert(src != xnoreg, "sanity");
2494   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2495   emit_int8(0x7F);
2496   emit_operand(src, dst);
2497 }
2498 
2499 // Move Unaligned EVEX enabled Vector (programmable : 8,16,32,64)
2500 void Assembler::evmovdqub(XMMRegister dst, XMMRegister src, int vector_len) {
2501   assert(VM_Version::supports_evex(), "");
2502   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
2503   attributes.set_is_evex_instruction();
2504   int prefix = (_legacy_mode_bw) ? VEX_SIMD_F2 : VEX_SIMD_F3;
2505   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), (Assembler::VexSimdPrefix)prefix, VEX_OPCODE_0F, &attributes);
2506   emit_int8(0x6F);
2507   emit_int8((unsigned char)(0xC0 | encode));
2508 }
2509 
2510 void Assembler::evmovdqub(XMMRegister dst, Address src, int vector_len) {
2511   assert(VM_Version::supports_evex(), "");
2512   InstructionMark im(this);
2513   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
2514   int prefix = (_legacy_mode_bw) ? VEX_SIMD_F2 : VEX_SIMD_F3;
2515   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2516   attributes.set_is_evex_instruction();
2517   vex_prefix(src, 0, dst->encoding(), (Assembler::VexSimdPrefix)prefix, VEX_OPCODE_0F, &attributes);
2518   emit_int8(0x6F);
2519   emit_operand(dst, src);
2520 }
2521 
2522 void Assembler::evmovdqub(Address dst, XMMRegister src, int vector_len) {
2523   assert(VM_Version::supports_evex(), "");
2524   assert(src != xnoreg, "sanity");
2525   InstructionMark im(this);
2526   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
2527   int prefix = (_legacy_mode_bw) ? VEX_SIMD_F2 : VEX_SIMD_F3;
2528   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2529   attributes.set_is_evex_instruction();
2530   vex_prefix(dst, 0, src->encoding(), (Assembler::VexSimdPrefix)prefix, VEX_OPCODE_0F, &attributes);
2531   emit_int8(0x7F);
2532   emit_operand(src, dst);
2533 }
2534 
2535 void Assembler::evmovdqub(XMMRegister dst, KRegister mask, Address src, int vector_len) {
2536   assert(VM_Version::supports_avx512vlbw(), "");
2537   assert(is_vector_masking(), "");    // For stub code use only
2538   InstructionMark im(this);
2539   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
2540   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2541   attributes.set_embedded_opmask_register_specifier(mask);
2542   attributes.set_is_evex_instruction();
2543   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2544   emit_int8(0x6F);
2545   emit_operand(dst, src);
2546 }
2547 
2548 void Assembler::evmovdquw(XMMRegister dst, Address src, int vector_len) {
2549   assert(VM_Version::supports_evex(), "");
2550   InstructionMark im(this);
2551   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
2552   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2553   attributes.set_is_evex_instruction();
2554   int prefix = (_legacy_mode_bw) ? VEX_SIMD_F2 : VEX_SIMD_F3;
2555   vex_prefix(src, 0, dst->encoding(), (Assembler::VexSimdPrefix)prefix, VEX_OPCODE_0F, &attributes);
2556   emit_int8(0x6F);
2557   emit_operand(dst, src);
2558 }
2559 
2560 void Assembler::evmovdquw(XMMRegister dst, KRegister mask, Address src, int vector_len) {
2561   assert(is_vector_masking(), "");
2562   assert(VM_Version::supports_avx512vlbw(), "");
2563   InstructionMark im(this);
2564   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
2565   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2566   attributes.set_embedded_opmask_register_specifier(mask);
2567   attributes.set_is_evex_instruction();
2568   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2569   emit_int8(0x6F);
2570   emit_operand(dst, src);
2571 }
2572 
2573 void Assembler::evmovdquw(Address dst, XMMRegister src, int vector_len) {
2574   assert(VM_Version::supports_evex(), "");
2575   assert(src != xnoreg, "sanity");
2576   InstructionMark im(this);
2577   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
2578   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2579   attributes.set_is_evex_instruction();
2580   int prefix = (_legacy_mode_bw) ? VEX_SIMD_F2 : VEX_SIMD_F3;
2581   vex_prefix(dst, 0, src->encoding(), (Assembler::VexSimdPrefix)prefix, VEX_OPCODE_0F, &attributes);
2582   emit_int8(0x7F);
2583   emit_operand(src, dst);
2584 }
2585 
2586 void Assembler::evmovdquw(Address dst, KRegister mask, XMMRegister src, int vector_len) {
2587   assert(VM_Version::supports_avx512vlbw(), "");
2588   assert(src != xnoreg, "sanity");
2589   InstructionMark im(this);
2590   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2591   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2592   attributes.set_embedded_opmask_register_specifier(mask);
2593   attributes.set_is_evex_instruction();
2594   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2595   emit_int8(0x7F);
2596   emit_operand(src, dst);
2597 }
2598 
2599 void Assembler::evmovdqul(XMMRegister dst, XMMRegister src, int vector_len) {
2600   assert(VM_Version::supports_evex(), "");
2601   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2602   attributes.set_is_evex_instruction();
2603   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2604   emit_int8(0x6F);
2605   emit_int8((unsigned char)(0xC0 | encode));
2606 }
2607 
2608 void Assembler::evmovdqul(XMMRegister dst, Address src, int vector_len) {
2609   assert(VM_Version::supports_evex(), "");
2610   InstructionMark im(this);
2611   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false , /* uses_vl */ true);
2612   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2613   attributes.set_is_evex_instruction();
2614   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2615   emit_int8(0x6F);
2616   emit_operand(dst, src);
2617 }
2618 
2619 void Assembler::evmovdqul(Address dst, XMMRegister src, int vector_len) {
2620   assert(VM_Version::supports_evex(), "");
2621   assert(src != xnoreg, "sanity");
2622   InstructionMark im(this);
2623   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2624   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2625   attributes.set_is_evex_instruction();
2626   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2627   emit_int8(0x7F);
2628   emit_operand(src, dst);
2629 }
2630 
2631 void Assembler::evmovdquq(XMMRegister dst, XMMRegister src, int vector_len) {
2632   assert(VM_Version::supports_evex(), "");
2633   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2634   attributes.set_is_evex_instruction();
2635   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2636   emit_int8(0x6F);
2637   emit_int8((unsigned char)(0xC0 | encode));
2638 }
2639 
2640 void Assembler::evmovdquq(XMMRegister dst, Address src, int vector_len) {
2641   assert(VM_Version::supports_evex(), "");
2642   InstructionMark im(this);
2643   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2644   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2645   attributes.set_is_evex_instruction();
2646   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2647   emit_int8(0x6F);
2648   emit_operand(dst, src);
2649 }
2650 
2651 void Assembler::evmovdquq(Address dst, XMMRegister src, int vector_len) {
2652   assert(VM_Version::supports_evex(), "");
2653   assert(src != xnoreg, "sanity");
2654   InstructionMark im(this);
2655   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2656   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2657   attributes.set_is_evex_instruction();
2658   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2659   emit_int8(0x7F);
2660   emit_operand(src, dst);
2661 }
2662 
2663 // Uses zero extension on 64bit
2664 
2665 void Assembler::movl(Register dst, int32_t imm32) {
2666   int encode = prefix_and_encode(dst->encoding());
2667   emit_int8((unsigned char)(0xB8 | encode));
2668   emit_int32(imm32);
2669 }
2670 
2671 void Assembler::movl(Register dst, Register src) {
2672   int encode = prefix_and_encode(dst->encoding(), src->encoding());
2673   emit_int8((unsigned char)0x8B);
2674   emit_int8((unsigned char)(0xC0 | encode));
2675 }
2676 
2677 void Assembler::movl(Register dst, Address src) {
2678   InstructionMark im(this);
2679   prefix(src, dst);
2680   emit_int8((unsigned char)0x8B);
2681   emit_operand(dst, src);
2682 }
2683 
2684 void Assembler::movl(Address dst, int32_t imm32) {
2685   InstructionMark im(this);
2686   prefix(dst);
2687   emit_int8((unsigned char)0xC7);
2688   emit_operand(rax, dst, 4);
2689   emit_int32(imm32);
2690 }
2691 
2692 void Assembler::movl(Address dst, Register src) {
2693   InstructionMark im(this);
2694   prefix(dst, src);
2695   emit_int8((unsigned char)0x89);
2696   emit_operand(src, dst);
2697 }
2698 
2699 // New cpus require to use movsd and movss to avoid partial register stall
2700 // when loading from memory. But for old Opteron use movlpd instead of movsd.
2701 // The selection is done in MacroAssembler::movdbl() and movflt().
2702 void Assembler::movlpd(XMMRegister dst, Address src) {
2703   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2704   InstructionMark im(this);
2705   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2706   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
2707   attributes.set_rex_vex_w_reverted();
2708   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2709   emit_int8(0x12);
2710   emit_operand(dst, src);
2711 }
2712 
2713 void Assembler::movq( MMXRegister dst, Address src ) {
2714   assert( VM_Version::supports_mmx(), "" );
2715   emit_int8(0x0F);
2716   emit_int8(0x6F);
2717   emit_operand(dst, src);
2718 }
2719 
2720 void Assembler::movq( Address dst, MMXRegister src ) {
2721   assert( VM_Version::supports_mmx(), "" );
2722   emit_int8(0x0F);
2723   emit_int8(0x7F);
2724   // workaround gcc (3.2.1-7a) bug
2725   // In that version of gcc with only an emit_operand(MMX, Address)
2726   // gcc will tail jump and try and reverse the parameters completely
2727   // obliterating dst in the process. By having a version available
2728   // that doesn't need to swap the args at the tail jump the bug is
2729   // avoided.
2730   emit_operand(dst, src);
2731 }
2732 
2733 void Assembler::movq(XMMRegister dst, Address src) {
2734   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2735   InstructionMark im(this);
2736   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2737   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
2738   attributes.set_rex_vex_w_reverted();
2739   simd_prefix(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2740   emit_int8(0x7E);
2741   emit_operand(dst, src);
2742 }
2743 
2744 void Assembler::movq(Address dst, XMMRegister src) {
2745   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2746   InstructionMark im(this);
2747   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2748   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
2749   attributes.set_rex_vex_w_reverted();
2750   simd_prefix(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2751   emit_int8((unsigned char)0xD6);
2752   emit_operand(src, dst);
2753 }
2754 
2755 void Assembler::movsbl(Register dst, Address src) { // movsxb
2756   InstructionMark im(this);
2757   prefix(src, dst);
2758   emit_int8(0x0F);
2759   emit_int8((unsigned char)0xBE);
2760   emit_operand(dst, src);
2761 }
2762 
2763 void Assembler::movsbl(Register dst, Register src) { // movsxb
2764   NOT_LP64(assert(src->has_byte_register(), "must have byte register"));
2765   int encode = prefix_and_encode(dst->encoding(), false, src->encoding(), true);
2766   emit_int8(0x0F);
2767   emit_int8((unsigned char)0xBE);
2768   emit_int8((unsigned char)(0xC0 | encode));
2769 }
2770 
2771 void Assembler::movsd(XMMRegister dst, XMMRegister src) {
2772   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2773   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
2774   attributes.set_rex_vex_w_reverted();
2775   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2776   emit_int8(0x10);
2777   emit_int8((unsigned char)(0xC0 | encode));
2778 }
2779 
2780 void Assembler::movsd(XMMRegister dst, Address src) {
2781   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2782   InstructionMark im(this);
2783   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
2784   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
2785   attributes.set_rex_vex_w_reverted();
2786   simd_prefix(dst, xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2787   emit_int8(0x10);
2788   emit_operand(dst, src);
2789 }
2790 
2791 void Assembler::movsd(Address dst, XMMRegister src) {
2792   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2793   InstructionMark im(this);
2794   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
2795   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
2796   attributes.set_rex_vex_w_reverted();
2797   simd_prefix(src, xnoreg, dst, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2798   emit_int8(0x11);
2799   emit_operand(src, dst);
2800 }
2801 
2802 void Assembler::movss(XMMRegister dst, XMMRegister src) {
2803   NOT_LP64(assert(VM_Version::supports_sse(), ""));
2804   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
2805   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2806   emit_int8(0x10);
2807   emit_int8((unsigned char)(0xC0 | encode));
2808 }
2809 
2810 void Assembler::movss(XMMRegister dst, Address src) {
2811   NOT_LP64(assert(VM_Version::supports_sse(), ""));
2812   InstructionMark im(this);
2813   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
2814   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
2815   simd_prefix(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2816   emit_int8(0x10);
2817   emit_operand(dst, src);
2818 }
2819 
2820 void Assembler::movss(Address dst, XMMRegister src) {
2821   NOT_LP64(assert(VM_Version::supports_sse(), ""));
2822   InstructionMark im(this);
2823   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
2824   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
2825   simd_prefix(src, xnoreg, dst, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2826   emit_int8(0x11);
2827   emit_operand(src, dst);
2828 }
2829 
2830 void Assembler::movswl(Register dst, Address src) { // movsxw
2831   InstructionMark im(this);
2832   prefix(src, dst);
2833   emit_int8(0x0F);
2834   emit_int8((unsigned char)0xBF);
2835   emit_operand(dst, src);
2836 }
2837 
2838 void Assembler::movswl(Register dst, Register src) { // movsxw
2839   int encode = prefix_and_encode(dst->encoding(), src->encoding());
2840   emit_int8(0x0F);
2841   emit_int8((unsigned char)0xBF);
2842   emit_int8((unsigned char)(0xC0 | encode));
2843 }
2844 
2845 void Assembler::movw(Address dst, int imm16) {
2846   InstructionMark im(this);
2847 
2848   emit_int8(0x66); // switch to 16-bit mode
2849   prefix(dst);
2850   emit_int8((unsigned char)0xC7);
2851   emit_operand(rax, dst, 2);
2852   emit_int16(imm16);
2853 }
2854 
2855 void Assembler::movw(Register dst, Address src) {
2856   InstructionMark im(this);
2857   emit_int8(0x66);
2858   prefix(src, dst);
2859   emit_int8((unsigned char)0x8B);
2860   emit_operand(dst, src);
2861 }
2862 
2863 void Assembler::movw(Address dst, Register src) {
2864   InstructionMark im(this);
2865   emit_int8(0x66);
2866   prefix(dst, src);
2867   emit_int8((unsigned char)0x89);
2868   emit_operand(src, dst);
2869 }
2870 
2871 void Assembler::movzbl(Register dst, Address src) { // movzxb
2872   InstructionMark im(this);
2873   prefix(src, dst);
2874   emit_int8(0x0F);
2875   emit_int8((unsigned char)0xB6);
2876   emit_operand(dst, src);
2877 }
2878 
2879 void Assembler::movzbl(Register dst, Register src) { // movzxb
2880   NOT_LP64(assert(src->has_byte_register(), "must have byte register"));
2881   int encode = prefix_and_encode(dst->encoding(), false, src->encoding(), true);
2882   emit_int8(0x0F);
2883   emit_int8((unsigned char)0xB6);
2884   emit_int8(0xC0 | encode);
2885 }
2886 
2887 void Assembler::movzwl(Register dst, Address src) { // movzxw
2888   InstructionMark im(this);
2889   prefix(src, dst);
2890   emit_int8(0x0F);
2891   emit_int8((unsigned char)0xB7);
2892   emit_operand(dst, src);
2893 }
2894 
2895 void Assembler::movzwl(Register dst, Register src) { // movzxw
2896   int encode = prefix_and_encode(dst->encoding(), src->encoding());
2897   emit_int8(0x0F);
2898   emit_int8((unsigned char)0xB7);
2899   emit_int8(0xC0 | encode);
2900 }
2901 
2902 void Assembler::mull(Address src) {
2903   InstructionMark im(this);
2904   prefix(src);
2905   emit_int8((unsigned char)0xF7);
2906   emit_operand(rsp, src);
2907 }
2908 
2909 void Assembler::mull(Register src) {
2910   int encode = prefix_and_encode(src->encoding());
2911   emit_int8((unsigned char)0xF7);
2912   emit_int8((unsigned char)(0xE0 | encode));
2913 }
2914 
2915 void Assembler::mulsd(XMMRegister dst, Address src) {
2916   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2917   InstructionMark im(this);
2918   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
2919   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
2920   attributes.set_rex_vex_w_reverted();
2921   simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2922   emit_int8(0x59);
2923   emit_operand(dst, src);
2924 }
2925 
2926 void Assembler::mulsd(XMMRegister dst, XMMRegister src) {
2927   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2928   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
2929   attributes.set_rex_vex_w_reverted();
2930   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2931   emit_int8(0x59);
2932   emit_int8((unsigned char)(0xC0 | encode));
2933 }
2934 
2935 void Assembler::mulss(XMMRegister dst, Address src) {
2936   NOT_LP64(assert(VM_Version::supports_sse(), ""));
2937   InstructionMark im(this);
2938   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
2939   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
2940   simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2941   emit_int8(0x59);
2942   emit_operand(dst, src);
2943 }
2944 
2945 void Assembler::mulss(XMMRegister dst, XMMRegister src) {
2946   NOT_LP64(assert(VM_Version::supports_sse(), ""));
2947   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
2948   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2949   emit_int8(0x59);
2950   emit_int8((unsigned char)(0xC0 | encode));
2951 }
2952 
2953 void Assembler::negl(Register dst) {
2954   int encode = prefix_and_encode(dst->encoding());
2955   emit_int8((unsigned char)0xF7);
2956   emit_int8((unsigned char)(0xD8 | encode));
2957 }
2958 
2959 void Assembler::nop(int i) {
2960 #ifdef ASSERT
2961   assert(i > 0, " ");
2962   // The fancy nops aren't currently recognized by debuggers making it a
2963   // pain to disassemble code while debugging. If asserts are on clearly
2964   // speed is not an issue so simply use the single byte traditional nop
2965   // to do alignment.
2966 
2967   for (; i > 0 ; i--) emit_int8((unsigned char)0x90);
2968   return;
2969 
2970 #endif // ASSERT
2971 
2972   if (UseAddressNop && VM_Version::is_intel()) {
2973     //
2974     // Using multi-bytes nops "0x0F 0x1F [address]" for Intel
2975     //  1: 0x90
2976     //  2: 0x66 0x90
2977     //  3: 0x66 0x66 0x90 (don't use "0x0F 0x1F 0x00" - need patching safe padding)
2978     //  4: 0x0F 0x1F 0x40 0x00
2979     //  5: 0x0F 0x1F 0x44 0x00 0x00
2980     //  6: 0x66 0x0F 0x1F 0x44 0x00 0x00
2981     //  7: 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00
2982     //  8: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
2983     //  9: 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
2984     // 10: 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
2985     // 11: 0x66 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
2986 
2987     // The rest coding is Intel specific - don't use consecutive address nops
2988 
2989     // 12: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
2990     // 13: 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
2991     // 14: 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
2992     // 15: 0x66 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
2993 
2994     while(i >= 15) {
2995       // For Intel don't generate consecutive addess nops (mix with regular nops)
2996       i -= 15;
2997       emit_int8(0x66);   // size prefix
2998       emit_int8(0x66);   // size prefix
2999       emit_int8(0x66);   // size prefix
3000       addr_nop_8();
3001       emit_int8(0x66);   // size prefix
3002       emit_int8(0x66);   // size prefix
3003       emit_int8(0x66);   // size prefix
3004       emit_int8((unsigned char)0x90);
3005                          // nop
3006     }
3007     switch (i) {
3008       case 14:
3009         emit_int8(0x66); // size prefix
3010       case 13:
3011         emit_int8(0x66); // size prefix
3012       case 12:
3013         addr_nop_8();
3014         emit_int8(0x66); // size prefix
3015         emit_int8(0x66); // size prefix
3016         emit_int8(0x66); // size prefix
3017         emit_int8((unsigned char)0x90);
3018                          // nop
3019         break;
3020       case 11:
3021         emit_int8(0x66); // size prefix
3022       case 10:
3023         emit_int8(0x66); // size prefix
3024       case 9:
3025         emit_int8(0x66); // size prefix
3026       case 8:
3027         addr_nop_8();
3028         break;
3029       case 7:
3030         addr_nop_7();
3031         break;
3032       case 6:
3033         emit_int8(0x66); // size prefix
3034       case 5:
3035         addr_nop_5();
3036         break;
3037       case 4:
3038         addr_nop_4();
3039         break;
3040       case 3:
3041         // Don't use "0x0F 0x1F 0x00" - need patching safe padding
3042         emit_int8(0x66); // size prefix
3043       case 2:
3044         emit_int8(0x66); // size prefix
3045       case 1:
3046         emit_int8((unsigned char)0x90);
3047                          // nop
3048         break;
3049       default:
3050         assert(i == 0, " ");
3051     }
3052     return;
3053   }
3054   if (UseAddressNop && VM_Version::is_amd()) {
3055     //
3056     // Using multi-bytes nops "0x0F 0x1F [address]" for AMD.
3057     //  1: 0x90
3058     //  2: 0x66 0x90
3059     //  3: 0x66 0x66 0x90 (don't use "0x0F 0x1F 0x00" - need patching safe padding)
3060     //  4: 0x0F 0x1F 0x40 0x00
3061     //  5: 0x0F 0x1F 0x44 0x00 0x00
3062     //  6: 0x66 0x0F 0x1F 0x44 0x00 0x00
3063     //  7: 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00
3064     //  8: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
3065     //  9: 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
3066     // 10: 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
3067     // 11: 0x66 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
3068 
3069     // The rest coding is AMD specific - use consecutive address nops
3070 
3071     // 12: 0x66 0x0F 0x1F 0x44 0x00 0x00 0x66 0x0F 0x1F 0x44 0x00 0x00
3072     // 13: 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00 0x66 0x0F 0x1F 0x44 0x00 0x00
3073     // 14: 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00
3074     // 15: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00
3075     // 16: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
3076     //     Size prefixes (0x66) are added for larger sizes
3077 
3078     while(i >= 22) {
3079       i -= 11;
3080       emit_int8(0x66); // size prefix
3081       emit_int8(0x66); // size prefix
3082       emit_int8(0x66); // size prefix
3083       addr_nop_8();
3084     }
3085     // Generate first nop for size between 21-12
3086     switch (i) {
3087       case 21:
3088         i -= 1;
3089         emit_int8(0x66); // size prefix
3090       case 20:
3091       case 19:
3092         i -= 1;
3093         emit_int8(0x66); // size prefix
3094       case 18:
3095       case 17:
3096         i -= 1;
3097         emit_int8(0x66); // size prefix
3098       case 16:
3099       case 15:
3100         i -= 8;
3101         addr_nop_8();
3102         break;
3103       case 14:
3104       case 13:
3105         i -= 7;
3106         addr_nop_7();
3107         break;
3108       case 12:
3109         i -= 6;
3110         emit_int8(0x66); // size prefix
3111         addr_nop_5();
3112         break;
3113       default:
3114         assert(i < 12, " ");
3115     }
3116 
3117     // Generate second nop for size between 11-1
3118     switch (i) {
3119       case 11:
3120         emit_int8(0x66); // size prefix
3121       case 10:
3122         emit_int8(0x66); // size prefix
3123       case 9:
3124         emit_int8(0x66); // size prefix
3125       case 8:
3126         addr_nop_8();
3127         break;
3128       case 7:
3129         addr_nop_7();
3130         break;
3131       case 6:
3132         emit_int8(0x66); // size prefix
3133       case 5:
3134         addr_nop_5();
3135         break;
3136       case 4:
3137         addr_nop_4();
3138         break;
3139       case 3:
3140         // Don't use "0x0F 0x1F 0x00" - need patching safe padding
3141         emit_int8(0x66); // size prefix
3142       case 2:
3143         emit_int8(0x66); // size prefix
3144       case 1:
3145         emit_int8((unsigned char)0x90);
3146                          // nop
3147         break;
3148       default:
3149         assert(i == 0, " ");
3150     }
3151     return;
3152   }
3153 
3154   // Using nops with size prefixes "0x66 0x90".
3155   // From AMD Optimization Guide:
3156   //  1: 0x90
3157   //  2: 0x66 0x90
3158   //  3: 0x66 0x66 0x90
3159   //  4: 0x66 0x66 0x66 0x90
3160   //  5: 0x66 0x66 0x90 0x66 0x90
3161   //  6: 0x66 0x66 0x90 0x66 0x66 0x90
3162   //  7: 0x66 0x66 0x66 0x90 0x66 0x66 0x90
3163   //  8: 0x66 0x66 0x66 0x90 0x66 0x66 0x66 0x90
3164   //  9: 0x66 0x66 0x90 0x66 0x66 0x90 0x66 0x66 0x90
3165   // 10: 0x66 0x66 0x66 0x90 0x66 0x66 0x90 0x66 0x66 0x90
3166   //
3167   while(i > 12) {
3168     i -= 4;
3169     emit_int8(0x66); // size prefix
3170     emit_int8(0x66);
3171     emit_int8(0x66);
3172     emit_int8((unsigned char)0x90);
3173                      // nop
3174   }
3175   // 1 - 12 nops
3176   if(i > 8) {
3177     if(i > 9) {
3178       i -= 1;
3179       emit_int8(0x66);
3180     }
3181     i -= 3;
3182     emit_int8(0x66);
3183     emit_int8(0x66);
3184     emit_int8((unsigned char)0x90);
3185   }
3186   // 1 - 8 nops
3187   if(i > 4) {
3188     if(i > 6) {
3189       i -= 1;
3190       emit_int8(0x66);
3191     }
3192     i -= 3;
3193     emit_int8(0x66);
3194     emit_int8(0x66);
3195     emit_int8((unsigned char)0x90);
3196   }
3197   switch (i) {
3198     case 4:
3199       emit_int8(0x66);
3200     case 3:
3201       emit_int8(0x66);
3202     case 2:
3203       emit_int8(0x66);
3204     case 1:
3205       emit_int8((unsigned char)0x90);
3206       break;
3207     default:
3208       assert(i == 0, " ");
3209   }
3210 }
3211 
3212 void Assembler::notl(Register dst) {
3213   int encode = prefix_and_encode(dst->encoding());
3214   emit_int8((unsigned char)0xF7);
3215   emit_int8((unsigned char)(0xD0 | encode));
3216 }
3217 
3218 void Assembler::orl(Address dst, int32_t imm32) {
3219   InstructionMark im(this);
3220   prefix(dst);
3221   emit_arith_operand(0x81, rcx, dst, imm32);
3222 }
3223 
3224 void Assembler::orl(Register dst, int32_t imm32) {
3225   prefix(dst);
3226   emit_arith(0x81, 0xC8, dst, imm32);
3227 }
3228 
3229 void Assembler::orl(Register dst, Address src) {
3230   InstructionMark im(this);
3231   prefix(src, dst);
3232   emit_int8(0x0B);
3233   emit_operand(dst, src);
3234 }
3235 
3236 void Assembler::orl(Register dst, Register src) {
3237   (void) prefix_and_encode(dst->encoding(), src->encoding());
3238   emit_arith(0x0B, 0xC0, dst, src);
3239 }
3240 
3241 void Assembler::orl(Address dst, Register src) {
3242   InstructionMark im(this);
3243   prefix(dst, src);
3244   emit_int8(0x09);
3245   emit_operand(src, dst);
3246 }
3247 
3248 void Assembler::packuswb(XMMRegister dst, Address src) {
3249   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3250   assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
3251   InstructionMark im(this);
3252   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
3253   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
3254   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3255   emit_int8(0x67);
3256   emit_operand(dst, src);
3257 }
3258 
3259 void Assembler::packuswb(XMMRegister dst, XMMRegister src) {
3260   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3261   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
3262   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3263   emit_int8(0x67);
3264   emit_int8((unsigned char)(0xC0 | encode));
3265 }
3266 
3267 void Assembler::vpackuswb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
3268   assert(UseAVX > 0, "some form of AVX must be enabled");
3269   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
3270   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3271   emit_int8(0x67);
3272   emit_int8((unsigned char)(0xC0 | encode));
3273 }
3274 
3275 void Assembler::vpermq(XMMRegister dst, XMMRegister src, int imm8, int vector_len) {
3276   assert(VM_Version::supports_avx2(), "");
3277   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
3278   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3279   emit_int8(0x00);
3280   emit_int8(0xC0 | encode);
3281   emit_int8(imm8);
3282 }
3283 
3284 void Assembler::vperm2i128(XMMRegister dst,  XMMRegister nds, XMMRegister src, int imm8) {
3285   assert(VM_Version::supports_avx2(), "");
3286   InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3287   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3288   emit_int8(0x46);
3289   emit_int8(0xC0 | encode);
3290   emit_int8(imm8);
3291 }
3292 
3293 
3294 void Assembler::pause() {
3295   emit_int8((unsigned char)0xF3);
3296   emit_int8((unsigned char)0x90);
3297 }
3298 
3299 void Assembler::pcmpestri(XMMRegister dst, Address src, int imm8) {
3300   assert(VM_Version::supports_sse4_2(), "");
3301   InstructionMark im(this);
3302   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3303   simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3304   emit_int8(0x61);
3305   emit_operand(dst, src);
3306   emit_int8(imm8);
3307 }
3308 
3309 void Assembler::pcmpestri(XMMRegister dst, XMMRegister src, int imm8) {
3310   assert(VM_Version::supports_sse4_2(), "");
3311   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3312   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3313   emit_int8(0x61);
3314   emit_int8((unsigned char)(0xC0 | encode));
3315   emit_int8(imm8);
3316 }
3317 
3318 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
3319 void Assembler::pcmpeqb(XMMRegister dst, XMMRegister src) {
3320   assert(VM_Version::supports_sse2(), "");
3321   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3322   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3323   emit_int8(0x74);
3324   emit_int8((unsigned char)(0xC0 | encode));
3325 }
3326 
3327 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
3328 void Assembler::vpcmpeqb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
3329   assert(VM_Version::supports_avx(), "");
3330   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3331   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3332   emit_int8(0x74);
3333   emit_int8((unsigned char)(0xC0 | encode));
3334 }
3335 
3336 // In this context, kdst is written the mask used to process the equal components
3337 void Assembler::evpcmpeqb(KRegister kdst, XMMRegister nds, XMMRegister src, int vector_len) {
3338   assert(VM_Version::supports_avx512bw(), "");
3339   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
3340   attributes.set_is_evex_instruction();
3341   int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3342   emit_int8(0x74);
3343   emit_int8((unsigned char)(0xC0 | encode));
3344 }
3345 
3346 void Assembler::evpcmpgtb(KRegister kdst, XMMRegister nds, Address src, int vector_len) {
3347   assert(VM_Version::supports_avx512vlbw(), "");
3348   InstructionMark im(this);
3349   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3350   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
3351   attributes.set_is_evex_instruction();
3352   int dst_enc = kdst->encoding();
3353   vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3354   emit_int8(0x64);
3355   emit_operand(as_Register(dst_enc), src);
3356 }
3357 
3358 void Assembler::evpcmpgtb(KRegister kdst, KRegister mask, XMMRegister nds, Address src, int vector_len) {
3359   assert(is_vector_masking(), "");
3360   assert(VM_Version::supports_avx512vlbw(), "");
3361   InstructionMark im(this);
3362   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
3363   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
3364   attributes.set_embedded_opmask_register_specifier(mask);
3365   attributes.set_is_evex_instruction();
3366   int dst_enc = kdst->encoding();
3367   vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3368   emit_int8(0x64);
3369   emit_operand(as_Register(dst_enc), src);
3370 }
3371 
3372 void Assembler::evpcmpuw(KRegister kdst, XMMRegister nds, XMMRegister src, ComparisonPredicate vcc, int vector_len) {
3373   assert(VM_Version::supports_avx512vlbw(), "");
3374   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3375   attributes.set_is_evex_instruction();
3376   int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3377   emit_int8(0x3E);
3378   emit_int8((unsigned char)(0xC0 | encode));
3379   emit_int8(vcc);
3380 }
3381 
3382 void Assembler::evpcmpuw(KRegister kdst, KRegister mask, XMMRegister nds, XMMRegister src, ComparisonPredicate vcc, int vector_len) {
3383   assert(is_vector_masking(), "");
3384   assert(VM_Version::supports_avx512vlbw(), "");
3385   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
3386   attributes.set_embedded_opmask_register_specifier(mask);
3387   attributes.set_is_evex_instruction();
3388   int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3389   emit_int8(0x3E);
3390   emit_int8((unsigned char)(0xC0 | encode));
3391   emit_int8(vcc);
3392 }
3393 
3394 void Assembler::evpcmpuw(KRegister kdst, XMMRegister nds, Address src, ComparisonPredicate vcc, int vector_len) {
3395   assert(VM_Version::supports_avx512vlbw(), "");
3396   InstructionMark im(this);
3397   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3398   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
3399   attributes.set_is_evex_instruction();
3400   int dst_enc = kdst->encoding();
3401   vex_prefix(src, nds->encoding(), kdst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3402   emit_int8(0x3E);
3403   emit_operand(as_Register(dst_enc), src);
3404   emit_int8(vcc);
3405 }
3406 
3407 void Assembler::evpcmpeqb(KRegister kdst, XMMRegister nds, Address src, int vector_len) {
3408   assert(VM_Version::supports_avx512bw(), "");
3409   InstructionMark im(this);
3410   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3411   attributes.set_is_evex_instruction();
3412   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
3413   int dst_enc = kdst->encoding();
3414   vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3415   emit_int8(0x74);
3416   emit_operand(as_Register(dst_enc), src);
3417 }
3418 
3419 void Assembler::evpcmpeqb(KRegister kdst, KRegister mask, XMMRegister nds, Address src, int vector_len) {
3420   assert(VM_Version::supports_avx512vlbw(), "");
3421   assert(is_vector_masking(), "");    // For stub code use only
3422   InstructionMark im(this);
3423   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_reg_mask */ false, /* uses_vl */ false);
3424   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
3425   attributes.set_embedded_opmask_register_specifier(mask);
3426   attributes.set_is_evex_instruction();
3427   vex_prefix(src, nds->encoding(), kdst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3428   emit_int8(0x74);
3429   emit_operand(as_Register(kdst->encoding()), src);
3430 }
3431 
3432 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
3433 void Assembler::pcmpeqw(XMMRegister dst, XMMRegister src) {
3434   assert(VM_Version::supports_sse2(), "");
3435   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3436   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3437   emit_int8(0x75);
3438   emit_int8((unsigned char)(0xC0 | encode));
3439 }
3440 
3441 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
3442 void Assembler::vpcmpeqw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
3443   assert(VM_Version::supports_avx(), "");
3444   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3445   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3446   emit_int8(0x75);
3447   emit_int8((unsigned char)(0xC0 | encode));
3448 }
3449 
3450 // In this context, kdst is written the mask used to process the equal components
3451 void Assembler::evpcmpeqw(KRegister kdst, XMMRegister nds, XMMRegister src, int vector_len) {
3452   assert(VM_Version::supports_avx512bw(), "");
3453   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
3454   attributes.set_is_evex_instruction();
3455   int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3456   emit_int8(0x75);
3457   emit_int8((unsigned char)(0xC0 | encode));
3458 }
3459 
3460 void Assembler::evpcmpeqw(KRegister kdst, XMMRegister nds, Address src, int vector_len) {
3461   assert(VM_Version::supports_avx512bw(), "");
3462   InstructionMark im(this);
3463   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
3464   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
3465   attributes.set_is_evex_instruction();
3466   int dst_enc = kdst->encoding();
3467   vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3468   emit_int8(0x75);
3469   emit_operand(as_Register(dst_enc), src);
3470 }
3471 
3472 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
3473 void Assembler::pcmpeqd(XMMRegister dst, XMMRegister src) {
3474   assert(VM_Version::supports_sse2(), "");
3475   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3476   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3477   emit_int8(0x76);
3478   emit_int8((unsigned char)(0xC0 | encode));
3479 }
3480 
3481 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
3482 void Assembler::vpcmpeqd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
3483   assert(VM_Version::supports_avx(), "");
3484   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3485   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3486   emit_int8(0x76);
3487   emit_int8((unsigned char)(0xC0 | encode));
3488 }
3489 
3490 // In this context, kdst is written the mask used to process the equal components
3491 void Assembler::evpcmpeqd(KRegister kdst, XMMRegister nds, XMMRegister src, int vector_len) {
3492   assert(VM_Version::supports_evex(), "");
3493   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
3494   attributes.set_is_evex_instruction();
3495   int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3496   emit_int8(0x76);
3497   emit_int8((unsigned char)(0xC0 | encode));
3498 }
3499 
3500 void Assembler::evpcmpeqd(KRegister kdst, XMMRegister nds, Address src, int vector_len) {
3501   assert(VM_Version::supports_evex(), "");
3502   InstructionMark im(this);
3503   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
3504   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
3505   attributes.set_is_evex_instruction();
3506   int dst_enc = kdst->encoding();
3507   vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3508   emit_int8(0x76);
3509   emit_operand(as_Register(dst_enc), src);
3510 }
3511 
3512 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
3513 void Assembler::pcmpeqq(XMMRegister dst, XMMRegister src) {
3514   assert(VM_Version::supports_sse4_1(), "");
3515   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3516   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3517   emit_int8(0x29);
3518   emit_int8((unsigned char)(0xC0 | encode));
3519 }
3520 
3521 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
3522 void Assembler::vpcmpeqq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
3523   assert(VM_Version::supports_avx(), "");
3524   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3525   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3526   emit_int8(0x29);
3527   emit_int8((unsigned char)(0xC0 | encode));
3528 }
3529 
3530 // In this context, kdst is written the mask used to process the equal components
3531 void Assembler::evpcmpeqq(KRegister kdst, XMMRegister nds, XMMRegister src, int vector_len) {
3532   assert(VM_Version::supports_evex(), "");
3533   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
3534   attributes.set_is_evex_instruction();
3535   int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3536   emit_int8(0x29);
3537   emit_int8((unsigned char)(0xC0 | encode));
3538 }
3539 
3540 // In this context, kdst is written the mask used to process the equal components
3541 void Assembler::evpcmpeqq(KRegister kdst, XMMRegister nds, Address src, int vector_len) {
3542   assert(VM_Version::supports_evex(), "");
3543   InstructionMark im(this);
3544   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
3545   attributes.set_is_evex_instruction();
3546   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
3547   int dst_enc = kdst->encoding();
3548   vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3549   emit_int8(0x29);
3550   emit_operand(as_Register(dst_enc), src);
3551 }
3552 
3553 void Assembler::pmovmskb(Register dst, XMMRegister src) {
3554   assert(VM_Version::supports_sse2(), "");
3555   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3556   int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3557   emit_int8((unsigned char)0xD7);
3558   emit_int8((unsigned char)(0xC0 | encode));
3559 }
3560 
3561 void Assembler::vpmovmskb(Register dst, XMMRegister src) {
3562   assert(VM_Version::supports_avx2(), "");
3563   InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3564   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3565   emit_int8((unsigned char)0xD7);
3566   emit_int8((unsigned char)(0xC0 | encode));
3567 }
3568 
3569 void Assembler::pextrd(Register dst, XMMRegister src, int imm8) {
3570   assert(VM_Version::supports_sse4_1(), "");
3571   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
3572   int encode = simd_prefix_and_encode(src, xnoreg, as_XMMRegister(dst->encoding()), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3573   emit_int8(0x16);
3574   emit_int8((unsigned char)(0xC0 | encode));
3575   emit_int8(imm8);
3576 }
3577 
3578 void Assembler::pextrd(Address dst, XMMRegister src, int imm8) {
3579   assert(VM_Version::supports_sse4_1(), "");
3580   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
3581   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
3582   simd_prefix(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3583   emit_int8(0x16);
3584   emit_operand(src, dst);
3585   emit_int8(imm8);
3586 }
3587 
3588 void Assembler::pextrq(Register dst, XMMRegister src, int imm8) {
3589   assert(VM_Version::supports_sse4_1(), "");
3590   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
3591   int encode = simd_prefix_and_encode(src, xnoreg, as_XMMRegister(dst->encoding()), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3592   emit_int8(0x16);
3593   emit_int8((unsigned char)(0xC0 | encode));
3594   emit_int8(imm8);
3595 }
3596 
3597 void Assembler::pextrq(Address dst, XMMRegister src, int imm8) {
3598   assert(VM_Version::supports_sse4_1(), "");
3599   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
3600   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
3601   simd_prefix(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3602   emit_int8(0x16);
3603   emit_operand(src, dst);
3604   emit_int8(imm8);
3605 }
3606 
3607 void Assembler::pextrw(Register dst, XMMRegister src, int imm8) {
3608   assert(VM_Version::supports_sse2(), "");
3609   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
3610   int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3611   emit_int8((unsigned char)0xC5);
3612   emit_int8((unsigned char)(0xC0 | encode));
3613   emit_int8(imm8);
3614 }
3615 
3616 void Assembler::pextrw(Address dst, XMMRegister src, int imm8) {
3617   assert(VM_Version::supports_sse4_1(), "");
3618   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
3619   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_16bit);
3620   simd_prefix(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3621   emit_int8((unsigned char)0x15);
3622   emit_operand(src, dst);
3623   emit_int8(imm8);
3624 }
3625 
3626 void Assembler::pextrb(Address dst, XMMRegister src, int imm8) {
3627   assert(VM_Version::supports_sse4_1(), "");
3628   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
3629   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_8bit);
3630   simd_prefix(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3631   emit_int8(0x14);
3632   emit_operand(src, dst);
3633   emit_int8(imm8);
3634 }
3635 
3636 void Assembler::pinsrd(XMMRegister dst, Register src, int imm8) {
3637   assert(VM_Version::supports_sse4_1(), "");
3638   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
3639   int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3640   emit_int8(0x22);
3641   emit_int8((unsigned char)(0xC0 | encode));
3642   emit_int8(imm8);
3643 }
3644 
3645 void Assembler::pinsrd(XMMRegister dst, Address src, int imm8) {
3646   assert(VM_Version::supports_sse4_1(), "");
3647   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
3648   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
3649   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3650   emit_int8(0x22);
3651   emit_operand(dst,src);
3652   emit_int8(imm8);
3653 }
3654 
3655 void Assembler::pinsrq(XMMRegister dst, Register src, int imm8) {
3656   assert(VM_Version::supports_sse4_1(), "");
3657   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
3658   int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3659   emit_int8(0x22);
3660   emit_int8((unsigned char)(0xC0 | encode));
3661   emit_int8(imm8);
3662 }
3663 
3664 void Assembler::pinsrq(XMMRegister dst, Address src, int imm8) {
3665   assert(VM_Version::supports_sse4_1(), "");
3666   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
3667   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
3668   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3669   emit_int8(0x22);
3670   emit_operand(dst, src);
3671   emit_int8(imm8);
3672 }
3673 
3674 void Assembler::pinsrw(XMMRegister dst, Register src, int imm8) {
3675   assert(VM_Version::supports_sse2(), "");
3676   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
3677   int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3678   emit_int8((unsigned char)0xC4);
3679   emit_int8((unsigned char)(0xC0 | encode));
3680   emit_int8(imm8);
3681 }
3682 
3683 void Assembler::pinsrw(XMMRegister dst, Address src, int imm8) {
3684   assert(VM_Version::supports_sse2(), "");
3685   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
3686   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_16bit);
3687   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3688   emit_int8((unsigned char)0xC4);
3689   emit_operand(dst, src);
3690   emit_int8(imm8);
3691 }
3692 
3693 void Assembler::pinsrb(XMMRegister dst, Address src, int imm8) {
3694   assert(VM_Version::supports_sse4_1(), "");
3695   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
3696   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_8bit);
3697   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3698   emit_int8(0x20);
3699   emit_operand(dst, src);
3700   emit_int8(imm8);
3701 }
3702 
3703 void Assembler::pmovzxbw(XMMRegister dst, Address src) {
3704   assert(VM_Version::supports_sse4_1(), "");
3705   InstructionMark im(this);
3706   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
3707   attributes.set_address_attributes(/* tuple_type */ EVEX_HVM, /* input_size_in_bits */ EVEX_NObit);
3708   simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3709   emit_int8(0x30);
3710   emit_operand(dst, src);
3711 }
3712 
3713 void Assembler::pmovzxbw(XMMRegister dst, XMMRegister src) {
3714   assert(VM_Version::supports_sse4_1(), "");
3715   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
3716   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3717   emit_int8(0x30);
3718   emit_int8((unsigned char)(0xC0 | encode));
3719 }
3720 
3721 void Assembler::vpmovzxbw(XMMRegister dst, Address src, int vector_len) {
3722   assert(VM_Version::supports_avx(), "");
3723   InstructionMark im(this);
3724   assert(dst != xnoreg, "sanity");
3725   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
3726   attributes.set_address_attributes(/* tuple_type */ EVEX_HVM, /* input_size_in_bits */ EVEX_NObit);
3727   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3728   emit_int8(0x30);
3729   emit_operand(dst, src);
3730 }
3731 
3732 void Assembler::evpmovzxbw(XMMRegister dst, KRegister mask, Address src, int vector_len) {
3733   assert(is_vector_masking(), "");
3734   assert(VM_Version::supports_avx512vlbw(), "");
3735   assert(dst != xnoreg, "sanity");
3736   InstructionMark im(this);
3737   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
3738   attributes.set_address_attributes(/* tuple_type */ EVEX_HVM, /* input_size_in_bits */ EVEX_NObit);
3739   attributes.set_embedded_opmask_register_specifier(mask);
3740   attributes.set_is_evex_instruction();
3741   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3742   emit_int8(0x30);
3743   emit_operand(dst, src);
3744 }
3745 
3746 void Assembler::evpmovwb(Address dst, XMMRegister src, int vector_len) {
3747   assert(VM_Version::supports_avx512vlbw(), "");
3748   assert(src != xnoreg, "sanity");
3749   InstructionMark im(this);
3750   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3751   attributes.set_address_attributes(/* tuple_type */ EVEX_HVM, /* input_size_in_bits */ EVEX_NObit);
3752   attributes.set_is_evex_instruction();
3753   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
3754   emit_int8(0x30);
3755   emit_operand(src, dst);
3756 }
3757 
3758 void Assembler::evpmovwb(Address dst, KRegister mask, XMMRegister src, int vector_len) {
3759   assert(is_vector_masking(), "");
3760   assert(VM_Version::supports_avx512vlbw(), "");
3761   assert(src != xnoreg, "sanity");
3762   InstructionMark im(this);
3763   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
3764   attributes.set_address_attributes(/* tuple_type */ EVEX_HVM, /* input_size_in_bits */ EVEX_NObit);
3765   attributes.set_embedded_opmask_register_specifier(mask);
3766   attributes.set_is_evex_instruction();
3767   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
3768   emit_int8(0x30);
3769   emit_operand(src, dst);
3770 }
3771 
3772 // generic
3773 void Assembler::pop(Register dst) {
3774   int encode = prefix_and_encode(dst->encoding());
3775   emit_int8(0x58 | encode);
3776 }
3777 
3778 void Assembler::popcntl(Register dst, Address src) {
3779   assert(VM_Version::supports_popcnt(), "must support");
3780   InstructionMark im(this);
3781   emit_int8((unsigned char)0xF3);
3782   prefix(src, dst);
3783   emit_int8(0x0F);
3784   emit_int8((unsigned char)0xB8);
3785   emit_operand(dst, src);
3786 }
3787 
3788 void Assembler::popcntl(Register dst, Register src) {
3789   assert(VM_Version::supports_popcnt(), "must support");
3790   emit_int8((unsigned char)0xF3);
3791   int encode = prefix_and_encode(dst->encoding(), src->encoding());
3792   emit_int8(0x0F);
3793   emit_int8((unsigned char)0xB8);
3794   emit_int8((unsigned char)(0xC0 | encode));
3795 }
3796 
3797 void Assembler::popf() {
3798   emit_int8((unsigned char)0x9D);
3799 }
3800 
3801 #ifndef _LP64 // no 32bit push/pop on amd64
3802 void Assembler::popl(Address dst) {
3803   // NOTE: this will adjust stack by 8byte on 64bits
3804   InstructionMark im(this);
3805   prefix(dst);
3806   emit_int8((unsigned char)0x8F);
3807   emit_operand(rax, dst);
3808 }
3809 #endif
3810 
3811 void Assembler::prefetch_prefix(Address src) {
3812   prefix(src);
3813   emit_int8(0x0F);
3814 }
3815 
3816 void Assembler::prefetchnta(Address src) {
3817   NOT_LP64(assert(VM_Version::supports_sse(), "must support"));
3818   InstructionMark im(this);
3819   prefetch_prefix(src);
3820   emit_int8(0x18);
3821   emit_operand(rax, src); // 0, src
3822 }
3823 
3824 void Assembler::prefetchr(Address src) {
3825   assert(VM_Version::supports_3dnow_prefetch(), "must support");
3826   InstructionMark im(this);
3827   prefetch_prefix(src);
3828   emit_int8(0x0D);
3829   emit_operand(rax, src); // 0, src
3830 }
3831 
3832 void Assembler::prefetcht0(Address src) {
3833   NOT_LP64(assert(VM_Version::supports_sse(), "must support"));
3834   InstructionMark im(this);
3835   prefetch_prefix(src);
3836   emit_int8(0x18);
3837   emit_operand(rcx, src); // 1, src
3838 }
3839 
3840 void Assembler::prefetcht1(Address src) {
3841   NOT_LP64(assert(VM_Version::supports_sse(), "must support"));
3842   InstructionMark im(this);
3843   prefetch_prefix(src);
3844   emit_int8(0x18);
3845   emit_operand(rdx, src); // 2, src
3846 }
3847 
3848 void Assembler::prefetcht2(Address src) {
3849   NOT_LP64(assert(VM_Version::supports_sse(), "must support"));
3850   InstructionMark im(this);
3851   prefetch_prefix(src);
3852   emit_int8(0x18);
3853   emit_operand(rbx, src); // 3, src
3854 }
3855 
3856 void Assembler::prefetchw(Address src) {
3857   assert(VM_Version::supports_3dnow_prefetch(), "must support");
3858   InstructionMark im(this);
3859   prefetch_prefix(src);
3860   emit_int8(0x0D);
3861   emit_operand(rcx, src); // 1, src
3862 }
3863 
3864 void Assembler::prefix(Prefix p) {
3865   emit_int8(p);
3866 }
3867 
3868 void Assembler::pshufb(XMMRegister dst, XMMRegister src) {
3869   assert(VM_Version::supports_ssse3(), "");
3870   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
3871   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3872   emit_int8(0x00);
3873   emit_int8((unsigned char)(0xC0 | encode));
3874 }
3875 
3876 void Assembler::vpshufb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
3877   assert(vector_len == AVX_128bit? VM_Version::supports_avx() :
3878          vector_len == AVX_256bit? VM_Version::supports_avx2() :
3879          0, "");
3880   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
3881   int encode = simd_prefix_and_encode(dst, nds, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3882   emit_int8(0x00);
3883   emit_int8((unsigned char)(0xC0 | encode));
3884 }
3885 
3886 void Assembler::pshufb(XMMRegister dst, Address src) {
3887   assert(VM_Version::supports_ssse3(), "");
3888   InstructionMark im(this);
3889   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
3890   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
3891   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3892   emit_int8(0x00);
3893   emit_operand(dst, src);
3894 }
3895 
3896 void Assembler::pshufd(XMMRegister dst, XMMRegister src, int mode) {
3897   assert(isByte(mode), "invalid value");
3898   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3899   int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_128bit;
3900   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
3901   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3902   emit_int8(0x70);
3903   emit_int8((unsigned char)(0xC0 | encode));
3904   emit_int8(mode & 0xFF);
3905 }
3906 
3907 void Assembler::vpshufd(XMMRegister dst, XMMRegister src, int mode, int vector_len) {
3908   assert(vector_len == AVX_128bit? VM_Version::supports_avx() :
3909          vector_len == AVX_256bit? VM_Version::supports_avx2() :
3910          0, "");
3911   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3912   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
3913   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3914   emit_int8(0x70);
3915   emit_int8((unsigned char)(0xC0 | encode));
3916   emit_int8(mode & 0xFF);
3917 }
3918 
3919 void Assembler::pshufd(XMMRegister dst, Address src, int mode) {
3920   assert(isByte(mode), "invalid value");
3921   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3922   assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
3923   InstructionMark im(this);
3924   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
3925   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
3926   simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3927   emit_int8(0x70);
3928   emit_operand(dst, src);
3929   emit_int8(mode & 0xFF);
3930 }
3931 
3932 void Assembler::pshuflw(XMMRegister dst, XMMRegister src, int mode) {
3933   assert(isByte(mode), "invalid value");
3934   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3935   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
3936   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
3937   emit_int8(0x70);
3938   emit_int8((unsigned char)(0xC0 | encode));
3939   emit_int8(mode & 0xFF);
3940 }
3941 
3942 void Assembler::pshuflw(XMMRegister dst, Address src, int mode) {
3943   assert(isByte(mode), "invalid value");
3944   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3945   assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
3946   InstructionMark im(this);
3947   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
3948   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
3949   simd_prefix(dst, xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
3950   emit_int8(0x70);
3951   emit_operand(dst, src);
3952   emit_int8(mode & 0xFF);
3953 }
3954 
3955 void Assembler::psrldq(XMMRegister dst, int shift) {
3956   // Shift left 128 bit value in dst XMMRegister by shift number of bytes.
3957   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3958   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
3959   int encode = simd_prefix_and_encode(xmm3, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3960   emit_int8(0x73);
3961   emit_int8((unsigned char)(0xC0 | encode));
3962   emit_int8(shift);
3963 }
3964 
3965 void Assembler::pslldq(XMMRegister dst, int shift) {
3966   // Shift left 128 bit value in dst XMMRegister by shift number of bytes.
3967   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3968   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
3969   // XMM7 is for /7 encoding: 66 0F 73 /7 ib
3970   int encode = simd_prefix_and_encode(xmm7, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3971   emit_int8(0x73);
3972   emit_int8((unsigned char)(0xC0 | encode));
3973   emit_int8(shift);
3974 }
3975 
3976 void Assembler::ptest(XMMRegister dst, Address src) {
3977   assert(VM_Version::supports_sse4_1(), "");
3978   assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
3979   InstructionMark im(this);
3980   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3981   simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3982   emit_int8(0x17);
3983   emit_operand(dst, src);
3984 }
3985 
3986 void Assembler::ptest(XMMRegister dst, XMMRegister src) {
3987   assert(VM_Version::supports_sse4_1(), "");
3988   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3989   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3990   emit_int8(0x17);
3991   emit_int8((unsigned char)(0xC0 | encode));
3992 }
3993 
3994 void Assembler::vptest(XMMRegister dst, Address src) {
3995   assert(VM_Version::supports_avx(), "");
3996   InstructionMark im(this);
3997   InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3998   assert(dst != xnoreg, "sanity");
3999   // swap src<->dst for encoding
4000   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4001   emit_int8(0x17);
4002   emit_operand(dst, src);
4003 }
4004 
4005 void Assembler::vptest(XMMRegister dst, XMMRegister src) {
4006   assert(VM_Version::supports_avx(), "");
4007   InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
4008   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4009   emit_int8(0x17);
4010   emit_int8((unsigned char)(0xC0 | encode));
4011 }
4012 
4013 void Assembler::punpcklbw(XMMRegister dst, Address src) {
4014   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4015   assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
4016   InstructionMark im(this);
4017   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_vlbw, /* no_mask_reg */ false, /* uses_vl */ true);
4018   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
4019   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4020   emit_int8(0x60);
4021   emit_operand(dst, src);
4022 }
4023 
4024 void Assembler::punpcklbw(XMMRegister dst, XMMRegister src) {
4025   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4026   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_vlbw, /* no_mask_reg */ false, /* uses_vl */ true);
4027   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4028   emit_int8(0x60);
4029   emit_int8((unsigned char)(0xC0 | encode));
4030 }
4031 
4032 void Assembler::punpckldq(XMMRegister dst, Address src) {
4033   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4034   assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
4035   InstructionMark im(this);
4036   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4037   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
4038   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4039   emit_int8(0x62);
4040   emit_operand(dst, src);
4041 }
4042 
4043 void Assembler::punpckldq(XMMRegister dst, XMMRegister src) {
4044   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4045   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4046   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4047   emit_int8(0x62);
4048   emit_int8((unsigned char)(0xC0 | encode));
4049 }
4050 
4051 void Assembler::punpcklqdq(XMMRegister dst, XMMRegister src) {
4052   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4053   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4054   attributes.set_rex_vex_w_reverted();
4055   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4056   emit_int8(0x6C);
4057   emit_int8((unsigned char)(0xC0 | encode));
4058 }
4059 
4060 void Assembler::push(int32_t imm32) {
4061   // in 64bits we push 64bits onto the stack but only
4062   // take a 32bit immediate
4063   emit_int8(0x68);
4064   emit_int32(imm32);
4065 }
4066 
4067 void Assembler::push(Register src) {
4068   int encode = prefix_and_encode(src->encoding());
4069 
4070   emit_int8(0x50 | encode);
4071 }
4072 
4073 void Assembler::pushf() {
4074   emit_int8((unsigned char)0x9C);
4075 }
4076 
4077 #ifndef _LP64 // no 32bit push/pop on amd64
4078 void Assembler::pushl(Address src) {
4079   // Note this will push 64bit on 64bit
4080   InstructionMark im(this);
4081   prefix(src);
4082   emit_int8((unsigned char)0xFF);
4083   emit_operand(rsi, src);
4084 }
4085 #endif
4086 
4087 void Assembler::rcll(Register dst, int imm8) {
4088   assert(isShiftCount(imm8), "illegal shift count");
4089   int encode = prefix_and_encode(dst->encoding());
4090   if (imm8 == 1) {
4091     emit_int8((unsigned char)0xD1);
4092     emit_int8((unsigned char)(0xD0 | encode));
4093   } else {
4094     emit_int8((unsigned char)0xC1);
4095     emit_int8((unsigned char)0xD0 | encode);
4096     emit_int8(imm8);
4097   }
4098 }
4099 
4100 void Assembler::rcpps(XMMRegister dst, XMMRegister src) {
4101   NOT_LP64(assert(VM_Version::supports_sse(), ""));
4102   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
4103   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
4104   emit_int8(0x53);
4105   emit_int8((unsigned char)(0xC0 | encode));
4106 }
4107 
4108 void Assembler::rcpss(XMMRegister dst, XMMRegister src) {
4109   NOT_LP64(assert(VM_Version::supports_sse(), ""));
4110   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
4111   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
4112   emit_int8(0x53);
4113   emit_int8((unsigned char)(0xC0 | encode));
4114 }
4115 
4116 void Assembler::rdtsc() {
4117   emit_int8((unsigned char)0x0F);
4118   emit_int8((unsigned char)0x31);
4119 }
4120 
4121 // copies data from [esi] to [edi] using rcx pointer sized words
4122 // generic
4123 void Assembler::rep_mov() {
4124   emit_int8((unsigned char)0xF3);
4125   // MOVSQ
4126   LP64_ONLY(prefix(REX_W));
4127   emit_int8((unsigned char)0xA5);
4128 }
4129 
4130 // sets rcx bytes with rax, value at [edi]
4131 void Assembler::rep_stosb() {
4132   emit_int8((unsigned char)0xF3); // REP
4133   LP64_ONLY(prefix(REX_W));
4134   emit_int8((unsigned char)0xAA); // STOSB
4135 }
4136 
4137 // sets rcx pointer sized words with rax, value at [edi]
4138 // generic
4139 void Assembler::rep_stos() {
4140   emit_int8((unsigned char)0xF3); // REP
4141   LP64_ONLY(prefix(REX_W));       // LP64:STOSQ, LP32:STOSD
4142   emit_int8((unsigned char)0xAB);
4143 }
4144 
4145 // scans rcx pointer sized words at [edi] for occurance of rax,
4146 // generic
4147 void Assembler::repne_scan() { // repne_scan
4148   emit_int8((unsigned char)0xF2);
4149   // SCASQ
4150   LP64_ONLY(prefix(REX_W));
4151   emit_int8((unsigned char)0xAF);
4152 }
4153 
4154 #ifdef _LP64
4155 // scans rcx 4 byte words at [edi] for occurance of rax,
4156 // generic
4157 void Assembler::repne_scanl() { // repne_scan
4158   emit_int8((unsigned char)0xF2);
4159   // SCASL
4160   emit_int8((unsigned char)0xAF);
4161 }
4162 #endif
4163 
4164 void Assembler::ret(int imm16) {
4165   if (imm16 == 0) {
4166     emit_int8((unsigned char)0xC3);
4167   } else {
4168     emit_int8((unsigned char)0xC2);
4169     emit_int16(imm16);
4170   }
4171 }
4172 
4173 void Assembler::sahf() {
4174 #ifdef _LP64
4175   // Not supported in 64bit mode
4176   ShouldNotReachHere();
4177 #endif
4178   emit_int8((unsigned char)0x9E);
4179 }
4180 
4181 void Assembler::sarl(Register dst, int imm8) {
4182   int encode = prefix_and_encode(dst->encoding());
4183   assert(isShiftCount(imm8), "illegal shift count");
4184   if (imm8 == 1) {
4185     emit_int8((unsigned char)0xD1);
4186     emit_int8((unsigned char)(0xF8 | encode));
4187   } else {
4188     emit_int8((unsigned char)0xC1);
4189     emit_int8((unsigned char)(0xF8 | encode));
4190     emit_int8(imm8);
4191   }
4192 }
4193 
4194 void Assembler::sarl(Register dst) {
4195   int encode = prefix_and_encode(dst->encoding());
4196   emit_int8((unsigned char)0xD3);
4197   emit_int8((unsigned char)(0xF8 | encode));
4198 }
4199 
4200 void Assembler::sbbl(Address dst, int32_t imm32) {
4201   InstructionMark im(this);
4202   prefix(dst);
4203   emit_arith_operand(0x81, rbx, dst, imm32);
4204 }
4205 
4206 void Assembler::sbbl(Register dst, int32_t imm32) {
4207   prefix(dst);
4208   emit_arith(0x81, 0xD8, dst, imm32);
4209 }
4210 
4211 
4212 void Assembler::sbbl(Register dst, Address src) {
4213   InstructionMark im(this);
4214   prefix(src, dst);
4215   emit_int8(0x1B);
4216   emit_operand(dst, src);
4217 }
4218 
4219 void Assembler::sbbl(Register dst, Register src) {
4220   (void) prefix_and_encode(dst->encoding(), src->encoding());
4221   emit_arith(0x1B, 0xC0, dst, src);
4222 }
4223 
4224 void Assembler::setb(Condition cc, Register dst) {
4225   assert(0 <= cc && cc < 16, "illegal cc");
4226   int encode = prefix_and_encode(dst->encoding(), true);
4227   emit_int8(0x0F);
4228   emit_int8((unsigned char)0x90 | cc);
4229   emit_int8((unsigned char)(0xC0 | encode));
4230 }
4231 
4232 void Assembler::palignr(XMMRegister dst, XMMRegister src, int imm8) {
4233   assert(VM_Version::supports_ssse3(), "");
4234   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ false);
4235   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
4236   emit_int8((unsigned char)0x0F);
4237   emit_int8((unsigned char)(0xC0 | encode));
4238   emit_int8(imm8);
4239 }
4240 
4241 void Assembler::vpalignr(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8, int vector_len) {
4242   assert(vector_len == AVX_128bit? VM_Version::supports_avx() :
4243          vector_len == AVX_256bit? VM_Version::supports_avx2() :
4244          0, "");
4245   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
4246   int encode = simd_prefix_and_encode(dst, nds, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
4247   emit_int8((unsigned char)0x0F);
4248   emit_int8((unsigned char)(0xC0 | encode));
4249   emit_int8(imm8);
4250 }
4251 
4252 void Assembler::pblendw(XMMRegister dst, XMMRegister src, int imm8) {
4253   assert(VM_Version::supports_sse4_1(), "");
4254   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
4255   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
4256   emit_int8((unsigned char)0x0E);
4257   emit_int8((unsigned char)(0xC0 | encode));
4258   emit_int8(imm8);
4259 }
4260 
4261 void Assembler::sha1rnds4(XMMRegister dst, XMMRegister src, int imm8) {
4262   assert(VM_Version::supports_sha(), "");
4263   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
4264   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F_3A, &attributes);
4265   emit_int8((unsigned char)0xCC);
4266   emit_int8((unsigned char)(0xC0 | encode));
4267   emit_int8((unsigned char)imm8);
4268 }
4269 
4270 void Assembler::sha1nexte(XMMRegister dst, XMMRegister src) {
4271   assert(VM_Version::supports_sha(), "");
4272   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
4273   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
4274   emit_int8((unsigned char)0xC8);
4275   emit_int8((unsigned char)(0xC0 | encode));
4276 }
4277 
4278 void Assembler::sha1msg1(XMMRegister dst, XMMRegister src) {
4279   assert(VM_Version::supports_sha(), "");
4280   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
4281   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
4282   emit_int8((unsigned char)0xC9);
4283   emit_int8((unsigned char)(0xC0 | encode));
4284 }
4285 
4286 void Assembler::sha1msg2(XMMRegister dst, XMMRegister src) {
4287   assert(VM_Version::supports_sha(), "");
4288   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
4289   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
4290   emit_int8((unsigned char)0xCA);
4291   emit_int8((unsigned char)(0xC0 | encode));
4292 }
4293 
4294 // xmm0 is implicit additional source to this instruction.
4295 void Assembler::sha256rnds2(XMMRegister dst, XMMRegister src) {
4296   assert(VM_Version::supports_sha(), "");
4297   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
4298   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
4299   emit_int8((unsigned char)0xCB);
4300   emit_int8((unsigned char)(0xC0 | encode));
4301 }
4302 
4303 void Assembler::sha256msg1(XMMRegister dst, XMMRegister src) {
4304   assert(VM_Version::supports_sha(), "");
4305   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
4306   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
4307   emit_int8((unsigned char)0xCC);
4308   emit_int8((unsigned char)(0xC0 | encode));
4309 }
4310 
4311 void Assembler::sha256msg2(XMMRegister dst, XMMRegister src) {
4312   assert(VM_Version::supports_sha(), "");
4313   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
4314   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
4315   emit_int8((unsigned char)0xCD);
4316   emit_int8((unsigned char)(0xC0 | encode));
4317 }
4318 
4319 
4320 void Assembler::shll(Register dst, int imm8) {
4321   assert(isShiftCount(imm8), "illegal shift count");
4322   int encode = prefix_and_encode(dst->encoding());
4323   if (imm8 == 1 ) {
4324     emit_int8((unsigned char)0xD1);
4325     emit_int8((unsigned char)(0xE0 | encode));
4326   } else {
4327     emit_int8((unsigned char)0xC1);
4328     emit_int8((unsigned char)(0xE0 | encode));
4329     emit_int8(imm8);
4330   }
4331 }
4332 
4333 void Assembler::shll(Register dst) {
4334   int encode = prefix_and_encode(dst->encoding());
4335   emit_int8((unsigned char)0xD3);
4336   emit_int8((unsigned char)(0xE0 | encode));
4337 }
4338 
4339 void Assembler::shrl(Register dst, int imm8) {
4340   assert(isShiftCount(imm8), "illegal shift count");
4341   int encode = prefix_and_encode(dst->encoding());
4342   emit_int8((unsigned char)0xC1);
4343   emit_int8((unsigned char)(0xE8 | encode));
4344   emit_int8(imm8);
4345 }
4346 
4347 void Assembler::shrl(Register dst) {
4348   int encode = prefix_and_encode(dst->encoding());
4349   emit_int8((unsigned char)0xD3);
4350   emit_int8((unsigned char)(0xE8 | encode));
4351 }
4352 
4353 // copies a single word from [esi] to [edi]
4354 void Assembler::smovl() {
4355   emit_int8((unsigned char)0xA5);
4356 }
4357 
4358 void Assembler::sqrtsd(XMMRegister dst, XMMRegister src) {
4359   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4360   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
4361   attributes.set_rex_vex_w_reverted();
4362   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
4363   emit_int8(0x51);
4364   emit_int8((unsigned char)(0xC0 | encode));
4365 }
4366 
4367 void Assembler::sqrtsd(XMMRegister dst, Address src) {
4368   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4369   InstructionMark im(this);
4370   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
4371   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
4372   attributes.set_rex_vex_w_reverted();
4373   simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
4374   emit_int8(0x51);
4375   emit_operand(dst, src);
4376 }
4377 
4378 void Assembler::sqrtss(XMMRegister dst, XMMRegister src) {
4379   NOT_LP64(assert(VM_Version::supports_sse(), ""));
4380   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
4381   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
4382   emit_int8(0x51);
4383   emit_int8((unsigned char)(0xC0 | encode));
4384 }
4385 
4386 void Assembler::std() {
4387   emit_int8((unsigned char)0xFD);
4388 }
4389 
4390 void Assembler::sqrtss(XMMRegister dst, Address src) {
4391   NOT_LP64(assert(VM_Version::supports_sse(), ""));
4392   InstructionMark im(this);
4393   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
4394   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
4395   simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
4396   emit_int8(0x51);
4397   emit_operand(dst, src);
4398 }
4399 
4400 void Assembler::stmxcsr( Address dst) {
4401   NOT_LP64(assert(VM_Version::supports_sse(), ""));
4402   InstructionMark im(this);
4403   prefix(dst);
4404   emit_int8(0x0F);
4405   emit_int8((unsigned char)0xAE);
4406   emit_operand(as_Register(3), dst);
4407 }
4408 
4409 void Assembler::subl(Address dst, int32_t imm32) {
4410   InstructionMark im(this);
4411   prefix(dst);
4412   emit_arith_operand(0x81, rbp, dst, imm32);
4413 }
4414 
4415 void Assembler::subl(Address dst, Register src) {
4416   InstructionMark im(this);
4417   prefix(dst, src);
4418   emit_int8(0x29);
4419   emit_operand(src, dst);
4420 }
4421 
4422 void Assembler::subl(Register dst, int32_t imm32) {
4423   prefix(dst);
4424   emit_arith(0x81, 0xE8, dst, imm32);
4425 }
4426 
4427 // Force generation of a 4 byte immediate value even if it fits into 8bit
4428 void Assembler::subl_imm32(Register dst, int32_t imm32) {
4429   prefix(dst);
4430   emit_arith_imm32(0x81, 0xE8, dst, imm32);
4431 }
4432 
4433 void Assembler::subl(Register dst, Address src) {
4434   InstructionMark im(this);
4435   prefix(src, dst);
4436   emit_int8(0x2B);
4437   emit_operand(dst, src);
4438 }
4439 
4440 void Assembler::subl(Register dst, Register src) {
4441   (void) prefix_and_encode(dst->encoding(), src->encoding());
4442   emit_arith(0x2B, 0xC0, dst, src);
4443 }
4444 
4445 void Assembler::subsd(XMMRegister dst, XMMRegister src) {
4446   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4447   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
4448   attributes.set_rex_vex_w_reverted();
4449   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
4450   emit_int8(0x5C);
4451   emit_int8((unsigned char)(0xC0 | encode));
4452 }
4453 
4454 void Assembler::subsd(XMMRegister dst, Address src) {
4455   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4456   InstructionMark im(this);
4457   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
4458   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
4459   attributes.set_rex_vex_w_reverted();
4460   simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
4461   emit_int8(0x5C);
4462   emit_operand(dst, src);
4463 }
4464 
4465 void Assembler::subss(XMMRegister dst, XMMRegister src) {
4466   NOT_LP64(assert(VM_Version::supports_sse(), ""));
4467   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false , /* uses_vl */ false);
4468   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
4469   emit_int8(0x5C);
4470   emit_int8((unsigned char)(0xC0 | encode));
4471 }
4472 
4473 void Assembler::subss(XMMRegister dst, Address src) {
4474   NOT_LP64(assert(VM_Version::supports_sse(), ""));
4475   InstructionMark im(this);
4476   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
4477   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
4478   simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
4479   emit_int8(0x5C);
4480   emit_operand(dst, src);
4481 }
4482 
4483 void Assembler::testb(Register dst, int imm8) {
4484   NOT_LP64(assert(dst->has_byte_register(), "must have byte register"));
4485   (void) prefix_and_encode(dst->encoding(), true);
4486   emit_arith_b(0xF6, 0xC0, dst, imm8);
4487 }
4488 
4489 void Assembler::testb(Address dst, int imm8) {
4490   InstructionMark im(this);
4491   prefix(dst);
4492   emit_int8((unsigned char)0xF6);
4493   emit_operand(rax, dst, 1);
4494   emit_int8(imm8);
4495 }
4496 
4497 void Assembler::testl(Register dst, int32_t imm32) {
4498   // not using emit_arith because test
4499   // doesn't support sign-extension of
4500   // 8bit operands
4501   int encode = dst->encoding();
4502   if (encode == 0) {
4503     emit_int8((unsigned char)0xA9);
4504   } else {
4505     encode = prefix_and_encode(encode);
4506     emit_int8((unsigned char)0xF7);
4507     emit_int8((unsigned char)(0xC0 | encode));
4508   }
4509   emit_int32(imm32);
4510 }
4511 
4512 void Assembler::testl(Register dst, Register src) {
4513   (void) prefix_and_encode(dst->encoding(), src->encoding());
4514   emit_arith(0x85, 0xC0, dst, src);
4515 }
4516 
4517 void Assembler::testl(Register dst, Address src) {
4518   InstructionMark im(this);
4519   prefix(src, dst);
4520   emit_int8((unsigned char)0x85);
4521   emit_operand(dst, src);
4522 }
4523 
4524 void Assembler::tzcntl(Register dst, Register src) {
4525   assert(VM_Version::supports_bmi1(), "tzcnt instruction not supported");
4526   emit_int8((unsigned char)0xF3);
4527   int encode = prefix_and_encode(dst->encoding(), src->encoding());
4528   emit_int8(0x0F);
4529   emit_int8((unsigned char)0xBC);
4530   emit_int8((unsigned char)0xC0 | encode);
4531 }
4532 
4533 void Assembler::tzcntq(Register dst, Register src) {
4534   assert(VM_Version::supports_bmi1(), "tzcnt instruction not supported");
4535   emit_int8((unsigned char)0xF3);
4536   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
4537   emit_int8(0x0F);
4538   emit_int8((unsigned char)0xBC);
4539   emit_int8((unsigned char)(0xC0 | encode));
4540 }
4541 
4542 void Assembler::ucomisd(XMMRegister dst, Address src) {
4543   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4544   InstructionMark im(this);
4545   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4546   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
4547   attributes.set_rex_vex_w_reverted();
4548   simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4549   emit_int8(0x2E);
4550   emit_operand(dst, src);
4551 }
4552 
4553 void Assembler::ucomisd(XMMRegister dst, XMMRegister src) {
4554   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4555   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4556   attributes.set_rex_vex_w_reverted();
4557   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4558   emit_int8(0x2E);
4559   emit_int8((unsigned char)(0xC0 | encode));
4560 }
4561 
4562 void Assembler::ucomiss(XMMRegister dst, Address src) {
4563   NOT_LP64(assert(VM_Version::supports_sse(), ""));
4564   InstructionMark im(this);
4565   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4566   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
4567   simd_prefix(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
4568   emit_int8(0x2E);
4569   emit_operand(dst, src);
4570 }
4571 
4572 void Assembler::ucomiss(XMMRegister dst, XMMRegister src) {
4573   NOT_LP64(assert(VM_Version::supports_sse(), ""));
4574   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4575   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
4576   emit_int8(0x2E);
4577   emit_int8((unsigned char)(0xC0 | encode));
4578 }
4579 
4580 void Assembler::xabort(int8_t imm8) {
4581   emit_int8((unsigned char)0xC6);
4582   emit_int8((unsigned char)0xF8);
4583   emit_int8((unsigned char)(imm8 & 0xFF));
4584 }
4585 
4586 void Assembler::xaddb(Address dst, Register src) {
4587   InstructionMark im(this);
4588   prefix(dst, src, true);
4589   emit_int8(0x0F);
4590   emit_int8((unsigned char)0xC0);
4591   emit_operand(src, dst);
4592 }
4593 
4594 void Assembler::xaddw(Address dst, Register src) {
4595   InstructionMark im(this);
4596   emit_int8(0x66);
4597   prefix(dst, src);
4598   emit_int8(0x0F);
4599   emit_int8((unsigned char)0xC1);
4600   emit_operand(src, dst);
4601 }
4602 
4603 void Assembler::xaddl(Address dst, Register src) {
4604   InstructionMark im(this);
4605   prefix(dst, src);
4606   emit_int8(0x0F);
4607   emit_int8((unsigned char)0xC1);
4608   emit_operand(src, dst);
4609 }
4610 
4611 void Assembler::xbegin(Label& abort, relocInfo::relocType rtype) {
4612   InstructionMark im(this);
4613   relocate(rtype);
4614   if (abort.is_bound()) {
4615     address entry = target(abort);
4616     assert(entry != NULL, "abort entry NULL");
4617     intptr_t offset = entry - pc();
4618     emit_int8((unsigned char)0xC7);
4619     emit_int8((unsigned char)0xF8);
4620     emit_int32(offset - 6); // 2 opcode + 4 address
4621   } else {
4622     abort.add_patch_at(code(), locator());
4623     emit_int8((unsigned char)0xC7);
4624     emit_int8((unsigned char)0xF8);
4625     emit_int32(0);
4626   }
4627 }
4628 
4629 void Assembler::xchgb(Register dst, Address src) { // xchg
4630   InstructionMark im(this);
4631   prefix(src, dst, true);
4632   emit_int8((unsigned char)0x86);
4633   emit_operand(dst, src);
4634 }
4635 
4636 void Assembler::xchgw(Register dst, Address src) { // xchg
4637   InstructionMark im(this);
4638   emit_int8(0x66);
4639   prefix(src, dst);
4640   emit_int8((unsigned char)0x87);
4641   emit_operand(dst, src);
4642 }
4643 
4644 void Assembler::xchgl(Register dst, Address src) { // xchg
4645   InstructionMark im(this);
4646   prefix(src, dst);
4647   emit_int8((unsigned char)0x87);
4648   emit_operand(dst, src);
4649 }
4650 
4651 void Assembler::xchgl(Register dst, Register src) {
4652   int encode = prefix_and_encode(dst->encoding(), src->encoding());
4653   emit_int8((unsigned char)0x87);
4654   emit_int8((unsigned char)(0xC0 | encode));
4655 }
4656 
4657 void Assembler::xend() {
4658   emit_int8((unsigned char)0x0F);
4659   emit_int8((unsigned char)0x01);
4660   emit_int8((unsigned char)0xD5);
4661 }
4662 
4663 void Assembler::xgetbv() {
4664   emit_int8(0x0F);
4665   emit_int8(0x01);
4666   emit_int8((unsigned char)0xD0);
4667 }
4668 
4669 void Assembler::xorl(Register dst, int32_t imm32) {
4670   prefix(dst);
4671   emit_arith(0x81, 0xF0, dst, imm32);
4672 }
4673 
4674 void Assembler::xorl(Register dst, Address src) {
4675   InstructionMark im(this);
4676   prefix(src, dst);
4677   emit_int8(0x33);
4678   emit_operand(dst, src);
4679 }
4680 
4681 void Assembler::xorl(Register dst, Register src) {
4682   (void) prefix_and_encode(dst->encoding(), src->encoding());
4683   emit_arith(0x33, 0xC0, dst, src);
4684 }
4685 
4686 void Assembler::xorb(Register dst, Address src) {
4687   InstructionMark im(this);
4688   prefix(src, dst);
4689   emit_int8(0x32);
4690   emit_operand(dst, src);
4691 }
4692 
4693 // AVX 3-operands scalar float-point arithmetic instructions
4694 
4695 void Assembler::vaddsd(XMMRegister dst, XMMRegister nds, Address src) {
4696   assert(VM_Version::supports_avx(), "");
4697   InstructionMark im(this);
4698   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
4699   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
4700   attributes.set_rex_vex_w_reverted();
4701   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
4702   emit_int8(0x58);
4703   emit_operand(dst, src);
4704 }
4705 
4706 void Assembler::vaddsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
4707   assert(VM_Version::supports_avx(), "");
4708   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
4709   attributes.set_rex_vex_w_reverted();
4710   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
4711   emit_int8(0x58);
4712   emit_int8((unsigned char)(0xC0 | encode));
4713 }
4714 
4715 void Assembler::vaddss(XMMRegister dst, XMMRegister nds, Address src) {
4716   assert(VM_Version::supports_avx(), "");
4717   InstructionMark im(this);
4718   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
4719   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
4720   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
4721   emit_int8(0x58);
4722   emit_operand(dst, src);
4723 }
4724 
4725 void Assembler::vaddss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
4726   assert(VM_Version::supports_avx(), "");
4727   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
4728   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
4729   emit_int8(0x58);
4730   emit_int8((unsigned char)(0xC0 | encode));
4731 }
4732 
4733 void Assembler::vdivsd(XMMRegister dst, XMMRegister nds, Address src) {
4734   assert(VM_Version::supports_avx(), "");
4735   InstructionMark im(this);
4736   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
4737   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
4738   attributes.set_rex_vex_w_reverted();
4739   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
4740   emit_int8(0x5E);
4741   emit_operand(dst, src);
4742 }
4743 
4744 void Assembler::vdivsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
4745   assert(VM_Version::supports_avx(), "");
4746   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
4747   attributes.set_rex_vex_w_reverted();
4748   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
4749   emit_int8(0x5E);
4750   emit_int8((unsigned char)(0xC0 | encode));
4751 }
4752 
4753 void Assembler::vdivss(XMMRegister dst, XMMRegister nds, Address src) {
4754   assert(VM_Version::supports_avx(), "");
4755   InstructionMark im(this);
4756   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
4757   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
4758   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
4759   emit_int8(0x5E);
4760   emit_operand(dst, src);
4761 }
4762 
4763 void Assembler::vdivss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
4764   assert(VM_Version::supports_avx(), "");
4765   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
4766   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
4767   emit_int8(0x5E);
4768   emit_int8((unsigned char)(0xC0 | encode));
4769 }
4770 
4771 void Assembler::vmulsd(XMMRegister dst, XMMRegister nds, Address src) {
4772   assert(VM_Version::supports_avx(), "");
4773   InstructionMark im(this);
4774   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
4775   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
4776   attributes.set_rex_vex_w_reverted();
4777   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
4778   emit_int8(0x59);
4779   emit_operand(dst, src);
4780 }
4781 
4782 void Assembler::vmulsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
4783   assert(VM_Version::supports_avx(), "");
4784   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
4785   attributes.set_rex_vex_w_reverted();
4786   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
4787   emit_int8(0x59);
4788   emit_int8((unsigned char)(0xC0 | encode));
4789 }
4790 
4791 void Assembler::vmulss(XMMRegister dst, XMMRegister nds, Address src) {
4792   assert(VM_Version::supports_avx(), "");
4793   InstructionMark im(this);
4794   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
4795   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
4796   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
4797   emit_int8(0x59);
4798   emit_operand(dst, src);
4799 }
4800 
4801 void Assembler::vmulss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
4802   assert(VM_Version::supports_avx(), "");
4803   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
4804   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
4805   emit_int8(0x59);
4806   emit_int8((unsigned char)(0xC0 | encode));
4807 }
4808 
4809 void Assembler::vsubsd(XMMRegister dst, XMMRegister nds, Address src) {
4810   assert(VM_Version::supports_avx(), "");
4811   InstructionMark im(this);
4812   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
4813   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
4814   attributes.set_rex_vex_w_reverted();
4815   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
4816   emit_int8(0x5C);
4817   emit_operand(dst, src);
4818 }
4819 
4820 void Assembler::vsubsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
4821   assert(VM_Version::supports_avx(), "");
4822   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
4823   attributes.set_rex_vex_w_reverted();
4824   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
4825   emit_int8(0x5C);
4826   emit_int8((unsigned char)(0xC0 | encode));
4827 }
4828 
4829 void Assembler::vsubss(XMMRegister dst, XMMRegister nds, Address src) {
4830   assert(VM_Version::supports_avx(), "");
4831   InstructionMark im(this);
4832   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
4833   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
4834   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
4835   emit_int8(0x5C);
4836   emit_operand(dst, src);
4837 }
4838 
4839 void Assembler::vsubss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
4840   assert(VM_Version::supports_avx(), "");
4841   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
4842   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
4843   emit_int8(0x5C);
4844   emit_int8((unsigned char)(0xC0 | encode));
4845 }
4846 
4847 //====================VECTOR ARITHMETIC=====================================
4848 
4849 // Float-point vector arithmetic
4850 
4851 void Assembler::addpd(XMMRegister dst, XMMRegister src) {
4852   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4853   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4854   attributes.set_rex_vex_w_reverted();
4855   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4856   emit_int8(0x58);
4857   emit_int8((unsigned char)(0xC0 | encode));
4858 }
4859 
4860 void Assembler::addpd(XMMRegister dst, Address src) {
4861   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4862   InstructionMark im(this);
4863   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4864   attributes.set_rex_vex_w_reverted();
4865   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
4866   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4867   emit_int8(0x58);
4868   emit_operand(dst, src);
4869 }
4870 
4871 
4872 void Assembler::addps(XMMRegister dst, XMMRegister src) {
4873   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4874   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4875   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
4876   emit_int8(0x58);
4877   emit_int8((unsigned char)(0xC0 | encode));
4878 }
4879 
4880 void Assembler::vaddpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4881   assert(VM_Version::supports_avx(), "");
4882   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4883   attributes.set_rex_vex_w_reverted();
4884   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4885   emit_int8(0x58);
4886   emit_int8((unsigned char)(0xC0 | encode));
4887 }
4888 
4889 void Assembler::vaddps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4890   assert(VM_Version::supports_avx(), "");
4891   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4892   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
4893   emit_int8(0x58);
4894   emit_int8((unsigned char)(0xC0 | encode));
4895 }
4896 
4897 void Assembler::vaddpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4898   assert(VM_Version::supports_avx(), "");
4899   InstructionMark im(this);
4900   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4901   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
4902   attributes.set_rex_vex_w_reverted();
4903   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4904   emit_int8(0x58);
4905   emit_operand(dst, src);
4906 }
4907 
4908 void Assembler::vaddps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4909   assert(VM_Version::supports_avx(), "");
4910   InstructionMark im(this);
4911   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4912   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
4913   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
4914   emit_int8(0x58);
4915   emit_operand(dst, src);
4916 }
4917 
4918 void Assembler::subpd(XMMRegister dst, XMMRegister src) {
4919   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4920   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4921   attributes.set_rex_vex_w_reverted();
4922   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4923   emit_int8(0x5C);
4924   emit_int8((unsigned char)(0xC0 | encode));
4925 }
4926 
4927 void Assembler::subps(XMMRegister dst, XMMRegister src) {
4928   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4929   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4930   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
4931   emit_int8(0x5C);
4932   emit_int8((unsigned char)(0xC0 | encode));
4933 }
4934 
4935 void Assembler::vsubpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4936   assert(VM_Version::supports_avx(), "");
4937   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4938   attributes.set_rex_vex_w_reverted();
4939   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4940   emit_int8(0x5C);
4941   emit_int8((unsigned char)(0xC0 | encode));
4942 }
4943 
4944 void Assembler::vsubps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4945   assert(VM_Version::supports_avx(), "");
4946   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4947   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
4948   emit_int8(0x5C);
4949   emit_int8((unsigned char)(0xC0 | encode));
4950 }
4951 
4952 void Assembler::vsubpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4953   assert(VM_Version::supports_avx(), "");
4954   InstructionMark im(this);
4955   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4956   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
4957   attributes.set_rex_vex_w_reverted();
4958   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4959   emit_int8(0x5C);
4960   emit_operand(dst, src);
4961 }
4962 
4963 void Assembler::vsubps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4964   assert(VM_Version::supports_avx(), "");
4965   InstructionMark im(this);
4966   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4967   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
4968   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
4969   emit_int8(0x5C);
4970   emit_operand(dst, src);
4971 }
4972 
4973 void Assembler::mulpd(XMMRegister dst, XMMRegister src) {
4974   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4975   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4976   attributes.set_rex_vex_w_reverted();
4977   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4978   emit_int8(0x59);
4979   emit_int8((unsigned char)(0xC0 | encode));
4980 }
4981 
4982 void Assembler::mulpd(XMMRegister dst, Address src) {
4983   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4984   InstructionMark im(this);
4985   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4986   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
4987   attributes.set_rex_vex_w_reverted();
4988   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4989   emit_int8(0x59);
4990   emit_operand(dst, src);
4991 }
4992 
4993 void Assembler::mulps(XMMRegister dst, XMMRegister src) {
4994   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4995   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4996   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
4997   emit_int8(0x59);
4998   emit_int8((unsigned char)(0xC0 | encode));
4999 }
5000 
5001 void Assembler::vmulpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5002   assert(VM_Version::supports_avx(), "");
5003   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5004   attributes.set_rex_vex_w_reverted();
5005   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5006   emit_int8(0x59);
5007   emit_int8((unsigned char)(0xC0 | encode));
5008 }
5009 
5010 void Assembler::vmulps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5011   assert(VM_Version::supports_avx(), "");
5012   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5013   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
5014   emit_int8(0x59);
5015   emit_int8((unsigned char)(0xC0 | encode));
5016 }
5017 
5018 void Assembler::vmulpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5019   assert(VM_Version::supports_avx(), "");
5020   InstructionMark im(this);
5021   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5022   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
5023   attributes.set_rex_vex_w_reverted();
5024   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5025   emit_int8(0x59);
5026   emit_operand(dst, src);
5027 }
5028 
5029 void Assembler::vmulps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5030   assert(VM_Version::supports_avx(), "");
5031   InstructionMark im(this);
5032   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5033   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
5034   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
5035   emit_int8(0x59);
5036   emit_operand(dst, src);
5037 }
5038 
5039 void Assembler::divpd(XMMRegister dst, XMMRegister src) {
5040   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5041   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5042   attributes.set_rex_vex_w_reverted();
5043   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5044   emit_int8(0x5E);
5045   emit_int8((unsigned char)(0xC0 | encode));
5046 }
5047 
5048 void Assembler::divps(XMMRegister dst, XMMRegister src) {
5049   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5050   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5051   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
5052   emit_int8(0x5E);
5053   emit_int8((unsigned char)(0xC0 | encode));
5054 }
5055 
5056 void Assembler::vdivpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5057   assert(VM_Version::supports_avx(), "");
5058   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5059   attributes.set_rex_vex_w_reverted();
5060   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5061   emit_int8(0x5E);
5062   emit_int8((unsigned char)(0xC0 | encode));
5063 }
5064 
5065 void Assembler::vdivps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5066   assert(VM_Version::supports_avx(), "");
5067   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5068   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
5069   emit_int8(0x5E);
5070   emit_int8((unsigned char)(0xC0 | encode));
5071 }
5072 
5073 void Assembler::vdivpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5074   assert(VM_Version::supports_avx(), "");
5075   InstructionMark im(this);
5076   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5077   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
5078   attributes.set_rex_vex_w_reverted();
5079   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5080   emit_int8(0x5E);
5081   emit_operand(dst, src);
5082 }
5083 
5084 void Assembler::vdivps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5085   assert(VM_Version::supports_avx(), "");
5086   InstructionMark im(this);
5087   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5088   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
5089   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
5090   emit_int8(0x5E);
5091   emit_operand(dst, src);
5092 }
5093 
5094 void Assembler::vsqrtpd(XMMRegister dst, XMMRegister src, int vector_len) {
5095   assert(VM_Version::supports_avx(), "");
5096   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5097   attributes.set_rex_vex_w_reverted();
5098   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5099   emit_int8(0x51);
5100   emit_int8((unsigned char)(0xC0 | encode));
5101 }
5102 
5103 void Assembler::vsqrtpd(XMMRegister dst, Address src, int vector_len) {
5104   assert(VM_Version::supports_avx(), "");
5105   InstructionMark im(this);
5106   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5107   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
5108   attributes.set_rex_vex_w_reverted();
5109   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5110   emit_int8(0x51);
5111   emit_operand(dst, src);
5112 }
5113 
5114 void Assembler::andpd(XMMRegister dst, XMMRegister src) {
5115   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5116   InstructionAttr attributes(AVX_128bit, /* rex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
5117   attributes.set_rex_vex_w_reverted();
5118   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5119   emit_int8(0x54);
5120   emit_int8((unsigned char)(0xC0 | encode));
5121 }
5122 
5123 void Assembler::andps(XMMRegister dst, XMMRegister src) {
5124   NOT_LP64(assert(VM_Version::supports_sse(), ""));
5125   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
5126   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
5127   emit_int8(0x54);
5128   emit_int8((unsigned char)(0xC0 | encode));
5129 }
5130 
5131 void Assembler::andps(XMMRegister dst, Address src) {
5132   NOT_LP64(assert(VM_Version::supports_sse(), ""));
5133   InstructionMark im(this);
5134   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
5135   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
5136   simd_prefix(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
5137   emit_int8(0x54);
5138   emit_operand(dst, src);
5139 }
5140 
5141 void Assembler::andpd(XMMRegister dst, Address src) {
5142   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5143   InstructionMark im(this);
5144   InstructionAttr attributes(AVX_128bit, /* rex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
5145   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
5146   attributes.set_rex_vex_w_reverted();
5147   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5148   emit_int8(0x54);
5149   emit_operand(dst, src);
5150 }
5151 
5152 void Assembler::vandpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5153   assert(VM_Version::supports_avx(), "");
5154   InstructionAttr attributes(vector_len, /* vex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
5155   attributes.set_rex_vex_w_reverted();
5156   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5157   emit_int8(0x54);
5158   emit_int8((unsigned char)(0xC0 | encode));
5159 }
5160 
5161 void Assembler::vandps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5162   assert(VM_Version::supports_avx(), "");
5163   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
5164   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
5165   emit_int8(0x54);
5166   emit_int8((unsigned char)(0xC0 | encode));
5167 }
5168 
5169 void Assembler::vandpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5170   assert(VM_Version::supports_avx(), "");
5171   InstructionMark im(this);
5172   InstructionAttr attributes(vector_len, /* vex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
5173   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
5174   attributes.set_rex_vex_w_reverted();
5175   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5176   emit_int8(0x54);
5177   emit_operand(dst, src);
5178 }
5179 
5180 void Assembler::vandps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5181   assert(VM_Version::supports_avx(), "");
5182   InstructionMark im(this);
5183   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
5184   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
5185   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
5186   emit_int8(0x54);
5187   emit_operand(dst, src);
5188 }
5189 
5190 void Assembler::unpckhpd(XMMRegister dst, XMMRegister src) {
5191   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5192   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5193   attributes.set_rex_vex_w_reverted();
5194   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5195   emit_int8(0x15);
5196   emit_int8((unsigned char)(0xC0 | encode));
5197 }
5198 
5199 void Assembler::unpcklpd(XMMRegister dst, XMMRegister src) {
5200   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5201   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5202   attributes.set_rex_vex_w_reverted();
5203   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5204   emit_int8(0x14);
5205   emit_int8((unsigned char)(0xC0 | encode));
5206 }
5207 
5208 void Assembler::xorpd(XMMRegister dst, XMMRegister src) {
5209   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5210   InstructionAttr attributes(AVX_128bit, /* rex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
5211   attributes.set_rex_vex_w_reverted();
5212   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5213   emit_int8(0x57);
5214   emit_int8((unsigned char)(0xC0 | encode));
5215 }
5216 
5217 void Assembler::xorps(XMMRegister dst, XMMRegister src) {
5218   NOT_LP64(assert(VM_Version::supports_sse(), ""));
5219   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
5220   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
5221   emit_int8(0x57);
5222   emit_int8((unsigned char)(0xC0 | encode));
5223 }
5224 
5225 void Assembler::xorpd(XMMRegister dst, Address src) {
5226   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5227   InstructionMark im(this);
5228   InstructionAttr attributes(AVX_128bit, /* rex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
5229   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
5230   attributes.set_rex_vex_w_reverted();
5231   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5232   emit_int8(0x57);
5233   emit_operand(dst, src);
5234 }
5235 
5236 void Assembler::xorps(XMMRegister dst, Address src) {
5237   NOT_LP64(assert(VM_Version::supports_sse(), ""));
5238   InstructionMark im(this);
5239   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
5240   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
5241   simd_prefix(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
5242   emit_int8(0x57);
5243   emit_operand(dst, src);
5244 }
5245 
5246 void Assembler::vxorpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5247   assert(VM_Version::supports_avx(), "");
5248   InstructionAttr attributes(vector_len, /* vex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
5249   attributes.set_rex_vex_w_reverted();
5250   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5251   emit_int8(0x57);
5252   emit_int8((unsigned char)(0xC0 | encode));
5253 }
5254 
5255 void Assembler::vxorps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5256   assert(VM_Version::supports_avx(), "");
5257   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
5258   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
5259   emit_int8(0x57);
5260   emit_int8((unsigned char)(0xC0 | encode));
5261 }
5262 
5263 void Assembler::vxorpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5264   assert(VM_Version::supports_avx(), "");
5265   InstructionMark im(this);
5266   InstructionAttr attributes(vector_len, /* vex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
5267   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
5268   attributes.set_rex_vex_w_reverted();
5269   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5270   emit_int8(0x57);
5271   emit_operand(dst, src);
5272 }
5273 
5274 void Assembler::vxorps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5275   assert(VM_Version::supports_avx(), "");
5276   InstructionMark im(this);
5277   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
5278   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
5279   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
5280   emit_int8(0x57);
5281   emit_operand(dst, src);
5282 }
5283 
5284 // Integer vector arithmetic
5285 void Assembler::vphaddw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5286   assert(VM_Version::supports_avx() && (vector_len == 0) ||
5287          VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
5288   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
5289   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5290   emit_int8(0x01);
5291   emit_int8((unsigned char)(0xC0 | encode));
5292 }
5293 
5294 void Assembler::vphaddd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5295   assert(VM_Version::supports_avx() && (vector_len == 0) ||
5296          VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
5297   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
5298   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5299   emit_int8(0x02);
5300   emit_int8((unsigned char)(0xC0 | encode));
5301 }
5302 
5303 void Assembler::paddb(XMMRegister dst, XMMRegister src) {
5304   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5305   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5306   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5307   emit_int8((unsigned char)0xFC);
5308   emit_int8((unsigned char)(0xC0 | encode));
5309 }
5310 
5311 void Assembler::paddw(XMMRegister dst, XMMRegister src) {
5312   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5313   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5314   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5315   emit_int8((unsigned char)0xFD);
5316   emit_int8((unsigned char)(0xC0 | encode));
5317 }
5318 
5319 void Assembler::paddd(XMMRegister dst, XMMRegister src) {
5320   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5321   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5322   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5323   emit_int8((unsigned char)0xFE);
5324   emit_int8((unsigned char)(0xC0 | encode));
5325 }
5326 
5327 void Assembler::paddd(XMMRegister dst, Address src) {
5328   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5329   InstructionMark im(this);
5330   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5331   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5332   emit_int8((unsigned char)0xFE);
5333   emit_operand(dst, src);
5334 }
5335 
5336 void Assembler::paddq(XMMRegister dst, XMMRegister src) {
5337   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5338   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5339   attributes.set_rex_vex_w_reverted();
5340   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5341   emit_int8((unsigned char)0xD4);
5342   emit_int8((unsigned char)(0xC0 | encode));
5343 }
5344 
5345 void Assembler::phaddw(XMMRegister dst, XMMRegister src) {
5346   assert(VM_Version::supports_sse3(), "");
5347   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
5348   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5349   emit_int8(0x01);
5350   emit_int8((unsigned char)(0xC0 | encode));
5351 }
5352 
5353 void Assembler::phaddd(XMMRegister dst, XMMRegister src) {
5354   assert(VM_Version::supports_sse3(), "");
5355   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
5356   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5357   emit_int8(0x02);
5358   emit_int8((unsigned char)(0xC0 | encode));
5359 }
5360 
5361 void Assembler::vpaddb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5362   assert(UseAVX > 0, "requires some form of AVX");
5363   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5364   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5365   emit_int8((unsigned char)0xFC);
5366   emit_int8((unsigned char)(0xC0 | encode));
5367 }
5368 
5369 void Assembler::vpaddw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5370   assert(UseAVX > 0, "requires some form of AVX");
5371   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5372   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5373   emit_int8((unsigned char)0xFD);
5374   emit_int8((unsigned char)(0xC0 | encode));
5375 }
5376 
5377 void Assembler::vpaddd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5378   assert(UseAVX > 0, "requires some form of AVX");
5379   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5380   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5381   emit_int8((unsigned char)0xFE);
5382   emit_int8((unsigned char)(0xC0 | encode));
5383 }
5384 
5385 void Assembler::vpaddq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5386   assert(UseAVX > 0, "requires some form of AVX");
5387   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5388   attributes.set_rex_vex_w_reverted();
5389   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5390   emit_int8((unsigned char)0xD4);
5391   emit_int8((unsigned char)(0xC0 | encode));
5392 }
5393 
5394 void Assembler::vpaddb(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5395   assert(UseAVX > 0, "requires some form of AVX");
5396   InstructionMark im(this);
5397   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5398   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
5399   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5400   emit_int8((unsigned char)0xFC);
5401   emit_operand(dst, src);
5402 }
5403 
5404 void Assembler::vpaddw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5405   assert(UseAVX > 0, "requires some form of AVX");
5406   InstructionMark im(this);
5407   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5408   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
5409   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5410   emit_int8((unsigned char)0xFD);
5411   emit_operand(dst, src);
5412 }
5413 
5414 void Assembler::vpaddd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5415   assert(UseAVX > 0, "requires some form of AVX");
5416   InstructionMark im(this);
5417   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5418   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
5419   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5420   emit_int8((unsigned char)0xFE);
5421   emit_operand(dst, src);
5422 }
5423 
5424 void Assembler::vpaddq(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5425   assert(UseAVX > 0, "requires some form of AVX");
5426   InstructionMark im(this);
5427   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5428   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
5429   attributes.set_rex_vex_w_reverted();
5430   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5431   emit_int8((unsigned char)0xD4);
5432   emit_operand(dst, src);
5433 }
5434 
5435 void Assembler::psubb(XMMRegister dst, XMMRegister src) {
5436   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5437   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5438   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5439   emit_int8((unsigned char)0xF8);
5440   emit_int8((unsigned char)(0xC0 | encode));
5441 }
5442 
5443 void Assembler::psubw(XMMRegister dst, XMMRegister src) {
5444   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5445   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5446   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5447   emit_int8((unsigned char)0xF9);
5448   emit_int8((unsigned char)(0xC0 | encode));
5449 }
5450 
5451 void Assembler::psubd(XMMRegister dst, XMMRegister src) {
5452   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5453   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5454   emit_int8((unsigned char)0xFA);
5455   emit_int8((unsigned char)(0xC0 | encode));
5456 }
5457 
5458 void Assembler::psubq(XMMRegister dst, XMMRegister src) {
5459   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5460   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5461   attributes.set_rex_vex_w_reverted();
5462   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5463   emit_int8((unsigned char)0xFB);
5464   emit_int8((unsigned char)(0xC0 | encode));
5465 }
5466 
5467 void Assembler::vpsubb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5468   assert(UseAVX > 0, "requires some form of AVX");
5469   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5470   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5471   emit_int8((unsigned char)0xF8);
5472   emit_int8((unsigned char)(0xC0 | encode));
5473 }
5474 
5475 void Assembler::vpsubw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5476   assert(UseAVX > 0, "requires some form of AVX");
5477   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5478   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5479   emit_int8((unsigned char)0xF9);
5480   emit_int8((unsigned char)(0xC0 | encode));
5481 }
5482 
5483 void Assembler::vpsubd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5484   assert(UseAVX > 0, "requires some form of AVX");
5485   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5486   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5487   emit_int8((unsigned char)0xFA);
5488   emit_int8((unsigned char)(0xC0 | encode));
5489 }
5490 
5491 void Assembler::vpsubq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5492   assert(UseAVX > 0, "requires some form of AVX");
5493   InstructionAttr attributes(vector_len, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5494   attributes.set_rex_vex_w_reverted();
5495   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5496   emit_int8((unsigned char)0xFB);
5497   emit_int8((unsigned char)(0xC0 | encode));
5498 }
5499 
5500 void Assembler::vpsubb(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5501   assert(UseAVX > 0, "requires some form of AVX");
5502   InstructionMark im(this);
5503   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5504   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
5505   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5506   emit_int8((unsigned char)0xF8);
5507   emit_operand(dst, src);
5508 }
5509 
5510 void Assembler::vpsubw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5511   assert(UseAVX > 0, "requires some form of AVX");
5512   InstructionMark im(this);
5513   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5514   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
5515   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5516   emit_int8((unsigned char)0xF9);
5517   emit_operand(dst, src);
5518 }
5519 
5520 void Assembler::vpsubd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5521   assert(UseAVX > 0, "requires some form of AVX");
5522   InstructionMark im(this);
5523   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5524   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
5525   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5526   emit_int8((unsigned char)0xFA);
5527   emit_operand(dst, src);
5528 }
5529 
5530 void Assembler::vpsubq(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5531   assert(UseAVX > 0, "requires some form of AVX");
5532   InstructionMark im(this);
5533   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5534   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
5535   attributes.set_rex_vex_w_reverted();
5536   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5537   emit_int8((unsigned char)0xFB);
5538   emit_operand(dst, src);
5539 }
5540 
5541 void Assembler::pmullw(XMMRegister dst, XMMRegister src) {
5542   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5543   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5544   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5545   emit_int8((unsigned char)0xD5);
5546   emit_int8((unsigned char)(0xC0 | encode));
5547 }
5548 
5549 void Assembler::pmulld(XMMRegister dst, XMMRegister src) {
5550   assert(VM_Version::supports_sse4_1(), "");
5551   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5552   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5553   emit_int8(0x40);
5554   emit_int8((unsigned char)(0xC0 | encode));
5555 }
5556 
5557 void Assembler::vpmullw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5558   assert(UseAVX > 0, "requires some form of AVX");
5559   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5560   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5561   emit_int8((unsigned char)0xD5);
5562   emit_int8((unsigned char)(0xC0 | encode));
5563 }
5564 
5565 void Assembler::vpmulld(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5566   assert(UseAVX > 0, "requires some form of AVX");
5567   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5568   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5569   emit_int8(0x40);
5570   emit_int8((unsigned char)(0xC0 | encode));
5571 }
5572 
5573 void Assembler::vpmullq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5574   assert(UseAVX > 2, "requires some form of EVEX");
5575   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
5576   attributes.set_is_evex_instruction();
5577   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5578   emit_int8(0x40);
5579   emit_int8((unsigned char)(0xC0 | encode));
5580 }
5581 
5582 void Assembler::vpmullw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5583   assert(UseAVX > 0, "requires some form of AVX");
5584   InstructionMark im(this);
5585   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5586   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
5587   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5588   emit_int8((unsigned char)0xD5);
5589   emit_operand(dst, src);
5590 }
5591 
5592 void Assembler::vpmulld(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5593   assert(UseAVX > 0, "requires some form of AVX");
5594   InstructionMark im(this);
5595   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5596   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
5597   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5598   emit_int8(0x40);
5599   emit_operand(dst, src);
5600 }
5601 
5602 void Assembler::vpmullq(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5603   assert(UseAVX > 2, "requires some form of EVEX");
5604   InstructionMark im(this);
5605   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
5606   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
5607   attributes.set_is_evex_instruction();
5608   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5609   emit_int8(0x40);
5610   emit_operand(dst, src);
5611 }
5612 
5613 // Shift packed integers left by specified number of bits.
5614 void Assembler::psllw(XMMRegister dst, int shift) {
5615   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5616   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5617   // XMM6 is for /6 encoding: 66 0F 71 /6 ib
5618   int encode = simd_prefix_and_encode(xmm6, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5619   emit_int8(0x71);
5620   emit_int8((unsigned char)(0xC0 | encode));
5621   emit_int8(shift & 0xFF);
5622 }
5623 
5624 void Assembler::pslld(XMMRegister dst, int shift) {
5625   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5626   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5627   // XMM6 is for /6 encoding: 66 0F 72 /6 ib
5628   int encode = simd_prefix_and_encode(xmm6, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5629   emit_int8(0x72);
5630   emit_int8((unsigned char)(0xC0 | encode));
5631   emit_int8(shift & 0xFF);
5632 }
5633 
5634 void Assembler::psllq(XMMRegister dst, int shift) {
5635   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5636   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5637   // XMM6 is for /6 encoding: 66 0F 73 /6 ib
5638   int encode = simd_prefix_and_encode(xmm6, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5639   emit_int8(0x73);
5640   emit_int8((unsigned char)(0xC0 | encode));
5641   emit_int8(shift & 0xFF);
5642 }
5643 
5644 void Assembler::psllw(XMMRegister dst, XMMRegister shift) {
5645   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5646   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5647   int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5648   emit_int8((unsigned char)0xF1);
5649   emit_int8((unsigned char)(0xC0 | encode));
5650 }
5651 
5652 void Assembler::pslld(XMMRegister dst, XMMRegister shift) {
5653   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5654   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5655   int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5656   emit_int8((unsigned char)0xF2);
5657   emit_int8((unsigned char)(0xC0 | encode));
5658 }
5659 
5660 void Assembler::psllq(XMMRegister dst, XMMRegister shift) {
5661   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5662   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5663   attributes.set_rex_vex_w_reverted();
5664   int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5665   emit_int8((unsigned char)0xF3);
5666   emit_int8((unsigned char)(0xC0 | encode));
5667 }
5668 
5669 void Assembler::vpsllw(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
5670   assert(UseAVX > 0, "requires some form of AVX");
5671   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5672   // XMM6 is for /6 encoding: 66 0F 71 /6 ib
5673   int encode = vex_prefix_and_encode(xmm6->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5674   emit_int8(0x71);
5675   emit_int8((unsigned char)(0xC0 | encode));
5676   emit_int8(shift & 0xFF);
5677 }
5678 
5679 void Assembler::vpslld(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
5680   assert(UseAVX > 0, "requires some form of AVX");
5681   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5682   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5683   // XMM6 is for /6 encoding: 66 0F 72 /6 ib
5684   int encode = vex_prefix_and_encode(xmm6->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5685   emit_int8(0x72);
5686   emit_int8((unsigned char)(0xC0 | encode));
5687   emit_int8(shift & 0xFF);
5688 }
5689 
5690 void Assembler::vpsllq(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
5691   assert(UseAVX > 0, "requires some form of AVX");
5692   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5693   attributes.set_rex_vex_w_reverted();
5694   // XMM6 is for /6 encoding: 66 0F 73 /6 ib
5695   int encode = vex_prefix_and_encode(xmm6->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5696   emit_int8(0x73);
5697   emit_int8((unsigned char)(0xC0 | encode));
5698   emit_int8(shift & 0xFF);
5699 }
5700 
5701 void Assembler::vpsllw(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
5702   assert(UseAVX > 0, "requires some form of AVX");
5703   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5704   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5705   emit_int8((unsigned char)0xF1);
5706   emit_int8((unsigned char)(0xC0 | encode));
5707 }
5708 
5709 void Assembler::vpslld(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
5710   assert(UseAVX > 0, "requires some form of AVX");
5711   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5712   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5713   emit_int8((unsigned char)0xF2);
5714   emit_int8((unsigned char)(0xC0 | encode));
5715 }
5716 
5717 void Assembler::vpsllq(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
5718   assert(UseAVX > 0, "requires some form of AVX");
5719   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5720   attributes.set_rex_vex_w_reverted();
5721   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5722   emit_int8((unsigned char)0xF3);
5723   emit_int8((unsigned char)(0xC0 | encode));
5724 }
5725 
5726 // Shift packed integers logically right by specified number of bits.
5727 void Assembler::psrlw(XMMRegister dst, int shift) {
5728   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5729   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5730   // XMM2 is for /2 encoding: 66 0F 71 /2 ib
5731   int encode = simd_prefix_and_encode(xmm2, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5732   emit_int8(0x71);
5733   emit_int8((unsigned char)(0xC0 | encode));
5734   emit_int8(shift & 0xFF);
5735 }
5736 
5737 void Assembler::psrld(XMMRegister dst, int shift) {
5738   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5739   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5740   // XMM2 is for /2 encoding: 66 0F 72 /2 ib
5741   int encode = simd_prefix_and_encode(xmm2, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5742   emit_int8(0x72);
5743   emit_int8((unsigned char)(0xC0 | encode));
5744   emit_int8(shift & 0xFF);
5745 }
5746 
5747 void Assembler::psrlq(XMMRegister dst, int shift) {
5748   // Do not confuse it with psrldq SSE2 instruction which
5749   // shifts 128 bit value in xmm register by number of bytes.
5750   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5751   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5752   attributes.set_rex_vex_w_reverted();
5753   // XMM2 is for /2 encoding: 66 0F 73 /2 ib
5754   int encode = simd_prefix_and_encode(xmm2, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5755   emit_int8(0x73);
5756   emit_int8((unsigned char)(0xC0 | encode));
5757   emit_int8(shift & 0xFF);
5758 }
5759 
5760 void Assembler::psrlw(XMMRegister dst, XMMRegister shift) {
5761   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5762   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5763   int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5764   emit_int8((unsigned char)0xD1);
5765   emit_int8((unsigned char)(0xC0 | encode));
5766 }
5767 
5768 void Assembler::psrld(XMMRegister dst, XMMRegister shift) {
5769   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5770   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5771   int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5772   emit_int8((unsigned char)0xD2);
5773   emit_int8((unsigned char)(0xC0 | encode));
5774 }
5775 
5776 void Assembler::psrlq(XMMRegister dst, XMMRegister shift) {
5777   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5778   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5779   attributes.set_rex_vex_w_reverted();
5780   int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5781   emit_int8((unsigned char)0xD3);
5782   emit_int8((unsigned char)(0xC0 | encode));
5783 }
5784 
5785 void Assembler::vpsrlw(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
5786   assert(UseAVX > 0, "requires some form of AVX");
5787   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5788   // XMM2 is for /2 encoding: 66 0F 71 /2 ib
5789   int encode = vex_prefix_and_encode(xmm2->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5790   emit_int8(0x71);
5791   emit_int8((unsigned char)(0xC0 | encode));
5792   emit_int8(shift & 0xFF);
5793 }
5794 
5795 void Assembler::vpsrld(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
5796   assert(UseAVX > 0, "requires some form of AVX");
5797   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5798   // XMM2 is for /2 encoding: 66 0F 72 /2 ib
5799   int encode = vex_prefix_and_encode(xmm2->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5800   emit_int8(0x72);
5801   emit_int8((unsigned char)(0xC0 | encode));
5802   emit_int8(shift & 0xFF);
5803 }
5804 
5805 void Assembler::vpsrlq(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
5806   assert(UseAVX > 0, "requires some form of AVX");
5807   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5808   attributes.set_rex_vex_w_reverted();
5809   // XMM2 is for /2 encoding: 66 0F 73 /2 ib
5810   int encode = vex_prefix_and_encode(xmm2->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5811   emit_int8(0x73);
5812   emit_int8((unsigned char)(0xC0 | encode));
5813   emit_int8(shift & 0xFF);
5814 }
5815 
5816 void Assembler::vpsrlw(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
5817   assert(UseAVX > 0, "requires some form of AVX");
5818   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5819   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5820   emit_int8((unsigned char)0xD1);
5821   emit_int8((unsigned char)(0xC0 | encode));
5822 }
5823 
5824 void Assembler::vpsrld(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
5825   assert(UseAVX > 0, "requires some form of AVX");
5826   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5827   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5828   emit_int8((unsigned char)0xD2);
5829   emit_int8((unsigned char)(0xC0 | encode));
5830 }
5831 
5832 void Assembler::vpsrlq(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
5833   assert(UseAVX > 0, "requires some form of AVX");
5834   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5835   attributes.set_rex_vex_w_reverted();
5836   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5837   emit_int8((unsigned char)0xD3);
5838   emit_int8((unsigned char)(0xC0 | encode));
5839 }
5840 
5841 // Shift packed integers arithmetically right by specified number of bits.
5842 void Assembler::psraw(XMMRegister dst, int shift) {
5843   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5844   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5845   // XMM4 is for /4 encoding: 66 0F 71 /4 ib
5846   int encode = simd_prefix_and_encode(xmm4, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5847   emit_int8(0x71);
5848   emit_int8((unsigned char)(0xC0 | encode));
5849   emit_int8(shift & 0xFF);
5850 }
5851 
5852 void Assembler::psrad(XMMRegister dst, int shift) {
5853   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5854   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5855   // XMM4 is for /4 encoding: 66 0F 72 /4 ib
5856   int encode = simd_prefix_and_encode(xmm4, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5857   emit_int8(0x72);
5858   emit_int8((unsigned char)(0xC0 | encode));
5859   emit_int8(shift & 0xFF);
5860 }
5861 
5862 void Assembler::psraw(XMMRegister dst, XMMRegister shift) {
5863   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5864   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5865   int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5866   emit_int8((unsigned char)0xE1);
5867   emit_int8((unsigned char)(0xC0 | encode));
5868 }
5869 
5870 void Assembler::psrad(XMMRegister dst, XMMRegister shift) {
5871   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5872   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5873   int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5874   emit_int8((unsigned char)0xE2);
5875   emit_int8((unsigned char)(0xC0 | encode));
5876 }
5877 
5878 void Assembler::vpsraw(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
5879   assert(UseAVX > 0, "requires some form of AVX");
5880   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5881   // XMM4 is for /4 encoding: 66 0F 71 /4 ib
5882   int encode = vex_prefix_and_encode(xmm4->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5883   emit_int8(0x71);
5884   emit_int8((unsigned char)(0xC0 | encode));
5885   emit_int8(shift & 0xFF);
5886 }
5887 
5888 void Assembler::vpsrad(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
5889   assert(UseAVX > 0, "requires some form of AVX");
5890   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5891   // XMM4 is for /4 encoding: 66 0F 71 /4 ib
5892   int encode = vex_prefix_and_encode(xmm4->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5893   emit_int8(0x72);
5894   emit_int8((unsigned char)(0xC0 | encode));
5895   emit_int8(shift & 0xFF);
5896 }
5897 
5898 void Assembler::vpsraw(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
5899   assert(UseAVX > 0, "requires some form of AVX");
5900   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5901   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5902   emit_int8((unsigned char)0xE1);
5903   emit_int8((unsigned char)(0xC0 | encode));
5904 }
5905 
5906 void Assembler::vpsrad(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
5907   assert(UseAVX > 0, "requires some form of AVX");
5908   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5909   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5910   emit_int8((unsigned char)0xE2);
5911   emit_int8((unsigned char)(0xC0 | encode));
5912 }
5913 
5914 
5915 // logical operations packed integers
5916 void Assembler::pand(XMMRegister dst, XMMRegister src) {
5917   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5918   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5919   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5920   emit_int8((unsigned char)0xDB);
5921   emit_int8((unsigned char)(0xC0 | encode));
5922 }
5923 
5924 void Assembler::vpand(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5925   assert(UseAVX > 0, "requires some form of AVX");
5926   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5927   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5928   emit_int8((unsigned char)0xDB);
5929   emit_int8((unsigned char)(0xC0 | encode));
5930 }
5931 
5932 void Assembler::vpand(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5933   assert(UseAVX > 0, "requires some form of AVX");
5934   InstructionMark im(this);
5935   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5936   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
5937   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5938   emit_int8((unsigned char)0xDB);
5939   emit_operand(dst, src);
5940 }
5941 
5942 void Assembler::pandn(XMMRegister dst, XMMRegister src) {
5943   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5944   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5945   attributes.set_rex_vex_w_reverted();
5946   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5947   emit_int8((unsigned char)0xDF);
5948   emit_int8((unsigned char)(0xC0 | encode));
5949 }
5950 
5951 void Assembler::por(XMMRegister dst, XMMRegister src) {
5952   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5953   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5954   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5955   emit_int8((unsigned char)0xEB);
5956   emit_int8((unsigned char)(0xC0 | encode));
5957 }
5958 
5959 void Assembler::vpor(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5960   assert(UseAVX > 0, "requires some form of AVX");
5961   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5962   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5963   emit_int8((unsigned char)0xEB);
5964   emit_int8((unsigned char)(0xC0 | encode));
5965 }
5966 
5967 void Assembler::vpor(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5968   assert(UseAVX > 0, "requires some form of AVX");
5969   InstructionMark im(this);
5970   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5971   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
5972   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5973   emit_int8((unsigned char)0xEB);
5974   emit_operand(dst, src);
5975 }
5976 
5977 void Assembler::pxor(XMMRegister dst, XMMRegister src) {
5978   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5979   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5980   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5981   emit_int8((unsigned char)0xEF);
5982   emit_int8((unsigned char)(0xC0 | encode));
5983 }
5984 
5985 void Assembler::vpxor(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5986   assert(UseAVX > 0, "requires some form of AVX");
5987   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5988   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5989   emit_int8((unsigned char)0xEF);
5990   emit_int8((unsigned char)(0xC0 | encode));
5991 }
5992 
5993 void Assembler::vpxor(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5994   assert(UseAVX > 0, "requires some form of AVX");
5995   InstructionMark im(this);
5996   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5997   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
5998   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5999   emit_int8((unsigned char)0xEF);
6000   emit_operand(dst, src);
6001 }
6002 
6003 
6004 // vinserti forms
6005 
6006 void Assembler::vinserti128(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) {
6007   assert(VM_Version::supports_avx2(), "");
6008   assert(imm8 <= 0x01, "imm8: %u", imm8);
6009   int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_256bit;
6010   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6011   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6012   emit_int8(0x38);
6013   emit_int8((unsigned char)(0xC0 | encode));
6014   // 0x00 - insert into lower 128 bits
6015   // 0x01 - insert into upper 128 bits
6016   emit_int8(imm8 & 0x01);
6017 }
6018 
6019 void Assembler::vinserti128(XMMRegister dst, XMMRegister nds, Address src, uint8_t imm8) {
6020   assert(VM_Version::supports_avx2(), "");
6021   assert(dst != xnoreg, "sanity");
6022   assert(imm8 <= 0x01, "imm8: %u", imm8);
6023   int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_256bit;
6024   InstructionMark im(this);
6025   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6026   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
6027   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6028   emit_int8(0x38);
6029   emit_operand(dst, src);
6030   // 0x00 - insert into lower 128 bits
6031   // 0x01 - insert into upper 128 bits
6032   emit_int8(imm8 & 0x01);
6033 }
6034 
6035 void Assembler::vinserti32x4(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) {
6036   assert(VM_Version::supports_evex(), "");
6037   assert(imm8 <= 0x03, "imm8: %u", imm8);
6038   InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6039   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6040   emit_int8(0x38);
6041   emit_int8((unsigned char)(0xC0 | encode));
6042   // 0x00 - insert into q0 128 bits (0..127)
6043   // 0x01 - insert into q1 128 bits (128..255)
6044   // 0x02 - insert into q2 128 bits (256..383)
6045   // 0x03 - insert into q3 128 bits (384..511)
6046   emit_int8(imm8 & 0x03);
6047 }
6048 
6049 void Assembler::vinserti32x4(XMMRegister dst, XMMRegister nds, Address src, uint8_t imm8) {
6050   assert(VM_Version::supports_avx(), "");
6051   assert(dst != xnoreg, "sanity");
6052   assert(imm8 <= 0x03, "imm8: %u", imm8);
6053   int vector_len = VM_Version::supports_evex() ? AVX_512bit : AVX_256bit;
6054   InstructionMark im(this);
6055   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6056   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
6057   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6058   emit_int8(0x18);
6059   emit_operand(dst, src);
6060   // 0x00 - insert into q0 128 bits (0..127)
6061   // 0x01 - insert into q1 128 bits (128..255)
6062   // 0x02 - insert into q2 128 bits (256..383)
6063   // 0x03 - insert into q3 128 bits (384..511)
6064   emit_int8(imm8 & 0x03);
6065 }
6066 
6067 void Assembler::vinserti64x4(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) {
6068   assert(VM_Version::supports_evex(), "");
6069   assert(imm8 <= 0x01, "imm8: %u", imm8);
6070   InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6071   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6072   emit_int8(0x38);
6073   emit_int8((unsigned char)(0xC0 | encode));
6074   // 0x00 - insert into lower 256 bits
6075   // 0x01 - insert into upper 256 bits
6076   emit_int8(imm8 & 0x01);
6077 }
6078 
6079 
6080 // vinsertf forms
6081 
6082 void Assembler::vinsertf128(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) {
6083   assert(VM_Version::supports_avx(), "");
6084   assert(imm8 <= 0x01, "imm8: %u", imm8);
6085   int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_256bit;
6086   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6087   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6088   emit_int8(0x18);
6089   emit_int8((unsigned char)(0xC0 | encode));
6090   // 0x00 - insert into lower 128 bits
6091   // 0x01 - insert into upper 128 bits
6092   emit_int8(imm8 & 0x01);
6093 }
6094 
6095 void Assembler::vinsertf128(XMMRegister dst, XMMRegister nds, Address src, uint8_t imm8) {
6096   assert(VM_Version::supports_avx(), "");
6097   assert(dst != xnoreg, "sanity");
6098   assert(imm8 <= 0x01, "imm8: %u", imm8);
6099   int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_256bit;
6100   InstructionMark im(this);
6101   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6102   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
6103   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6104   emit_int8(0x18);
6105   emit_operand(dst, src);
6106   // 0x00 - insert into lower 128 bits
6107   // 0x01 - insert into upper 128 bits
6108   emit_int8(imm8 & 0x01);
6109 }
6110 
6111 void Assembler::vinsertf32x4(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) {
6112   assert(VM_Version::supports_evex(), "");
6113   assert(imm8 <= 0x03, "imm8: %u", imm8);
6114   InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6115   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6116   emit_int8(0x18);
6117   emit_int8((unsigned char)(0xC0 | encode));
6118   // 0x00 - insert into q0 128 bits (0..127)
6119   // 0x01 - insert into q1 128 bits (128..255)
6120   // 0x02 - insert into q2 128 bits (256..383)
6121   // 0x03 - insert into q3 128 bits (384..511)
6122   emit_int8(imm8 & 0x03);
6123 }
6124 
6125 void Assembler::vinsertf32x4(XMMRegister dst, XMMRegister nds, Address src, uint8_t imm8) {
6126   assert(VM_Version::supports_avx(), "");
6127   assert(dst != xnoreg, "sanity");
6128   assert(imm8 <= 0x03, "imm8: %u", imm8);
6129   int vector_len = VM_Version::supports_evex() ? AVX_512bit : AVX_256bit;
6130   InstructionMark im(this);
6131   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6132   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
6133   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6134   emit_int8(0x18);
6135   emit_operand(dst, src);
6136   // 0x00 - insert into q0 128 bits (0..127)
6137   // 0x01 - insert into q1 128 bits (128..255)
6138   // 0x02 - insert into q2 128 bits (256..383)
6139   // 0x03 - insert into q3 128 bits (384..511)
6140   emit_int8(imm8 & 0x03);
6141 }
6142 
6143 void Assembler::vinsertf64x4(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) {
6144   assert(VM_Version::supports_evex(), "");
6145   assert(imm8 <= 0x01, "imm8: %u", imm8);
6146   InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6147   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6148   emit_int8(0x1A);
6149   emit_int8((unsigned char)(0xC0 | encode));
6150   // 0x00 - insert into lower 256 bits
6151   // 0x01 - insert into upper 256 bits
6152   emit_int8(imm8 & 0x01);
6153 }
6154 
6155 void Assembler::vinsertf64x4(XMMRegister dst, XMMRegister nds, Address src, uint8_t imm8) {
6156   assert(VM_Version::supports_evex(), "");
6157   assert(dst != xnoreg, "sanity");
6158   assert(imm8 <= 0x01, "imm8: %u", imm8);
6159   InstructionMark im(this);
6160   InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6161   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_64bit);
6162   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6163   emit_int8(0x1A);
6164   emit_operand(dst, src);
6165   // 0x00 - insert into lower 256 bits
6166   // 0x01 - insert into upper 256 bits
6167   emit_int8(imm8 & 0x01);
6168 }
6169 
6170 
6171 // vextracti forms
6172 
6173 void Assembler::vextracti128(XMMRegister dst, XMMRegister src, uint8_t imm8) {
6174   assert(VM_Version::supports_avx(), "");
6175   assert(imm8 <= 0x01, "imm8: %u", imm8);
6176   int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_256bit;
6177   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6178   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6179   emit_int8(0x39);
6180   emit_int8((unsigned char)(0xC0 | encode));
6181   // 0x00 - extract from lower 128 bits
6182   // 0x01 - extract from upper 128 bits
6183   emit_int8(imm8 & 0x01);
6184 }
6185 
6186 void Assembler::vextracti128(Address dst, XMMRegister src, uint8_t imm8) {
6187   assert(VM_Version::supports_avx2(), "");
6188   assert(src != xnoreg, "sanity");
6189   assert(imm8 <= 0x01, "imm8: %u", imm8);
6190   int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_256bit;
6191   InstructionMark im(this);
6192   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6193   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
6194   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6195   emit_int8(0x39);
6196   emit_operand(src, dst);
6197   // 0x00 - extract from lower 128 bits
6198   // 0x01 - extract from upper 128 bits
6199   emit_int8(imm8 & 0x01);
6200 }
6201 
6202 void Assembler::vextracti32x4(XMMRegister dst, XMMRegister src, uint8_t imm8) {
6203   assert(VM_Version::supports_avx(), "");
6204   assert(imm8 <= 0x03, "imm8: %u", imm8);
6205   int vector_len = VM_Version::supports_evex() ? AVX_512bit : AVX_256bit;
6206   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6207   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6208   emit_int8(0x39);
6209   emit_int8((unsigned char)(0xC0 | encode));
6210   // 0x00 - extract from bits 127:0
6211   // 0x01 - extract from bits 255:128
6212   // 0x02 - extract from bits 383:256
6213   // 0x03 - extract from bits 511:384
6214   emit_int8(imm8 & 0x03);
6215 }
6216 
6217 void Assembler::vextracti32x4(Address dst, XMMRegister src, uint8_t imm8) {
6218   assert(VM_Version::supports_evex(), "");
6219   assert(src != xnoreg, "sanity");
6220   assert(imm8 <= 0x03, "imm8: %u", imm8);
6221   InstructionMark im(this);
6222   InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6223   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
6224   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6225   emit_int8(0x39);
6226   emit_operand(src, dst);
6227   // 0x00 - extract from bits 127:0
6228   // 0x01 - extract from bits 255:128
6229   // 0x02 - extract from bits 383:256
6230   // 0x03 - extract from bits 511:384
6231   emit_int8(imm8 & 0x03);
6232 }
6233 
6234 void Assembler::vextracti64x2(XMMRegister dst, XMMRegister src, uint8_t imm8) {
6235   assert(VM_Version::supports_avx512dq(), "");
6236   assert(imm8 <= 0x03, "imm8: %u", imm8);
6237   InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6238   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6239   emit_int8(0x39);
6240   emit_int8((unsigned char)(0xC0 | encode));
6241   // 0x00 - extract from bits 127:0
6242   // 0x01 - extract from bits 255:128
6243   // 0x02 - extract from bits 383:256
6244   // 0x03 - extract from bits 511:384
6245   emit_int8(imm8 & 0x03);
6246 }
6247 
6248 void Assembler::vextracti64x4(XMMRegister dst, XMMRegister src, uint8_t imm8) {
6249   assert(VM_Version::supports_evex(), "");
6250   assert(imm8 <= 0x01, "imm8: %u", imm8);
6251   InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6252   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6253   emit_int8(0x3B);
6254   emit_int8((unsigned char)(0xC0 | encode));
6255   // 0x00 - extract from lower 256 bits
6256   // 0x01 - extract from upper 256 bits
6257   emit_int8(imm8 & 0x01);
6258 }
6259 
6260 
6261 // vextractf forms
6262 
6263 void Assembler::vextractf128(XMMRegister dst, XMMRegister src, uint8_t imm8) {
6264   assert(VM_Version::supports_avx(), "");
6265   assert(imm8 <= 0x01, "imm8: %u", imm8);
6266   int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_256bit;
6267   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6268   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6269   emit_int8(0x19);
6270   emit_int8((unsigned char)(0xC0 | encode));
6271   // 0x00 - extract from lower 128 bits
6272   // 0x01 - extract from upper 128 bits
6273   emit_int8(imm8 & 0x01);
6274 }
6275 
6276 void Assembler::vextractf128(Address dst, XMMRegister src, uint8_t imm8) {
6277   assert(VM_Version::supports_avx(), "");
6278   assert(src != xnoreg, "sanity");
6279   assert(imm8 <= 0x01, "imm8: %u", imm8);
6280   int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_256bit;
6281   InstructionMark im(this);
6282   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6283   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
6284   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6285   emit_int8(0x19);
6286   emit_operand(src, dst);
6287   // 0x00 - extract from lower 128 bits
6288   // 0x01 - extract from upper 128 bits
6289   emit_int8(imm8 & 0x01);
6290 }
6291 
6292 void Assembler::vextractf32x4(XMMRegister dst, XMMRegister src, uint8_t imm8) {
6293   assert(VM_Version::supports_avx(), "");
6294   assert(imm8 <= 0x03, "imm8: %u", imm8);
6295   int vector_len = VM_Version::supports_evex() ? AVX_512bit : AVX_256bit;
6296   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6297   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6298   emit_int8(0x19);
6299   emit_int8((unsigned char)(0xC0 | encode));
6300   // 0x00 - extract from bits 127:0
6301   // 0x01 - extract from bits 255:128
6302   // 0x02 - extract from bits 383:256
6303   // 0x03 - extract from bits 511:384
6304   emit_int8(imm8 & 0x03);
6305 }
6306 
6307 void Assembler::vextractf32x4(Address dst, XMMRegister src, uint8_t imm8) {
6308   assert(VM_Version::supports_evex(), "");
6309   assert(src != xnoreg, "sanity");
6310   assert(imm8 <= 0x03, "imm8: %u", imm8);
6311   InstructionMark im(this);
6312   InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6313   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
6314   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6315   emit_int8(0x19);
6316   emit_operand(src, dst);
6317   // 0x00 - extract from bits 127:0
6318   // 0x01 - extract from bits 255:128
6319   // 0x02 - extract from bits 383:256
6320   // 0x03 - extract from bits 511:384
6321   emit_int8(imm8 & 0x03);
6322 }
6323 
6324 void Assembler::vextractf64x2(XMMRegister dst, XMMRegister src, uint8_t imm8) {
6325   assert(VM_Version::supports_avx512dq(), "");
6326   assert(imm8 <= 0x03, "imm8: %u", imm8);
6327   InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6328   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6329   emit_int8(0x19);
6330   emit_int8((unsigned char)(0xC0 | encode));
6331   // 0x00 - extract from bits 127:0
6332   // 0x01 - extract from bits 255:128
6333   // 0x02 - extract from bits 383:256
6334   // 0x03 - extract from bits 511:384
6335   emit_int8(imm8 & 0x03);
6336 }
6337 
6338 void Assembler::vextractf64x4(XMMRegister dst, XMMRegister src, uint8_t imm8) {
6339   assert(VM_Version::supports_evex(), "");
6340   assert(imm8 <= 0x01, "imm8: %u", imm8);
6341   InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6342   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6343   emit_int8(0x1B);
6344   emit_int8((unsigned char)(0xC0 | encode));
6345   // 0x00 - extract from lower 256 bits
6346   // 0x01 - extract from upper 256 bits
6347   emit_int8(imm8 & 0x01);
6348 }
6349 
6350 void Assembler::vextractf64x4(Address dst, XMMRegister src, uint8_t imm8) {
6351   assert(VM_Version::supports_evex(), "");
6352   assert(src != xnoreg, "sanity");
6353   assert(imm8 <= 0x01, "imm8: %u", imm8);
6354   InstructionMark im(this);
6355   InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6356   attributes.set_address_attributes(/* tuple_type */ EVEX_T4,/* input_size_in_bits */  EVEX_64bit);
6357   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6358   emit_int8(0x1B);
6359   emit_operand(src, dst);
6360   // 0x00 - extract from lower 256 bits
6361   // 0x01 - extract from upper 256 bits
6362   emit_int8(imm8 & 0x01);
6363 }
6364 
6365 
6366 // legacy word/dword replicate
6367 void Assembler::vpbroadcastw(XMMRegister dst, XMMRegister src) {
6368   assert(VM_Version::supports_avx2(), "");
6369   InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
6370   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6371   emit_int8(0x79);
6372   emit_int8((unsigned char)(0xC0 | encode));
6373 }
6374 
6375 void Assembler::vpbroadcastd(XMMRegister dst, XMMRegister src) {
6376   assert(VM_Version::supports_avx2(), "");
6377   InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6378   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6379   emit_int8(0x58);
6380   emit_int8((unsigned char)(0xC0 | encode));
6381 }
6382 
6383 
6384 // xmm/mem sourced byte/word/dword/qword replicate
6385 
6386 // duplicate 1-byte integer data from src into programmed locations in dest : requires AVX512BW and AVX512VL
6387 void Assembler::evpbroadcastb(XMMRegister dst, XMMRegister src, int vector_len) {
6388   assert(VM_Version::supports_evex(), "");
6389   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
6390   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6391   emit_int8(0x78);
6392   emit_int8((unsigned char)(0xC0 | encode));
6393 }
6394 
6395 void Assembler::evpbroadcastb(XMMRegister dst, Address src, int vector_len) {
6396   assert(VM_Version::supports_evex(), "");
6397   assert(dst != xnoreg, "sanity");
6398   InstructionMark im(this);
6399   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
6400   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_8bit);
6401   // swap src<->dst for encoding
6402   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6403   emit_int8(0x78);
6404   emit_operand(dst, src);
6405 }
6406 
6407 // duplicate 2-byte integer data from src into programmed locations in dest : requires AVX512BW and AVX512VL
6408 void Assembler::evpbroadcastw(XMMRegister dst, XMMRegister src, int vector_len) {
6409   assert(VM_Version::supports_evex(), "");
6410   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
6411   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6412   emit_int8(0x79);
6413   emit_int8((unsigned char)(0xC0 | encode));
6414 }
6415 
6416 void Assembler::evpbroadcastw(XMMRegister dst, Address src, int vector_len) {
6417   assert(VM_Version::supports_evex(), "");
6418   assert(dst != xnoreg, "sanity");
6419   InstructionMark im(this);
6420   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
6421   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_16bit);
6422   // swap src<->dst for encoding
6423   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6424   emit_int8(0x79);
6425   emit_operand(dst, src);
6426 }
6427 
6428 // duplicate 4-byte integer data from src into programmed locations in dest : requires AVX512VL
6429 void Assembler::evpbroadcastd(XMMRegister dst, XMMRegister src, int vector_len) {
6430   assert(VM_Version::supports_evex(), "");
6431   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6432   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6433   emit_int8(0x58);
6434   emit_int8((unsigned char)(0xC0 | encode));
6435 }
6436 
6437 void Assembler::evpbroadcastd(XMMRegister dst, Address src, int vector_len) {
6438   assert(VM_Version::supports_evex(), "");
6439   assert(dst != xnoreg, "sanity");
6440   InstructionMark im(this);
6441   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6442   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
6443   // swap src<->dst for encoding
6444   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6445   emit_int8(0x58);
6446   emit_operand(dst, src);
6447 }
6448 
6449 // duplicate 8-byte integer data from src into programmed locations in dest : requires AVX512VL
6450 void Assembler::evpbroadcastq(XMMRegister dst, XMMRegister src, int vector_len) {
6451   assert(VM_Version::supports_evex(), "");
6452   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6453   attributes.set_rex_vex_w_reverted();
6454   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6455   emit_int8(0x59);
6456   emit_int8((unsigned char)(0xC0 | encode));
6457 }
6458 
6459 void Assembler::evpbroadcastq(XMMRegister dst, Address src, int vector_len) {
6460   assert(VM_Version::supports_evex(), "");
6461   assert(dst != xnoreg, "sanity");
6462   InstructionMark im(this);
6463   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6464   attributes.set_rex_vex_w_reverted();
6465   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
6466   // swap src<->dst for encoding
6467   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6468   emit_int8(0x59);
6469   emit_operand(dst, src);
6470 }
6471 
6472 
6473 // scalar single/double precision replicate
6474 
6475 // duplicate single precision data from src into programmed locations in dest : requires AVX512VL
6476 void Assembler::evpbroadcastss(XMMRegister dst, XMMRegister src, int vector_len) {
6477   assert(VM_Version::supports_evex(), "");
6478   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6479   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6480   emit_int8(0x18);
6481   emit_int8((unsigned char)(0xC0 | encode));
6482 }
6483 
6484 void Assembler::evpbroadcastss(XMMRegister dst, Address src, int vector_len) {
6485   assert(VM_Version::supports_evex(), "");
6486   assert(dst != xnoreg, "sanity");
6487   InstructionMark im(this);
6488   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6489   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
6490   // swap src<->dst for encoding
6491   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6492   emit_int8(0x18);
6493   emit_operand(dst, src);
6494 }
6495 
6496 // duplicate double precision data from src into programmed locations in dest : requires AVX512VL
6497 void Assembler::evpbroadcastsd(XMMRegister dst, XMMRegister src, int vector_len) {
6498   assert(VM_Version::supports_evex(), "");
6499   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6500   attributes.set_rex_vex_w_reverted();
6501   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6502   emit_int8(0x19);
6503   emit_int8((unsigned char)(0xC0 | encode));
6504 }
6505 
6506 void Assembler::evpbroadcastsd(XMMRegister dst, Address src, int vector_len) {
6507   assert(VM_Version::supports_evex(), "");
6508   assert(dst != xnoreg, "sanity");
6509   InstructionMark im(this);
6510   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6511   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
6512   attributes.set_rex_vex_w_reverted();
6513   // swap src<->dst for encoding
6514   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6515   emit_int8(0x19);
6516   emit_operand(dst, src);
6517 }
6518 
6519 
6520 // gpr source broadcast forms
6521 
6522 // duplicate 1-byte integer data from src into programmed locations in dest : requires AVX512BW and AVX512VL
6523 void Assembler::evpbroadcastb(XMMRegister dst, Register src, int vector_len) {
6524   assert(VM_Version::supports_evex(), "");
6525   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
6526   attributes.set_is_evex_instruction();
6527   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6528   emit_int8(0x7A);
6529   emit_int8((unsigned char)(0xC0 | encode));
6530 }
6531 
6532 // duplicate 2-byte integer data from src into programmed locations in dest : requires AVX512BW and AVX512VL
6533 void Assembler::evpbroadcastw(XMMRegister dst, Register src, int vector_len) {
6534   assert(VM_Version::supports_evex(), "");
6535   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
6536   attributes.set_is_evex_instruction();
6537   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6538   emit_int8(0x7B);
6539   emit_int8((unsigned char)(0xC0 | encode));
6540 }
6541 
6542 // duplicate 4-byte integer data from src into programmed locations in dest : requires AVX512VL
6543 void Assembler::evpbroadcastd(XMMRegister dst, Register src, int vector_len) {
6544   assert(VM_Version::supports_evex(), "");
6545   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6546   attributes.set_is_evex_instruction();
6547   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6548   emit_int8(0x7C);
6549   emit_int8((unsigned char)(0xC0 | encode));
6550 }
6551 
6552 // duplicate 8-byte integer data from src into programmed locations in dest : requires AVX512VL
6553 void Assembler::evpbroadcastq(XMMRegister dst, Register src, int vector_len) {
6554   assert(VM_Version::supports_evex(), "");
6555   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6556   attributes.set_is_evex_instruction();
6557   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6558   emit_int8(0x7C);
6559   emit_int8((unsigned char)(0xC0 | encode));
6560 }
6561 
6562 
6563 // Carry-Less Multiplication Quadword
6564 void Assembler::pclmulqdq(XMMRegister dst, XMMRegister src, int mask) {
6565   assert(VM_Version::supports_clmul(), "");
6566   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
6567   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6568   emit_int8(0x44);
6569   emit_int8((unsigned char)(0xC0 | encode));
6570   emit_int8((unsigned char)mask);
6571 }
6572 
6573 // Carry-Less Multiplication Quadword
6574 void Assembler::vpclmulqdq(XMMRegister dst, XMMRegister nds, XMMRegister src, int mask) {
6575   assert(VM_Version::supports_avx() && VM_Version::supports_clmul(), "");
6576   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
6577   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6578   emit_int8(0x44);
6579   emit_int8((unsigned char)(0xC0 | encode));
6580   emit_int8((unsigned char)mask);
6581 }
6582 
6583 void Assembler::vzeroupper() {
6584   assert(VM_Version::supports_avx(), "");
6585   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
6586   (void)vex_prefix_and_encode(0, 0, 0, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
6587   emit_int8(0x77);
6588 }
6589 
6590 #ifndef _LP64
6591 // 32bit only pieces of the assembler
6592 
6593 void Assembler::cmp_literal32(Register src1, int32_t imm32, RelocationHolder const& rspec) {
6594   // NO PREFIX AS NEVER 64BIT
6595   InstructionMark im(this);
6596   emit_int8((unsigned char)0x81);
6597   emit_int8((unsigned char)(0xF8 | src1->encoding()));
6598   emit_data(imm32, rspec, 0);
6599 }
6600 
6601 void Assembler::cmp_literal32(Address src1, int32_t imm32, RelocationHolder const& rspec) {
6602   // NO PREFIX AS NEVER 64BIT (not even 32bit versions of 64bit regs
6603   InstructionMark im(this);
6604   emit_int8((unsigned char)0x81);
6605   emit_operand(rdi, src1);
6606   emit_data(imm32, rspec, 0);
6607 }
6608 
6609 // The 64-bit (32bit platform) cmpxchg compares the value at adr with the contents of rdx:rax,
6610 // and stores rcx:rbx into adr if so; otherwise, the value at adr is loaded
6611 // into rdx:rax.  The ZF is set if the compared values were equal, and cleared otherwise.
6612 void Assembler::cmpxchg8(Address adr) {
6613   InstructionMark im(this);
6614   emit_int8(0x0F);
6615   emit_int8((unsigned char)0xC7);
6616   emit_operand(rcx, adr);
6617 }
6618 
6619 void Assembler::decl(Register dst) {
6620   // Don't use it directly. Use MacroAssembler::decrementl() instead.
6621  emit_int8(0x48 | dst->encoding());
6622 }
6623 
6624 #endif // _LP64
6625 
6626 // 64bit typically doesn't use the x87 but needs to for the trig funcs
6627 
6628 void Assembler::fabs() {
6629   emit_int8((unsigned char)0xD9);
6630   emit_int8((unsigned char)0xE1);
6631 }
6632 
6633 void Assembler::fadd(int i) {
6634   emit_farith(0xD8, 0xC0, i);
6635 }
6636 
6637 void Assembler::fadd_d(Address src) {
6638   InstructionMark im(this);
6639   emit_int8((unsigned char)0xDC);
6640   emit_operand32(rax, src);
6641 }
6642 
6643 void Assembler::fadd_s(Address src) {
6644   InstructionMark im(this);
6645   emit_int8((unsigned char)0xD8);
6646   emit_operand32(rax, src);
6647 }
6648 
6649 void Assembler::fadda(int i) {
6650   emit_farith(0xDC, 0xC0, i);
6651 }
6652 
6653 void Assembler::faddp(int i) {
6654   emit_farith(0xDE, 0xC0, i);
6655 }
6656 
6657 void Assembler::fchs() {
6658   emit_int8((unsigned char)0xD9);
6659   emit_int8((unsigned char)0xE0);
6660 }
6661 
6662 void Assembler::fcom(int i) {
6663   emit_farith(0xD8, 0xD0, i);
6664 }
6665 
6666 void Assembler::fcomp(int i) {
6667   emit_farith(0xD8, 0xD8, i);
6668 }
6669 
6670 void Assembler::fcomp_d(Address src) {
6671   InstructionMark im(this);
6672   emit_int8((unsigned char)0xDC);
6673   emit_operand32(rbx, src);
6674 }
6675 
6676 void Assembler::fcomp_s(Address src) {
6677   InstructionMark im(this);
6678   emit_int8((unsigned char)0xD8);
6679   emit_operand32(rbx, src);
6680 }
6681 
6682 void Assembler::fcompp() {
6683   emit_int8((unsigned char)0xDE);
6684   emit_int8((unsigned char)0xD9);
6685 }
6686 
6687 void Assembler::fcos() {
6688   emit_int8((unsigned char)0xD9);
6689   emit_int8((unsigned char)0xFF);
6690 }
6691 
6692 void Assembler::fdecstp() {
6693   emit_int8((unsigned char)0xD9);
6694   emit_int8((unsigned char)0xF6);
6695 }
6696 
6697 void Assembler::fdiv(int i) {
6698   emit_farith(0xD8, 0xF0, i);
6699 }
6700 
6701 void Assembler::fdiv_d(Address src) {
6702   InstructionMark im(this);
6703   emit_int8((unsigned char)0xDC);
6704   emit_operand32(rsi, src);
6705 }
6706 
6707 void Assembler::fdiv_s(Address src) {
6708   InstructionMark im(this);
6709   emit_int8((unsigned char)0xD8);
6710   emit_operand32(rsi, src);
6711 }
6712 
6713 void Assembler::fdiva(int i) {
6714   emit_farith(0xDC, 0xF8, i);
6715 }
6716 
6717 // Note: The Intel manual (Pentium Processor User's Manual, Vol.3, 1994)
6718 //       is erroneous for some of the floating-point instructions below.
6719 
6720 void Assembler::fdivp(int i) {
6721   emit_farith(0xDE, 0xF8, i);                    // ST(0) <- ST(0) / ST(1) and pop (Intel manual wrong)
6722 }
6723 
6724 void Assembler::fdivr(int i) {
6725   emit_farith(0xD8, 0xF8, i);
6726 }
6727 
6728 void Assembler::fdivr_d(Address src) {
6729   InstructionMark im(this);
6730   emit_int8((unsigned char)0xDC);
6731   emit_operand32(rdi, src);
6732 }
6733 
6734 void Assembler::fdivr_s(Address src) {
6735   InstructionMark im(this);
6736   emit_int8((unsigned char)0xD8);
6737   emit_operand32(rdi, src);
6738 }
6739 
6740 void Assembler::fdivra(int i) {
6741   emit_farith(0xDC, 0xF0, i);
6742 }
6743 
6744 void Assembler::fdivrp(int i) {
6745   emit_farith(0xDE, 0xF0, i);                    // ST(0) <- ST(1) / ST(0) and pop (Intel manual wrong)
6746 }
6747 
6748 void Assembler::ffree(int i) {
6749   emit_farith(0xDD, 0xC0, i);
6750 }
6751 
6752 void Assembler::fild_d(Address adr) {
6753   InstructionMark im(this);
6754   emit_int8((unsigned char)0xDF);
6755   emit_operand32(rbp, adr);
6756 }
6757 
6758 void Assembler::fild_s(Address adr) {
6759   InstructionMark im(this);
6760   emit_int8((unsigned char)0xDB);
6761   emit_operand32(rax, adr);
6762 }
6763 
6764 void Assembler::fincstp() {
6765   emit_int8((unsigned char)0xD9);
6766   emit_int8((unsigned char)0xF7);
6767 }
6768 
6769 void Assembler::finit() {
6770   emit_int8((unsigned char)0x9B);
6771   emit_int8((unsigned char)0xDB);
6772   emit_int8((unsigned char)0xE3);
6773 }
6774 
6775 void Assembler::fist_s(Address adr) {
6776   InstructionMark im(this);
6777   emit_int8((unsigned char)0xDB);
6778   emit_operand32(rdx, adr);
6779 }
6780 
6781 void Assembler::fistp_d(Address adr) {
6782   InstructionMark im(this);
6783   emit_int8((unsigned char)0xDF);
6784   emit_operand32(rdi, adr);
6785 }
6786 
6787 void Assembler::fistp_s(Address adr) {
6788   InstructionMark im(this);
6789   emit_int8((unsigned char)0xDB);
6790   emit_operand32(rbx, adr);
6791 }
6792 
6793 void Assembler::fld1() {
6794   emit_int8((unsigned char)0xD9);
6795   emit_int8((unsigned char)0xE8);
6796 }
6797 
6798 void Assembler::fld_d(Address adr) {
6799   InstructionMark im(this);
6800   emit_int8((unsigned char)0xDD);
6801   emit_operand32(rax, adr);
6802 }
6803 
6804 void Assembler::fld_s(Address adr) {
6805   InstructionMark im(this);
6806   emit_int8((unsigned char)0xD9);
6807   emit_operand32(rax, adr);
6808 }
6809 
6810 
6811 void Assembler::fld_s(int index) {
6812   emit_farith(0xD9, 0xC0, index);
6813 }
6814 
6815 void Assembler::fld_x(Address adr) {
6816   InstructionMark im(this);
6817   emit_int8((unsigned char)0xDB);
6818   emit_operand32(rbp, adr);
6819 }
6820 
6821 void Assembler::fldcw(Address src) {
6822   InstructionMark im(this);
6823   emit_int8((unsigned char)0xD9);
6824   emit_operand32(rbp, src);
6825 }
6826 
6827 void Assembler::fldenv(Address src) {
6828   InstructionMark im(this);
6829   emit_int8((unsigned char)0xD9);
6830   emit_operand32(rsp, src);
6831 }
6832 
6833 void Assembler::fldlg2() {
6834   emit_int8((unsigned char)0xD9);
6835   emit_int8((unsigned char)0xEC);
6836 }
6837 
6838 void Assembler::fldln2() {
6839   emit_int8((unsigned char)0xD9);
6840   emit_int8((unsigned char)0xED);
6841 }
6842 
6843 void Assembler::fldz() {
6844   emit_int8((unsigned char)0xD9);
6845   emit_int8((unsigned char)0xEE);
6846 }
6847 
6848 void Assembler::flog() {
6849   fldln2();
6850   fxch();
6851   fyl2x();
6852 }
6853 
6854 void Assembler::flog10() {
6855   fldlg2();
6856   fxch();
6857   fyl2x();
6858 }
6859 
6860 void Assembler::fmul(int i) {
6861   emit_farith(0xD8, 0xC8, i);
6862 }
6863 
6864 void Assembler::fmul_d(Address src) {
6865   InstructionMark im(this);
6866   emit_int8((unsigned char)0xDC);
6867   emit_operand32(rcx, src);
6868 }
6869 
6870 void Assembler::fmul_s(Address src) {
6871   InstructionMark im(this);
6872   emit_int8((unsigned char)0xD8);
6873   emit_operand32(rcx, src);
6874 }
6875 
6876 void Assembler::fmula(int i) {
6877   emit_farith(0xDC, 0xC8, i);
6878 }
6879 
6880 void Assembler::fmulp(int i) {
6881   emit_farith(0xDE, 0xC8, i);
6882 }
6883 
6884 void Assembler::fnsave(Address dst) {
6885   InstructionMark im(this);
6886   emit_int8((unsigned char)0xDD);
6887   emit_operand32(rsi, dst);
6888 }
6889 
6890 void Assembler::fnstcw(Address src) {
6891   InstructionMark im(this);
6892   emit_int8((unsigned char)0x9B);
6893   emit_int8((unsigned char)0xD9);
6894   emit_operand32(rdi, src);
6895 }
6896 
6897 void Assembler::fnstsw_ax() {
6898   emit_int8((unsigned char)0xDF);
6899   emit_int8((unsigned char)0xE0);
6900 }
6901 
6902 void Assembler::fprem() {
6903   emit_int8((unsigned char)0xD9);
6904   emit_int8((unsigned char)0xF8);
6905 }
6906 
6907 void Assembler::fprem1() {
6908   emit_int8((unsigned char)0xD9);
6909   emit_int8((unsigned char)0xF5);
6910 }
6911 
6912 void Assembler::frstor(Address src) {
6913   InstructionMark im(this);
6914   emit_int8((unsigned char)0xDD);
6915   emit_operand32(rsp, src);
6916 }
6917 
6918 void Assembler::fsin() {
6919   emit_int8((unsigned char)0xD9);
6920   emit_int8((unsigned char)0xFE);
6921 }
6922 
6923 void Assembler::fsqrt() {
6924   emit_int8((unsigned char)0xD9);
6925   emit_int8((unsigned char)0xFA);
6926 }
6927 
6928 void Assembler::fst_d(Address adr) {
6929   InstructionMark im(this);
6930   emit_int8((unsigned char)0xDD);
6931   emit_operand32(rdx, adr);
6932 }
6933 
6934 void Assembler::fst_s(Address adr) {
6935   InstructionMark im(this);
6936   emit_int8((unsigned char)0xD9);
6937   emit_operand32(rdx, adr);
6938 }
6939 
6940 void Assembler::fstp_d(Address adr) {
6941   InstructionMark im(this);
6942   emit_int8((unsigned char)0xDD);
6943   emit_operand32(rbx, adr);
6944 }
6945 
6946 void Assembler::fstp_d(int index) {
6947   emit_farith(0xDD, 0xD8, index);
6948 }
6949 
6950 void Assembler::fstp_s(Address adr) {
6951   InstructionMark im(this);
6952   emit_int8((unsigned char)0xD9);
6953   emit_operand32(rbx, adr);
6954 }
6955 
6956 void Assembler::fstp_x(Address adr) {
6957   InstructionMark im(this);
6958   emit_int8((unsigned char)0xDB);
6959   emit_operand32(rdi, adr);
6960 }
6961 
6962 void Assembler::fsub(int i) {
6963   emit_farith(0xD8, 0xE0, i);
6964 }
6965 
6966 void Assembler::fsub_d(Address src) {
6967   InstructionMark im(this);
6968   emit_int8((unsigned char)0xDC);
6969   emit_operand32(rsp, src);
6970 }
6971 
6972 void Assembler::fsub_s(Address src) {
6973   InstructionMark im(this);
6974   emit_int8((unsigned char)0xD8);
6975   emit_operand32(rsp, src);
6976 }
6977 
6978 void Assembler::fsuba(int i) {
6979   emit_farith(0xDC, 0xE8, i);
6980 }
6981 
6982 void Assembler::fsubp(int i) {
6983   emit_farith(0xDE, 0xE8, i);                    // ST(0) <- ST(0) - ST(1) and pop (Intel manual wrong)
6984 }
6985 
6986 void Assembler::fsubr(int i) {
6987   emit_farith(0xD8, 0xE8, i);
6988 }
6989 
6990 void Assembler::fsubr_d(Address src) {
6991   InstructionMark im(this);
6992   emit_int8((unsigned char)0xDC);
6993   emit_operand32(rbp, src);
6994 }
6995 
6996 void Assembler::fsubr_s(Address src) {
6997   InstructionMark im(this);
6998   emit_int8((unsigned char)0xD8);
6999   emit_operand32(rbp, src);
7000 }
7001 
7002 void Assembler::fsubra(int i) {
7003   emit_farith(0xDC, 0xE0, i);
7004 }
7005 
7006 void Assembler::fsubrp(int i) {
7007   emit_farith(0xDE, 0xE0, i);                    // ST(0) <- ST(1) - ST(0) and pop (Intel manual wrong)
7008 }
7009 
7010 void Assembler::ftan() {
7011   emit_int8((unsigned char)0xD9);
7012   emit_int8((unsigned char)0xF2);
7013   emit_int8((unsigned char)0xDD);
7014   emit_int8((unsigned char)0xD8);
7015 }
7016 
7017 void Assembler::ftst() {
7018   emit_int8((unsigned char)0xD9);
7019   emit_int8((unsigned char)0xE4);
7020 }
7021 
7022 void Assembler::fucomi(int i) {
7023   // make sure the instruction is supported (introduced for P6, together with cmov)
7024   guarantee(VM_Version::supports_cmov(), "illegal instruction");
7025   emit_farith(0xDB, 0xE8, i);
7026 }
7027 
7028 void Assembler::fucomip(int i) {
7029   // make sure the instruction is supported (introduced for P6, together with cmov)
7030   guarantee(VM_Version::supports_cmov(), "illegal instruction");
7031   emit_farith(0xDF, 0xE8, i);
7032 }
7033 
7034 void Assembler::fwait() {
7035   emit_int8((unsigned char)0x9B);
7036 }
7037 
7038 void Assembler::fxch(int i) {
7039   emit_farith(0xD9, 0xC8, i);
7040 }
7041 
7042 void Assembler::fyl2x() {
7043   emit_int8((unsigned char)0xD9);
7044   emit_int8((unsigned char)0xF1);
7045 }
7046 
7047 void Assembler::frndint() {
7048   emit_int8((unsigned char)0xD9);
7049   emit_int8((unsigned char)0xFC);
7050 }
7051 
7052 void Assembler::f2xm1() {
7053   emit_int8((unsigned char)0xD9);
7054   emit_int8((unsigned char)0xF0);
7055 }
7056 
7057 void Assembler::fldl2e() {
7058   emit_int8((unsigned char)0xD9);
7059   emit_int8((unsigned char)0xEA);
7060 }
7061 
7062 // SSE SIMD prefix byte values corresponding to VexSimdPrefix encoding.
7063 static int simd_pre[4] = { 0, 0x66, 0xF3, 0xF2 };
7064 // SSE opcode second byte values (first is 0x0F) corresponding to VexOpcode encoding.
7065 static int simd_opc[4] = { 0,    0, 0x38, 0x3A };
7066 
7067 // Generate SSE legacy REX prefix and SIMD opcode based on VEX encoding.
7068 void Assembler::rex_prefix(Address adr, XMMRegister xreg, VexSimdPrefix pre, VexOpcode opc, bool rex_w) {
7069   if (pre > 0) {
7070     emit_int8(simd_pre[pre]);
7071   }
7072   if (rex_w) {
7073     prefixq(adr, xreg);
7074   } else {
7075     prefix(adr, xreg);
7076   }
7077   if (opc > 0) {
7078     emit_int8(0x0F);
7079     int opc2 = simd_opc[opc];
7080     if (opc2 > 0) {
7081       emit_int8(opc2);
7082     }
7083   }
7084 }
7085 
7086 int Assembler::rex_prefix_and_encode(int dst_enc, int src_enc, VexSimdPrefix pre, VexOpcode opc, bool rex_w) {
7087   if (pre > 0) {
7088     emit_int8(simd_pre[pre]);
7089   }
7090   int encode = (rex_w) ? prefixq_and_encode(dst_enc, src_enc) : prefix_and_encode(dst_enc, src_enc);
7091   if (opc > 0) {
7092     emit_int8(0x0F);
7093     int opc2 = simd_opc[opc];
7094     if (opc2 > 0) {
7095       emit_int8(opc2);
7096     }
7097   }
7098   return encode;
7099 }
7100 
7101 
7102 void Assembler::vex_prefix(bool vex_r, bool vex_b, bool vex_x, int nds_enc, VexSimdPrefix pre, VexOpcode opc) {
7103   int vector_len = _attributes->get_vector_len();
7104   bool vex_w = _attributes->is_rex_vex_w();
7105   if (vex_b || vex_x || vex_w || (opc == VEX_OPCODE_0F_38) || (opc == VEX_OPCODE_0F_3A)) {
7106     prefix(VEX_3bytes);
7107 
7108     int byte1 = (vex_r ? VEX_R : 0) | (vex_x ? VEX_X : 0) | (vex_b ? VEX_B : 0);
7109     byte1 = (~byte1) & 0xE0;
7110     byte1 |= opc;
7111     emit_int8(byte1);
7112 
7113     int byte2 = ((~nds_enc) & 0xf) << 3;
7114     byte2 |= (vex_w ? VEX_W : 0) | ((vector_len > 0) ? 4 : 0) | pre;
7115     emit_int8(byte2);
7116   } else {
7117     prefix(VEX_2bytes);
7118 
7119     int byte1 = vex_r ? VEX_R : 0;
7120     byte1 = (~byte1) & 0x80;
7121     byte1 |= ((~nds_enc) & 0xf) << 3;
7122     byte1 |= ((vector_len > 0 ) ? 4 : 0) | pre;
7123     emit_int8(byte1);
7124   }
7125 }
7126 
7127 // This is a 4 byte encoding
7128 void Assembler::evex_prefix(bool vex_r, bool vex_b, bool vex_x, bool evex_r, bool evex_v, int nds_enc, VexSimdPrefix pre, VexOpcode opc){
7129   // EVEX 0x62 prefix
7130   prefix(EVEX_4bytes);
7131   bool vex_w = _attributes->is_rex_vex_w();
7132   int evex_encoding = (vex_w ? VEX_W : 0);
7133   // EVEX.b is not currently used for broadcast of single element or data rounding modes
7134   _attributes->set_evex_encoding(evex_encoding);
7135 
7136   // P0: byte 2, initialized to RXBR`00mm
7137   // instead of not'd
7138   int byte2 = (vex_r ? VEX_R : 0) | (vex_x ? VEX_X : 0) | (vex_b ? VEX_B : 0) | (evex_r ? EVEX_Rb : 0);
7139   byte2 = (~byte2) & 0xF0;
7140   // confine opc opcode extensions in mm bits to lower two bits
7141   // of form {0F, 0F_38, 0F_3A}
7142   byte2 |= opc;
7143   emit_int8(byte2);
7144 
7145   // P1: byte 3 as Wvvvv1pp
7146   int byte3 = ((~nds_enc) & 0xf) << 3;
7147   // p[10] is always 1
7148   byte3 |= EVEX_F;
7149   byte3 |= (vex_w & 1) << 7;
7150   // confine pre opcode extensions in pp bits to lower two bits
7151   // of form {66, F3, F2}
7152   byte3 |= pre;
7153   emit_int8(byte3);
7154 
7155   // P2: byte 4 as zL'Lbv'aaa
7156   // kregs are implemented in the low 3 bits as aaa (hard code k1, it will be initialized for now)
7157   int byte4 = (_attributes->is_no_reg_mask()) ?
7158               0 :
7159               _attributes->get_embedded_opmask_register_specifier();
7160   // EVEX.v` for extending EVEX.vvvv or VIDX
7161   byte4 |= (evex_v ? 0: EVEX_V);
7162   // third EXEC.b for broadcast actions
7163   byte4 |= (_attributes->is_extended_context() ? EVEX_Rb : 0);
7164   // fourth EVEX.L'L for vector length : 0 is 128, 1 is 256, 2 is 512, currently we do not support 1024
7165   byte4 |= ((_attributes->get_vector_len())& 0x3) << 5;
7166   // last is EVEX.z for zero/merge actions
7167   byte4 |= (_attributes->is_clear_context() ? EVEX_Z : 0);
7168   emit_int8(byte4);
7169 }
7170 
7171 void Assembler::vex_prefix(Address adr, int nds_enc, int xreg_enc, VexSimdPrefix pre, VexOpcode opc, InstructionAttr *attributes) {
7172   bool vex_r = ((xreg_enc & 8) == 8) ? 1 : 0;
7173   bool vex_b = adr.base_needs_rex();
7174   bool vex_x = adr.index_needs_rex();
7175   set_attributes(attributes);
7176   attributes->set_current_assembler(this);
7177 
7178   // if vector length is turned off, revert to AVX for vectors smaller than 512-bit
7179   if (UseAVX > 2 && _legacy_mode_vl && attributes->uses_vl()) {
7180     switch (attributes->get_vector_len()) {
7181     case AVX_128bit:
7182     case AVX_256bit:
7183       attributes->set_is_legacy_mode();
7184       break;
7185     }
7186   }
7187 
7188   // For pure EVEX check and see if this instruction
7189   // is allowed in legacy mode and has resources which will
7190   // fit in it.  Pure EVEX instructions will use set_is_evex_instruction in their definition,
7191   // else that field is set when we encode to EVEX
7192   if (UseAVX > 2 && !attributes->is_legacy_mode() &&
7193       !_is_managed && !attributes->is_evex_instruction()) {
7194     if (!_legacy_mode_vl && attributes->get_vector_len() != AVX_512bit) {
7195       bool check_register_bank = NOT_IA32(true) IA32_ONLY(false);
7196       if (check_register_bank) {
7197         // check nds_enc and xreg_enc for upper bank usage
7198         if (nds_enc < 16 && xreg_enc < 16) {
7199           attributes->set_is_legacy_mode();
7200         }
7201       } else {
7202         attributes->set_is_legacy_mode();
7203       }
7204     }
7205   }
7206 
7207   _is_managed = false;
7208   if (UseAVX > 2 && !attributes->is_legacy_mode())
7209   {
7210     bool evex_r = (xreg_enc >= 16);
7211     bool evex_v = (nds_enc >= 16);
7212     attributes->set_is_evex_instruction();
7213     evex_prefix(vex_r, vex_b, vex_x, evex_r, evex_v, nds_enc, pre, opc);
7214   } else {
7215     if (UseAVX > 2 && attributes->is_rex_vex_w_reverted()) {
7216       attributes->set_rex_vex_w(false);
7217     }
7218     vex_prefix(vex_r, vex_b, vex_x, nds_enc, pre, opc);
7219   }
7220 }
7221 
7222 int Assembler::vex_prefix_and_encode(int dst_enc, int nds_enc, int src_enc, VexSimdPrefix pre, VexOpcode opc, InstructionAttr *attributes) {
7223   bool vex_r = ((dst_enc & 8) == 8) ? 1 : 0;
7224   bool vex_b = ((src_enc & 8) == 8) ? 1 : 0;
7225   bool vex_x = false;
7226   set_attributes(attributes);
7227   attributes->set_current_assembler(this);
7228   bool check_register_bank = NOT_IA32(true) IA32_ONLY(false);
7229 
7230   // if vector length is turned off, revert to AVX for vectors smaller than 512-bit
7231   if (UseAVX > 2 && _legacy_mode_vl && attributes->uses_vl()) {
7232     switch (attributes->get_vector_len()) {
7233     case AVX_128bit:
7234     case AVX_256bit:
7235       if (check_register_bank) {
7236         if (dst_enc >= 16 || nds_enc >= 16 || src_enc >= 16) {
7237           // up propagate arithmetic instructions to meet RA requirements
7238           attributes->set_vector_len(AVX_512bit);
7239         } else {
7240           attributes->set_is_legacy_mode();
7241         }
7242       } else {
7243         attributes->set_is_legacy_mode();
7244       }
7245       break;
7246     }
7247   }
7248 
7249   // For pure EVEX check and see if this instruction
7250   // is allowed in legacy mode and has resources which will
7251   // fit in it.  Pure EVEX instructions will use set_is_evex_instruction in their definition,
7252   // else that field is set when we encode to EVEX
7253   if (UseAVX > 2 && !attributes->is_legacy_mode() &&
7254       !_is_managed && !attributes->is_evex_instruction()) {
7255     if (!_legacy_mode_vl && attributes->get_vector_len() != AVX_512bit) {
7256       if (check_register_bank) {
7257         // check dst_enc, nds_enc and src_enc for upper bank usage
7258         if (dst_enc < 16 && nds_enc < 16 && src_enc < 16) {
7259           attributes->set_is_legacy_mode();
7260         }
7261       } else {
7262         attributes->set_is_legacy_mode();
7263       }
7264     }
7265   }
7266 
7267   _is_managed = false;
7268   if (UseAVX > 2 && !attributes->is_legacy_mode())
7269   {
7270     bool evex_r = (dst_enc >= 16);
7271     bool evex_v = (nds_enc >= 16);
7272     // can use vex_x as bank extender on rm encoding
7273     vex_x = (src_enc >= 16);
7274     attributes->set_is_evex_instruction();
7275     evex_prefix(vex_r, vex_b, vex_x, evex_r, evex_v, nds_enc, pre, opc);
7276   } else {
7277     if (UseAVX > 2 && attributes->is_rex_vex_w_reverted()) {
7278       attributes->set_rex_vex_w(false);
7279     }
7280     vex_prefix(vex_r, vex_b, vex_x, nds_enc, pre, opc);
7281   }
7282 
7283   // return modrm byte components for operands
7284   return (((dst_enc & 7) << 3) | (src_enc & 7));
7285 }
7286 
7287 
7288 void Assembler::simd_prefix(XMMRegister xreg, XMMRegister nds, Address adr, VexSimdPrefix pre,
7289                             VexOpcode opc, InstructionAttr *attributes) {
7290   if (UseAVX > 0) {
7291     int xreg_enc = xreg->encoding();
7292     int nds_enc = nds->is_valid() ? nds->encoding() : 0;
7293     vex_prefix(adr, nds_enc, xreg_enc, pre, opc, attributes);
7294   } else {
7295     assert((nds == xreg) || (nds == xnoreg), "wrong sse encoding");
7296     rex_prefix(adr, xreg, pre, opc, attributes->is_rex_vex_w());
7297   }
7298 }
7299 
7300 int Assembler::simd_prefix_and_encode(XMMRegister dst, XMMRegister nds, XMMRegister src, VexSimdPrefix pre,
7301                                       VexOpcode opc, InstructionAttr *attributes) {
7302   int dst_enc = dst->encoding();
7303   int src_enc = src->encoding();
7304   if (UseAVX > 0) {
7305     int nds_enc = nds->is_valid() ? nds->encoding() : 0;
7306     return vex_prefix_and_encode(dst_enc, nds_enc, src_enc, pre, opc, attributes);
7307   } else {
7308     assert((nds == dst) || (nds == src) || (nds == xnoreg), "wrong sse encoding");
7309     return rex_prefix_and_encode(dst_enc, src_enc, pre, opc, attributes->is_rex_vex_w());
7310   }
7311 }
7312 
7313 void Assembler::cmppd(XMMRegister dst, XMMRegister nds, XMMRegister src, int cop, int vector_len) {
7314   assert(VM_Version::supports_avx(), "");
7315   assert(!VM_Version::supports_evex(), "");
7316   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
7317   int encode = simd_prefix_and_encode(dst, nds, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7318   emit_int8((unsigned char)0xC2);
7319   emit_int8((unsigned char)(0xC0 | encode));
7320   emit_int8((unsigned char)(0xF & cop));
7321 }
7322 
7323 void Assembler::vpblendd(XMMRegister dst, XMMRegister nds, XMMRegister src1, XMMRegister src2, int vector_len) {
7324   assert(VM_Version::supports_avx(), "");
7325   assert(!VM_Version::supports_evex(), "");
7326   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
7327   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src1->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
7328   emit_int8((unsigned char)0x4B);
7329   emit_int8((unsigned char)(0xC0 | encode));
7330   int src2_enc = src2->encoding();
7331   emit_int8((unsigned char)(0xF0 & src2_enc<<4));
7332 }
7333 
7334 void Assembler::shlxl(Register dst, Register src1, Register src2) {
7335   assert(VM_Version::supports_bmi2(), "");
7336   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
7337   int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src1->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
7338   emit_int8((unsigned char)0xF7);
7339   emit_int8((unsigned char)(0xC0 | encode));
7340 }
7341 
7342 void Assembler::shlxq(Register dst, Register src1, Register src2) {
7343   assert(VM_Version::supports_bmi2(), "");
7344   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
7345   int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src1->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
7346   emit_int8((unsigned char)0xF7);
7347   emit_int8((unsigned char)(0xC0 | encode));
7348 }
7349 
7350 #ifndef _LP64
7351 
7352 void Assembler::incl(Register dst) {
7353   // Don't use it directly. Use MacroAssembler::incrementl() instead.
7354   emit_int8(0x40 | dst->encoding());
7355 }
7356 
7357 void Assembler::lea(Register dst, Address src) {
7358   leal(dst, src);
7359 }
7360 
7361 void Assembler::mov_literal32(Address dst, int32_t imm32, RelocationHolder const& rspec) {
7362   InstructionMark im(this);
7363   emit_int8((unsigned char)0xC7);
7364   emit_operand(rax, dst);
7365   emit_data((int)imm32, rspec, 0);
7366 }
7367 
7368 void Assembler::mov_literal32(Register dst, int32_t imm32, RelocationHolder const& rspec) {
7369   InstructionMark im(this);
7370   int encode = prefix_and_encode(dst->encoding());
7371   emit_int8((unsigned char)(0xB8 | encode));
7372   emit_data((int)imm32, rspec, 0);
7373 }
7374 
7375 void Assembler::popa() { // 32bit
7376   emit_int8(0x61);
7377 }
7378 
7379 void Assembler::push_literal32(int32_t imm32, RelocationHolder const& rspec) {
7380   InstructionMark im(this);
7381   emit_int8(0x68);
7382   emit_data(imm32, rspec, 0);
7383 }
7384 
7385 void Assembler::pusha() { // 32bit
7386   emit_int8(0x60);
7387 }
7388 
7389 void Assembler::set_byte_if_not_zero(Register dst) {
7390   emit_int8(0x0F);
7391   emit_int8((unsigned char)0x95);
7392   emit_int8((unsigned char)(0xE0 | dst->encoding()));
7393 }
7394 
7395 void Assembler::shldl(Register dst, Register src) {
7396   emit_int8(0x0F);
7397   emit_int8((unsigned char)0xA5);
7398   emit_int8((unsigned char)(0xC0 | src->encoding() << 3 | dst->encoding()));
7399 }
7400 
7401 // 0F A4 / r ib
7402 void Assembler::shldl(Register dst, Register src, int8_t imm8) {
7403   emit_int8(0x0F);
7404   emit_int8((unsigned char)0xA4);
7405   emit_int8((unsigned char)(0xC0 | src->encoding() << 3 | dst->encoding()));
7406   emit_int8(imm8);
7407 }
7408 
7409 void Assembler::shrdl(Register dst, Register src) {
7410   emit_int8(0x0F);
7411   emit_int8((unsigned char)0xAD);
7412   emit_int8((unsigned char)(0xC0 | src->encoding() << 3 | dst->encoding()));
7413 }
7414 
7415 #else // LP64
7416 
7417 void Assembler::set_byte_if_not_zero(Register dst) {
7418   int enc = prefix_and_encode(dst->encoding(), true);
7419   emit_int8(0x0F);
7420   emit_int8((unsigned char)0x95);
7421   emit_int8((unsigned char)(0xE0 | enc));
7422 }
7423 
7424 // 64bit only pieces of the assembler
7425 // This should only be used by 64bit instructions that can use rip-relative
7426 // it cannot be used by instructions that want an immediate value.
7427 
7428 bool Assembler::reachable(AddressLiteral adr) {
7429   int64_t disp;
7430   // None will force a 64bit literal to the code stream. Likely a placeholder
7431   // for something that will be patched later and we need to certain it will
7432   // always be reachable.
7433   if (adr.reloc() == relocInfo::none) {
7434     return false;
7435   }
7436   if (adr.reloc() == relocInfo::internal_word_type) {
7437     // This should be rip relative and easily reachable.
7438     return true;
7439   }
7440   if (adr.reloc() == relocInfo::virtual_call_type ||
7441       adr.reloc() == relocInfo::opt_virtual_call_type ||
7442       adr.reloc() == relocInfo::static_call_type ||
7443       adr.reloc() == relocInfo::static_stub_type ) {
7444     // This should be rip relative within the code cache and easily
7445     // reachable until we get huge code caches. (At which point
7446     // ic code is going to have issues).
7447     return true;
7448   }
7449   if (adr.reloc() != relocInfo::external_word_type &&
7450       adr.reloc() != relocInfo::poll_return_type &&  // these are really external_word but need special
7451       adr.reloc() != relocInfo::poll_type &&         // relocs to identify them
7452       adr.reloc() != relocInfo::runtime_call_type ) {
7453     return false;
7454   }
7455 
7456   // Stress the correction code
7457   if (ForceUnreachable) {
7458     // Must be runtimecall reloc, see if it is in the codecache
7459     // Flipping stuff in the codecache to be unreachable causes issues
7460     // with things like inline caches where the additional instructions
7461     // are not handled.
7462     if (CodeCache::find_blob(adr._target) == NULL) {
7463       return false;
7464     }
7465   }
7466   // For external_word_type/runtime_call_type if it is reachable from where we
7467   // are now (possibly a temp buffer) and where we might end up
7468   // anywhere in the codeCache then we are always reachable.
7469   // This would have to change if we ever save/restore shared code
7470   // to be more pessimistic.
7471   disp = (int64_t)adr._target - ((int64_t)CodeCache::low_bound() + sizeof(int));
7472   if (!is_simm32(disp)) return false;
7473   disp = (int64_t)adr._target - ((int64_t)CodeCache::high_bound() + sizeof(int));
7474   if (!is_simm32(disp)) return false;
7475 
7476   disp = (int64_t)adr._target - ((int64_t)pc() + sizeof(int));
7477 
7478   // Because rip relative is a disp + address_of_next_instruction and we
7479   // don't know the value of address_of_next_instruction we apply a fudge factor
7480   // to make sure we will be ok no matter the size of the instruction we get placed into.
7481   // We don't have to fudge the checks above here because they are already worst case.
7482 
7483   // 12 == override/rex byte, opcode byte, rm byte, sib byte, a 4-byte disp , 4-byte literal
7484   // + 4 because better safe than sorry.
7485   const int fudge = 12 + 4;
7486   if (disp < 0) {
7487     disp -= fudge;
7488   } else {
7489     disp += fudge;
7490   }
7491   return is_simm32(disp);
7492 }
7493 
7494 // Check if the polling page is not reachable from the code cache using rip-relative
7495 // addressing.
7496 bool Assembler::is_polling_page_far() {
7497   intptr_t addr = (intptr_t)os::get_polling_page();
7498   return ForceUnreachable ||
7499          !is_simm32(addr - (intptr_t)CodeCache::low_bound()) ||
7500          !is_simm32(addr - (intptr_t)CodeCache::high_bound());
7501 }
7502 
7503 void Assembler::emit_data64(jlong data,
7504                             relocInfo::relocType rtype,
7505                             int format) {
7506   if (rtype == relocInfo::none) {
7507     emit_int64(data);
7508   } else {
7509     emit_data64(data, Relocation::spec_simple(rtype), format);
7510   }
7511 }
7512 
7513 void Assembler::emit_data64(jlong data,
7514                             RelocationHolder const& rspec,
7515                             int format) {
7516   assert(imm_operand == 0, "default format must be immediate in this file");
7517   assert(imm_operand == format, "must be immediate");
7518   assert(inst_mark() != NULL, "must be inside InstructionMark");
7519   // Do not use AbstractAssembler::relocate, which is not intended for
7520   // embedded words.  Instead, relocate to the enclosing instruction.
7521   code_section()->relocate(inst_mark(), rspec, format);
7522 #ifdef ASSERT
7523   check_relocation(rspec, format);
7524 #endif
7525   emit_int64(data);
7526 }
7527 
7528 int Assembler::prefix_and_encode(int reg_enc, bool byteinst) {
7529   if (reg_enc >= 8) {
7530     prefix(REX_B);
7531     reg_enc -= 8;
7532   } else if (byteinst && reg_enc >= 4) {
7533     prefix(REX);
7534   }
7535   return reg_enc;
7536 }
7537 
7538 int Assembler::prefixq_and_encode(int reg_enc) {
7539   if (reg_enc < 8) {
7540     prefix(REX_W);
7541   } else {
7542     prefix(REX_WB);
7543     reg_enc -= 8;
7544   }
7545   return reg_enc;
7546 }
7547 
7548 int Assembler::prefix_and_encode(int dst_enc, bool dst_is_byte, int src_enc, bool src_is_byte) {
7549   if (dst_enc < 8) {
7550     if (src_enc >= 8) {
7551       prefix(REX_B);
7552       src_enc -= 8;
7553     } else if ((src_is_byte && src_enc >= 4) || (dst_is_byte && dst_enc >= 4)) {
7554       prefix(REX);
7555     }
7556   } else {
7557     if (src_enc < 8) {
7558       prefix(REX_R);
7559     } else {
7560       prefix(REX_RB);
7561       src_enc -= 8;
7562     }
7563     dst_enc -= 8;
7564   }
7565   return dst_enc << 3 | src_enc;
7566 }
7567 
7568 int Assembler::prefixq_and_encode(int dst_enc, int src_enc) {
7569   if (dst_enc < 8) {
7570     if (src_enc < 8) {
7571       prefix(REX_W);
7572     } else {
7573       prefix(REX_WB);
7574       src_enc -= 8;
7575     }
7576   } else {
7577     if (src_enc < 8) {
7578       prefix(REX_WR);
7579     } else {
7580       prefix(REX_WRB);
7581       src_enc -= 8;
7582     }
7583     dst_enc -= 8;
7584   }
7585   return dst_enc << 3 | src_enc;
7586 }
7587 
7588 void Assembler::prefix(Register reg) {
7589   if (reg->encoding() >= 8) {
7590     prefix(REX_B);
7591   }
7592 }
7593 
7594 void Assembler::prefix(Register dst, Register src, Prefix p) {
7595   if (src->encoding() >= 8) {
7596     p = (Prefix)(p | REX_B);
7597   }
7598   if (dst->encoding() >= 8) {
7599     p = (Prefix)( p | REX_R);
7600   }
7601   if (p != Prefix_EMPTY) {
7602     // do not generate an empty prefix
7603     prefix(p);
7604   }
7605 }
7606 
7607 void Assembler::prefix(Register dst, Address adr, Prefix p) {
7608   if (adr.base_needs_rex()) {
7609     if (adr.index_needs_rex()) {
7610       assert(false, "prefix(Register dst, Address adr, Prefix p) does not support handling of an X");
7611     } else {
7612       prefix(REX_B);
7613     }
7614   } else {
7615     if (adr.index_needs_rex()) {
7616       assert(false, "prefix(Register dst, Address adr, Prefix p) does not support handling of an X");
7617     }
7618   }
7619   if (dst->encoding() >= 8) {
7620     p = (Prefix)(p | REX_R);
7621   }
7622   if (p != Prefix_EMPTY) {
7623     // do not generate an empty prefix
7624     prefix(p);
7625   }
7626 }
7627 
7628 void Assembler::prefix(Address adr) {
7629   if (adr.base_needs_rex()) {
7630     if (adr.index_needs_rex()) {
7631       prefix(REX_XB);
7632     } else {
7633       prefix(REX_B);
7634     }
7635   } else {
7636     if (adr.index_needs_rex()) {
7637       prefix(REX_X);
7638     }
7639   }
7640 }
7641 
7642 void Assembler::prefixq(Address adr) {
7643   if (adr.base_needs_rex()) {
7644     if (adr.index_needs_rex()) {
7645       prefix(REX_WXB);
7646     } else {
7647       prefix(REX_WB);
7648     }
7649   } else {
7650     if (adr.index_needs_rex()) {
7651       prefix(REX_WX);
7652     } else {
7653       prefix(REX_W);
7654     }
7655   }
7656 }
7657 
7658 
7659 void Assembler::prefix(Address adr, Register reg, bool byteinst) {
7660   if (reg->encoding() < 8) {
7661     if (adr.base_needs_rex()) {
7662       if (adr.index_needs_rex()) {
7663         prefix(REX_XB);
7664       } else {
7665         prefix(REX_B);
7666       }
7667     } else {
7668       if (adr.index_needs_rex()) {
7669         prefix(REX_X);
7670       } else if (byteinst && reg->encoding() >= 4 ) {
7671         prefix(REX);
7672       }
7673     }
7674   } else {
7675     if (adr.base_needs_rex()) {
7676       if (adr.index_needs_rex()) {
7677         prefix(REX_RXB);
7678       } else {
7679         prefix(REX_RB);
7680       }
7681     } else {
7682       if (adr.index_needs_rex()) {
7683         prefix(REX_RX);
7684       } else {
7685         prefix(REX_R);
7686       }
7687     }
7688   }
7689 }
7690 
7691 void Assembler::prefixq(Address adr, Register src) {
7692   if (src->encoding() < 8) {
7693     if (adr.base_needs_rex()) {
7694       if (adr.index_needs_rex()) {
7695         prefix(REX_WXB);
7696       } else {
7697         prefix(REX_WB);
7698       }
7699     } else {
7700       if (adr.index_needs_rex()) {
7701         prefix(REX_WX);
7702       } else {
7703         prefix(REX_W);
7704       }
7705     }
7706   } else {
7707     if (adr.base_needs_rex()) {
7708       if (adr.index_needs_rex()) {
7709         prefix(REX_WRXB);
7710       } else {
7711         prefix(REX_WRB);
7712       }
7713     } else {
7714       if (adr.index_needs_rex()) {
7715         prefix(REX_WRX);
7716       } else {
7717         prefix(REX_WR);
7718       }
7719     }
7720   }
7721 }
7722 
7723 void Assembler::prefix(Address adr, XMMRegister reg) {
7724   if (reg->encoding() < 8) {
7725     if (adr.base_needs_rex()) {
7726       if (adr.index_needs_rex()) {
7727         prefix(REX_XB);
7728       } else {
7729         prefix(REX_B);
7730       }
7731     } else {
7732       if (adr.index_needs_rex()) {
7733         prefix(REX_X);
7734       }
7735     }
7736   } else {
7737     if (adr.base_needs_rex()) {
7738       if (adr.index_needs_rex()) {
7739         prefix(REX_RXB);
7740       } else {
7741         prefix(REX_RB);
7742       }
7743     } else {
7744       if (adr.index_needs_rex()) {
7745         prefix(REX_RX);
7746       } else {
7747         prefix(REX_R);
7748       }
7749     }
7750   }
7751 }
7752 
7753 void Assembler::prefixq(Address adr, XMMRegister src) {
7754   if (src->encoding() < 8) {
7755     if (adr.base_needs_rex()) {
7756       if (adr.index_needs_rex()) {
7757         prefix(REX_WXB);
7758       } else {
7759         prefix(REX_WB);
7760       }
7761     } else {
7762       if (adr.index_needs_rex()) {
7763         prefix(REX_WX);
7764       } else {
7765         prefix(REX_W);
7766       }
7767     }
7768   } else {
7769     if (adr.base_needs_rex()) {
7770       if (adr.index_needs_rex()) {
7771         prefix(REX_WRXB);
7772       } else {
7773         prefix(REX_WRB);
7774       }
7775     } else {
7776       if (adr.index_needs_rex()) {
7777         prefix(REX_WRX);
7778       } else {
7779         prefix(REX_WR);
7780       }
7781     }
7782   }
7783 }
7784 
7785 void Assembler::adcq(Register dst, int32_t imm32) {
7786   (void) prefixq_and_encode(dst->encoding());
7787   emit_arith(0x81, 0xD0, dst, imm32);
7788 }
7789 
7790 void Assembler::adcq(Register dst, Address src) {
7791   InstructionMark im(this);
7792   prefixq(src, dst);
7793   emit_int8(0x13);
7794   emit_operand(dst, src);
7795 }
7796 
7797 void Assembler::adcq(Register dst, Register src) {
7798   (void) prefixq_and_encode(dst->encoding(), src->encoding());
7799   emit_arith(0x13, 0xC0, dst, src);
7800 }
7801 
7802 void Assembler::addq(Address dst, int32_t imm32) {
7803   InstructionMark im(this);
7804   prefixq(dst);
7805   emit_arith_operand(0x81, rax, dst,imm32);
7806 }
7807 
7808 void Assembler::addq(Address dst, Register src) {
7809   InstructionMark im(this);
7810   prefixq(dst, src);
7811   emit_int8(0x01);
7812   emit_operand(src, dst);
7813 }
7814 
7815 void Assembler::addq(Register dst, int32_t imm32) {
7816   (void) prefixq_and_encode(dst->encoding());
7817   emit_arith(0x81, 0xC0, dst, imm32);
7818 }
7819 
7820 void Assembler::addq(Register dst, Address src) {
7821   InstructionMark im(this);
7822   prefixq(src, dst);
7823   emit_int8(0x03);
7824   emit_operand(dst, src);
7825 }
7826 
7827 void Assembler::addq(Register dst, Register src) {
7828   (void) prefixq_and_encode(dst->encoding(), src->encoding());
7829   emit_arith(0x03, 0xC0, dst, src);
7830 }
7831 
7832 void Assembler::adcxq(Register dst, Register src) {
7833   //assert(VM_Version::supports_adx(), "adx instructions not supported");
7834   emit_int8((unsigned char)0x66);
7835   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
7836   emit_int8(0x0F);
7837   emit_int8(0x38);
7838   emit_int8((unsigned char)0xF6);
7839   emit_int8((unsigned char)(0xC0 | encode));
7840 }
7841 
7842 void Assembler::adoxq(Register dst, Register src) {
7843   //assert(VM_Version::supports_adx(), "adx instructions not supported");
7844   emit_int8((unsigned char)0xF3);
7845   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
7846   emit_int8(0x0F);
7847   emit_int8(0x38);
7848   emit_int8((unsigned char)0xF6);
7849   emit_int8((unsigned char)(0xC0 | encode));
7850 }
7851 
7852 void Assembler::andq(Address dst, int32_t imm32) {
7853   InstructionMark im(this);
7854   prefixq(dst);
7855   emit_int8((unsigned char)0x81);
7856   emit_operand(rsp, dst, 4);
7857   emit_int32(imm32);
7858 }
7859 
7860 void Assembler::andq(Register dst, int32_t imm32) {
7861   (void) prefixq_and_encode(dst->encoding());
7862   emit_arith(0x81, 0xE0, dst, imm32);
7863 }
7864 
7865 void Assembler::andq(Register dst, Address src) {
7866   InstructionMark im(this);
7867   prefixq(src, dst);
7868   emit_int8(0x23);
7869   emit_operand(dst, src);
7870 }
7871 
7872 void Assembler::andq(Register dst, Register src) {
7873   (void) prefixq_and_encode(dst->encoding(), src->encoding());
7874   emit_arith(0x23, 0xC0, dst, src);
7875 }
7876 
7877 void Assembler::andnq(Register dst, Register src1, Register src2) {
7878   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
7879   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
7880   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
7881   emit_int8((unsigned char)0xF2);
7882   emit_int8((unsigned char)(0xC0 | encode));
7883 }
7884 
7885 void Assembler::andnq(Register dst, Register src1, Address src2) {
7886   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
7887   InstructionMark im(this);
7888   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
7889   vex_prefix(src2, src1->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
7890   emit_int8((unsigned char)0xF2);
7891   emit_operand(dst, src2);
7892 }
7893 
7894 void Assembler::bsfq(Register dst, Register src) {
7895   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
7896   emit_int8(0x0F);
7897   emit_int8((unsigned char)0xBC);
7898   emit_int8((unsigned char)(0xC0 | encode));
7899 }
7900 
7901 void Assembler::bsrq(Register dst, Register src) {
7902   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
7903   emit_int8(0x0F);
7904   emit_int8((unsigned char)0xBD);
7905   emit_int8((unsigned char)(0xC0 | encode));
7906 }
7907 
7908 void Assembler::bswapq(Register reg) {
7909   int encode = prefixq_and_encode(reg->encoding());
7910   emit_int8(0x0F);
7911   emit_int8((unsigned char)(0xC8 | encode));
7912 }
7913 
7914 void Assembler::blsiq(Register dst, Register src) {
7915   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
7916   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
7917   int encode = vex_prefix_and_encode(rbx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
7918   emit_int8((unsigned char)0xF3);
7919   emit_int8((unsigned char)(0xC0 | encode));
7920 }
7921 
7922 void Assembler::blsiq(Register dst, Address src) {
7923   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
7924   InstructionMark im(this);
7925   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
7926   vex_prefix(src, dst->encoding(), rbx->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
7927   emit_int8((unsigned char)0xF3);
7928   emit_operand(rbx, src);
7929 }
7930 
7931 void Assembler::blsmskq(Register dst, Register src) {
7932   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
7933   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
7934   int encode = vex_prefix_and_encode(rdx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
7935   emit_int8((unsigned char)0xF3);
7936   emit_int8((unsigned char)(0xC0 | encode));
7937 }
7938 
7939 void Assembler::blsmskq(Register dst, Address src) {
7940   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
7941   InstructionMark im(this);
7942   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
7943   vex_prefix(src, dst->encoding(), rdx->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
7944   emit_int8((unsigned char)0xF3);
7945   emit_operand(rdx, src);
7946 }
7947 
7948 void Assembler::blsrq(Register dst, Register src) {
7949   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
7950   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
7951   int encode = vex_prefix_and_encode(rcx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
7952   emit_int8((unsigned char)0xF3);
7953   emit_int8((unsigned char)(0xC0 | encode));
7954 }
7955 
7956 void Assembler::blsrq(Register dst, Address src) {
7957   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
7958   InstructionMark im(this);
7959   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
7960   vex_prefix(src, dst->encoding(), rcx->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
7961   emit_int8((unsigned char)0xF3);
7962   emit_operand(rcx, src);
7963 }
7964 
7965 void Assembler::cdqq() {
7966   prefix(REX_W);
7967   emit_int8((unsigned char)0x99);
7968 }
7969 
7970 void Assembler::clflush(Address adr) {
7971   prefix(adr);
7972   emit_int8(0x0F);
7973   emit_int8((unsigned char)0xAE);
7974   emit_operand(rdi, adr);
7975 }
7976 
7977 void Assembler::cmovq(Condition cc, Register dst, Register src) {
7978   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
7979   emit_int8(0x0F);
7980   emit_int8(0x40 | cc);
7981   emit_int8((unsigned char)(0xC0 | encode));
7982 }
7983 
7984 void Assembler::cmovq(Condition cc, Register dst, Address src) {
7985   InstructionMark im(this);
7986   prefixq(src, dst);
7987   emit_int8(0x0F);
7988   emit_int8(0x40 | cc);
7989   emit_operand(dst, src);
7990 }
7991 
7992 void Assembler::cmpq(Address dst, int32_t imm32) {
7993   InstructionMark im(this);
7994   prefixq(dst);
7995   emit_int8((unsigned char)0x81);
7996   emit_operand(rdi, dst, 4);
7997   emit_int32(imm32);
7998 }
7999 
8000 void Assembler::cmpq(Register dst, int32_t imm32) {
8001   (void) prefixq_and_encode(dst->encoding());
8002   emit_arith(0x81, 0xF8, dst, imm32);
8003 }
8004 
8005 void Assembler::cmpq(Address dst, Register src) {
8006   InstructionMark im(this);
8007   prefixq(dst, src);
8008   emit_int8(0x3B);
8009   emit_operand(src, dst);
8010 }
8011 
8012 void Assembler::cmpq(Register dst, Register src) {
8013   (void) prefixq_and_encode(dst->encoding(), src->encoding());
8014   emit_arith(0x3B, 0xC0, dst, src);
8015 }
8016 
8017 void Assembler::cmpq(Register dst, Address  src) {
8018   InstructionMark im(this);
8019   prefixq(src, dst);
8020   emit_int8(0x3B);
8021   emit_operand(dst, src);
8022 }
8023 
8024 void Assembler::cmpxchgq(Register reg, Address adr) {
8025   InstructionMark im(this);
8026   prefixq(adr, reg);
8027   emit_int8(0x0F);
8028   emit_int8((unsigned char)0xB1);
8029   emit_operand(reg, adr);
8030 }
8031 
8032 void Assembler::cvtsi2sdq(XMMRegister dst, Register src) {
8033   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
8034   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
8035   int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
8036   emit_int8(0x2A);
8037   emit_int8((unsigned char)(0xC0 | encode));
8038 }
8039 
8040 void Assembler::cvtsi2sdq(XMMRegister dst, Address src) {
8041   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
8042   InstructionMark im(this);
8043   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
8044   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
8045   simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
8046   emit_int8(0x2A);
8047   emit_operand(dst, src);
8048 }
8049 
8050 void Assembler::cvtsi2ssq(XMMRegister dst, Address src) {
8051   NOT_LP64(assert(VM_Version::supports_sse(), ""));
8052   InstructionMark im(this);
8053   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
8054   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
8055   simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
8056   emit_int8(0x2A);
8057   emit_operand(dst, src);
8058 }
8059 
8060 void Assembler::cvttsd2siq(Register dst, XMMRegister src) {
8061   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
8062   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
8063   int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
8064   emit_int8(0x2C);
8065   emit_int8((unsigned char)(0xC0 | encode));
8066 }
8067 
8068 void Assembler::cvttss2siq(Register dst, XMMRegister src) {
8069   NOT_LP64(assert(VM_Version::supports_sse(), ""));
8070   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
8071   int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
8072   emit_int8(0x2C);
8073   emit_int8((unsigned char)(0xC0 | encode));
8074 }
8075 
8076 void Assembler::decl(Register dst) {
8077   // Don't use it directly. Use MacroAssembler::decrementl() instead.
8078   // Use two-byte form (one-byte form is a REX prefix in 64-bit mode)
8079   int encode = prefix_and_encode(dst->encoding());
8080   emit_int8((unsigned char)0xFF);
8081   emit_int8((unsigned char)(0xC8 | encode));
8082 }
8083 
8084 void Assembler::decq(Register dst) {
8085   // Don't use it directly. Use MacroAssembler::decrementq() instead.
8086   // Use two-byte form (one-byte from is a REX prefix in 64-bit mode)
8087   int encode = prefixq_and_encode(dst->encoding());
8088   emit_int8((unsigned char)0xFF);
8089   emit_int8(0xC8 | encode);
8090 }
8091 
8092 void Assembler::decq(Address dst) {
8093   // Don't use it directly. Use MacroAssembler::decrementq() instead.
8094   InstructionMark im(this);
8095   prefixq(dst);
8096   emit_int8((unsigned char)0xFF);
8097   emit_operand(rcx, dst);
8098 }
8099 
8100 void Assembler::fxrstor(Address src) {
8101   prefixq(src);
8102   emit_int8(0x0F);
8103   emit_int8((unsigned char)0xAE);
8104   emit_operand(as_Register(1), src);
8105 }
8106 
8107 void Assembler::xrstor(Address src) {
8108   prefixq(src);
8109   emit_int8(0x0F);
8110   emit_int8((unsigned char)0xAE);
8111   emit_operand(as_Register(5), src);
8112 }
8113 
8114 void Assembler::fxsave(Address dst) {
8115   prefixq(dst);
8116   emit_int8(0x0F);
8117   emit_int8((unsigned char)0xAE);
8118   emit_operand(as_Register(0), dst);
8119 }
8120 
8121 void Assembler::xsave(Address dst) {
8122   prefixq(dst);
8123   emit_int8(0x0F);
8124   emit_int8((unsigned char)0xAE);
8125   emit_operand(as_Register(4), dst);
8126 }
8127 
8128 void Assembler::idivq(Register src) {
8129   int encode = prefixq_and_encode(src->encoding());
8130   emit_int8((unsigned char)0xF7);
8131   emit_int8((unsigned char)(0xF8 | encode));
8132 }
8133 
8134 void Assembler::imulq(Register dst, Register src) {
8135   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
8136   emit_int8(0x0F);
8137   emit_int8((unsigned char)0xAF);
8138   emit_int8((unsigned char)(0xC0 | encode));
8139 }
8140 
8141 void Assembler::imulq(Register dst, Register src, int value) {
8142   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
8143   if (is8bit(value)) {
8144     emit_int8(0x6B);
8145     emit_int8((unsigned char)(0xC0 | encode));
8146     emit_int8(value & 0xFF);
8147   } else {
8148     emit_int8(0x69);
8149     emit_int8((unsigned char)(0xC0 | encode));
8150     emit_int32(value);
8151   }
8152 }
8153 
8154 void Assembler::imulq(Register dst, Address src) {
8155   InstructionMark im(this);
8156   prefixq(src, dst);
8157   emit_int8(0x0F);
8158   emit_int8((unsigned char) 0xAF);
8159   emit_operand(dst, src);
8160 }
8161 
8162 void Assembler::incl(Register dst) {
8163   // Don't use it directly. Use MacroAssembler::incrementl() instead.
8164   // Use two-byte form (one-byte from is a REX prefix in 64-bit mode)
8165   int encode = prefix_and_encode(dst->encoding());
8166   emit_int8((unsigned char)0xFF);
8167   emit_int8((unsigned char)(0xC0 | encode));
8168 }
8169 
8170 void Assembler::incq(Register dst) {
8171   // Don't use it directly. Use MacroAssembler::incrementq() instead.
8172   // Use two-byte form (one-byte from is a REX prefix in 64-bit mode)
8173   int encode = prefixq_and_encode(dst->encoding());
8174   emit_int8((unsigned char)0xFF);
8175   emit_int8((unsigned char)(0xC0 | encode));
8176 }
8177 
8178 void Assembler::incq(Address dst) {
8179   // Don't use it directly. Use MacroAssembler::incrementq() instead.
8180   InstructionMark im(this);
8181   prefixq(dst);
8182   emit_int8((unsigned char)0xFF);
8183   emit_operand(rax, dst);
8184 }
8185 
8186 void Assembler::lea(Register dst, Address src) {
8187   leaq(dst, src);
8188 }
8189 
8190 void Assembler::leaq(Register dst, Address src) {
8191   InstructionMark im(this);
8192   prefixq(src, dst);
8193   emit_int8((unsigned char)0x8D);
8194   emit_operand(dst, src);
8195 }
8196 
8197 void Assembler::mov64(Register dst, int64_t imm64) {
8198   InstructionMark im(this);
8199   int encode = prefixq_and_encode(dst->encoding());
8200   emit_int8((unsigned char)(0xB8 | encode));
8201   emit_int64(imm64);
8202 }
8203 
8204 void Assembler::mov_literal64(Register dst, intptr_t imm64, RelocationHolder const& rspec) {
8205   InstructionMark im(this);
8206   int encode = prefixq_and_encode(dst->encoding());
8207   emit_int8(0xB8 | encode);
8208   emit_data64(imm64, rspec);
8209 }
8210 
8211 void Assembler::mov_narrow_oop(Register dst, int32_t imm32, RelocationHolder const& rspec) {
8212   InstructionMark im(this);
8213   int encode = prefix_and_encode(dst->encoding());
8214   emit_int8((unsigned char)(0xB8 | encode));
8215   emit_data((int)imm32, rspec, narrow_oop_operand);
8216 }
8217 
8218 void Assembler::mov_narrow_oop(Address dst, int32_t imm32,  RelocationHolder const& rspec) {
8219   InstructionMark im(this);
8220   prefix(dst);
8221   emit_int8((unsigned char)0xC7);
8222   emit_operand(rax, dst, 4);
8223   emit_data((int)imm32, rspec, narrow_oop_operand);
8224 }
8225 
8226 void Assembler::cmp_narrow_oop(Register src1, int32_t imm32, RelocationHolder const& rspec) {
8227   InstructionMark im(this);
8228   int encode = prefix_and_encode(src1->encoding());
8229   emit_int8((unsigned char)0x81);
8230   emit_int8((unsigned char)(0xF8 | encode));
8231   emit_data((int)imm32, rspec, narrow_oop_operand);
8232 }
8233 
8234 void Assembler::cmp_narrow_oop(Address src1, int32_t imm32, RelocationHolder const& rspec) {
8235   InstructionMark im(this);
8236   prefix(src1);
8237   emit_int8((unsigned char)0x81);
8238   emit_operand(rax, src1, 4);
8239   emit_data((int)imm32, rspec, narrow_oop_operand);
8240 }
8241 
8242 void Assembler::lzcntq(Register dst, Register src) {
8243   assert(VM_Version::supports_lzcnt(), "encoding is treated as BSR");
8244   emit_int8((unsigned char)0xF3);
8245   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
8246   emit_int8(0x0F);
8247   emit_int8((unsigned char)0xBD);
8248   emit_int8((unsigned char)(0xC0 | encode));
8249 }
8250 
8251 void Assembler::movdq(XMMRegister dst, Register src) {
8252   // table D-1 says MMX/SSE2
8253   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
8254   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
8255   int encode = simd_prefix_and_encode(dst, xnoreg, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
8256   emit_int8(0x6E);
8257   emit_int8((unsigned char)(0xC0 | encode));
8258 }
8259 
8260 void Assembler::movdq(Register dst, XMMRegister src) {
8261   // table D-1 says MMX/SSE2
8262   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
8263   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
8264   // swap src/dst to get correct prefix
8265   int encode = simd_prefix_and_encode(src, xnoreg, as_XMMRegister(dst->encoding()), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
8266   emit_int8(0x7E);
8267   emit_int8((unsigned char)(0xC0 | encode));
8268 }
8269 
8270 void Assembler::movq(Register dst, Register src) {
8271   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
8272   emit_int8((unsigned char)0x8B);
8273   emit_int8((unsigned char)(0xC0 | encode));
8274 }
8275 
8276 void Assembler::movq(Register dst, Address src) {
8277   InstructionMark im(this);
8278   prefixq(src, dst);
8279   emit_int8((unsigned char)0x8B);
8280   emit_operand(dst, src);
8281 }
8282 
8283 void Assembler::movq(Address dst, Register src) {
8284   InstructionMark im(this);
8285   prefixq(dst, src);
8286   emit_int8((unsigned char)0x89);
8287   emit_operand(src, dst);
8288 }
8289 
8290 void Assembler::movsbq(Register dst, Address src) {
8291   InstructionMark im(this);
8292   prefixq(src, dst);
8293   emit_int8(0x0F);
8294   emit_int8((unsigned char)0xBE);
8295   emit_operand(dst, src);
8296 }
8297 
8298 void Assembler::movsbq(Register dst, Register src) {
8299   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
8300   emit_int8(0x0F);
8301   emit_int8((unsigned char)0xBE);
8302   emit_int8((unsigned char)(0xC0 | encode));
8303 }
8304 
8305 void Assembler::movslq(Register dst, int32_t imm32) {
8306   // dbx shows movslq(rcx, 3) as movq     $0x0000000049000000,(%rbx)
8307   // and movslq(r8, 3); as movl     $0x0000000048000000,(%rbx)
8308   // as a result we shouldn't use until tested at runtime...
8309   ShouldNotReachHere();
8310   InstructionMark im(this);
8311   int encode = prefixq_and_encode(dst->encoding());
8312   emit_int8((unsigned char)(0xC7 | encode));
8313   emit_int32(imm32);
8314 }
8315 
8316 void Assembler::movslq(Address dst, int32_t imm32) {
8317   assert(is_simm32(imm32), "lost bits");
8318   InstructionMark im(this);
8319   prefixq(dst);
8320   emit_int8((unsigned char)0xC7);
8321   emit_operand(rax, dst, 4);
8322   emit_int32(imm32);
8323 }
8324 
8325 void Assembler::movslq(Register dst, Address src) {
8326   InstructionMark im(this);
8327   prefixq(src, dst);
8328   emit_int8(0x63);
8329   emit_operand(dst, src);
8330 }
8331 
8332 void Assembler::movslq(Register dst, Register src) {
8333   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
8334   emit_int8(0x63);
8335   emit_int8((unsigned char)(0xC0 | encode));
8336 }
8337 
8338 void Assembler::movswq(Register dst, Address src) {
8339   InstructionMark im(this);
8340   prefixq(src, dst);
8341   emit_int8(0x0F);
8342   emit_int8((unsigned char)0xBF);
8343   emit_operand(dst, src);
8344 }
8345 
8346 void Assembler::movswq(Register dst, Register src) {
8347   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
8348   emit_int8((unsigned char)0x0F);
8349   emit_int8((unsigned char)0xBF);
8350   emit_int8((unsigned char)(0xC0 | encode));
8351 }
8352 
8353 void Assembler::movzbq(Register dst, Address src) {
8354   InstructionMark im(this);
8355   prefixq(src, dst);
8356   emit_int8((unsigned char)0x0F);
8357   emit_int8((unsigned char)0xB6);
8358   emit_operand(dst, src);
8359 }
8360 
8361 void Assembler::movzbq(Register dst, Register src) {
8362   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
8363   emit_int8(0x0F);
8364   emit_int8((unsigned char)0xB6);
8365   emit_int8(0xC0 | encode);
8366 }
8367 
8368 void Assembler::movzwq(Register dst, Address src) {
8369   InstructionMark im(this);
8370   prefixq(src, dst);
8371   emit_int8((unsigned char)0x0F);
8372   emit_int8((unsigned char)0xB7);
8373   emit_operand(dst, src);
8374 }
8375 
8376 void Assembler::movzwq(Register dst, Register src) {
8377   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
8378   emit_int8((unsigned char)0x0F);
8379   emit_int8((unsigned char)0xB7);
8380   emit_int8((unsigned char)(0xC0 | encode));
8381 }
8382 
8383 void Assembler::mulq(Address src) {
8384   InstructionMark im(this);
8385   prefixq(src);
8386   emit_int8((unsigned char)0xF7);
8387   emit_operand(rsp, src);
8388 }
8389 
8390 void Assembler::mulq(Register src) {
8391   int encode = prefixq_and_encode(src->encoding());
8392   emit_int8((unsigned char)0xF7);
8393   emit_int8((unsigned char)(0xE0 | encode));
8394 }
8395 
8396 void Assembler::mulxq(Register dst1, Register dst2, Register src) {
8397   assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
8398   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
8399   int encode = vex_prefix_and_encode(dst1->encoding(), dst2->encoding(), src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_38, &attributes);
8400   emit_int8((unsigned char)0xF6);
8401   emit_int8((unsigned char)(0xC0 | encode));
8402 }
8403 
8404 void Assembler::negq(Register dst) {
8405   int encode = prefixq_and_encode(dst->encoding());
8406   emit_int8((unsigned char)0xF7);
8407   emit_int8((unsigned char)(0xD8 | encode));
8408 }
8409 
8410 void Assembler::notq(Register dst) {
8411   int encode = prefixq_and_encode(dst->encoding());
8412   emit_int8((unsigned char)0xF7);
8413   emit_int8((unsigned char)(0xD0 | encode));
8414 }
8415 
8416 void Assembler::orq(Address dst, int32_t imm32) {
8417   InstructionMark im(this);
8418   prefixq(dst);
8419   emit_int8((unsigned char)0x81);
8420   emit_operand(rcx, dst, 4);
8421   emit_int32(imm32);
8422 }
8423 
8424 void Assembler::orq(Register dst, int32_t imm32) {
8425   (void) prefixq_and_encode(dst->encoding());
8426   emit_arith(0x81, 0xC8, dst, imm32);
8427 }
8428 
8429 void Assembler::orq(Register dst, Address src) {
8430   InstructionMark im(this);
8431   prefixq(src, dst);
8432   emit_int8(0x0B);
8433   emit_operand(dst, src);
8434 }
8435 
8436 void Assembler::orq(Register dst, Register src) {
8437   (void) prefixq_and_encode(dst->encoding(), src->encoding());
8438   emit_arith(0x0B, 0xC0, dst, src);
8439 }
8440 
8441 void Assembler::popa() { // 64bit
8442   movq(r15, Address(rsp, 0));
8443   movq(r14, Address(rsp, wordSize));
8444   movq(r13, Address(rsp, 2 * wordSize));
8445   movq(r12, Address(rsp, 3 * wordSize));
8446   movq(r11, Address(rsp, 4 * wordSize));
8447   movq(r10, Address(rsp, 5 * wordSize));
8448   movq(r9,  Address(rsp, 6 * wordSize));
8449   movq(r8,  Address(rsp, 7 * wordSize));
8450   movq(rdi, Address(rsp, 8 * wordSize));
8451   movq(rsi, Address(rsp, 9 * wordSize));
8452   movq(rbp, Address(rsp, 10 * wordSize));
8453   // skip rsp
8454   movq(rbx, Address(rsp, 12 * wordSize));
8455   movq(rdx, Address(rsp, 13 * wordSize));
8456   movq(rcx, Address(rsp, 14 * wordSize));
8457   movq(rax, Address(rsp, 15 * wordSize));
8458 
8459   addq(rsp, 16 * wordSize);
8460 }
8461 
8462 void Assembler::popcntq(Register dst, Address src) {
8463   assert(VM_Version::supports_popcnt(), "must support");
8464   InstructionMark im(this);
8465   emit_int8((unsigned char)0xF3);
8466   prefixq(src, dst);
8467   emit_int8((unsigned char)0x0F);
8468   emit_int8((unsigned char)0xB8);
8469   emit_operand(dst, src);
8470 }
8471 
8472 void Assembler::popcntq(Register dst, Register src) {
8473   assert(VM_Version::supports_popcnt(), "must support");
8474   emit_int8((unsigned char)0xF3);
8475   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
8476   emit_int8((unsigned char)0x0F);
8477   emit_int8((unsigned char)0xB8);
8478   emit_int8((unsigned char)(0xC0 | encode));
8479 }
8480 
8481 void Assembler::popq(Address dst) {
8482   InstructionMark im(this);
8483   prefixq(dst);
8484   emit_int8((unsigned char)0x8F);
8485   emit_operand(rax, dst);
8486 }
8487 
8488 void Assembler::pusha() { // 64bit
8489   // we have to store original rsp.  ABI says that 128 bytes
8490   // below rsp are local scratch.
8491   movq(Address(rsp, -5 * wordSize), rsp);
8492 
8493   subq(rsp, 16 * wordSize);
8494 
8495   movq(Address(rsp, 15 * wordSize), rax);
8496   movq(Address(rsp, 14 * wordSize), rcx);
8497   movq(Address(rsp, 13 * wordSize), rdx);
8498   movq(Address(rsp, 12 * wordSize), rbx);
8499   // skip rsp
8500   movq(Address(rsp, 10 * wordSize), rbp);
8501   movq(Address(rsp, 9 * wordSize), rsi);
8502   movq(Address(rsp, 8 * wordSize), rdi);
8503   movq(Address(rsp, 7 * wordSize), r8);
8504   movq(Address(rsp, 6 * wordSize), r9);
8505   movq(Address(rsp, 5 * wordSize), r10);
8506   movq(Address(rsp, 4 * wordSize), r11);
8507   movq(Address(rsp, 3 * wordSize), r12);
8508   movq(Address(rsp, 2 * wordSize), r13);
8509   movq(Address(rsp, wordSize), r14);
8510   movq(Address(rsp, 0), r15);
8511 }
8512 
8513 void Assembler::pushq(Address src) {
8514   InstructionMark im(this);
8515   prefixq(src);
8516   emit_int8((unsigned char)0xFF);
8517   emit_operand(rsi, src);
8518 }
8519 
8520 void Assembler::rclq(Register dst, int imm8) {
8521   assert(isShiftCount(imm8 >> 1), "illegal shift count");
8522   int encode = prefixq_and_encode(dst->encoding());
8523   if (imm8 == 1) {
8524     emit_int8((unsigned char)0xD1);
8525     emit_int8((unsigned char)(0xD0 | encode));
8526   } else {
8527     emit_int8((unsigned char)0xC1);
8528     emit_int8((unsigned char)(0xD0 | encode));
8529     emit_int8(imm8);
8530   }
8531 }
8532 
8533 void Assembler::rcrq(Register dst, int imm8) {
8534   assert(isShiftCount(imm8 >> 1), "illegal shift count");
8535   int encode = prefixq_and_encode(dst->encoding());
8536   if (imm8 == 1) {
8537     emit_int8((unsigned char)0xD1);
8538     emit_int8((unsigned char)(0xD8 | encode));
8539   } else {
8540     emit_int8((unsigned char)0xC1);
8541     emit_int8((unsigned char)(0xD8 | encode));
8542     emit_int8(imm8);
8543   }
8544 }
8545 
8546 void Assembler::rorq(Register dst, int imm8) {
8547   assert(isShiftCount(imm8 >> 1), "illegal shift count");
8548   int encode = prefixq_and_encode(dst->encoding());
8549   if (imm8 == 1) {
8550     emit_int8((unsigned char)0xD1);
8551     emit_int8((unsigned char)(0xC8 | encode));
8552   } else {
8553     emit_int8((unsigned char)0xC1);
8554     emit_int8((unsigned char)(0xc8 | encode));
8555     emit_int8(imm8);
8556   }
8557 }
8558 
8559 void Assembler::rorxq(Register dst, Register src, int imm8) {
8560   assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
8561   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
8562   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_3A, &attributes);
8563   emit_int8((unsigned char)0xF0);
8564   emit_int8((unsigned char)(0xC0 | encode));
8565   emit_int8(imm8);
8566 }
8567 
8568 void Assembler::rorxd(Register dst, Register src, int imm8) {
8569   assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
8570   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
8571   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_3A, &attributes);
8572   emit_int8((unsigned char)0xF0);
8573   emit_int8((unsigned char)(0xC0 | encode));
8574   emit_int8(imm8);
8575 }
8576 
8577 void Assembler::sarq(Register dst, int imm8) {
8578   assert(isShiftCount(imm8 >> 1), "illegal shift count");
8579   int encode = prefixq_and_encode(dst->encoding());
8580   if (imm8 == 1) {
8581     emit_int8((unsigned char)0xD1);
8582     emit_int8((unsigned char)(0xF8 | encode));
8583   } else {
8584     emit_int8((unsigned char)0xC1);
8585     emit_int8((unsigned char)(0xF8 | encode));
8586     emit_int8(imm8);
8587   }
8588 }
8589 
8590 void Assembler::sarq(Register dst) {
8591   int encode = prefixq_and_encode(dst->encoding());
8592   emit_int8((unsigned char)0xD3);
8593   emit_int8((unsigned char)(0xF8 | encode));
8594 }
8595 
8596 void Assembler::sbbq(Address dst, int32_t imm32) {
8597   InstructionMark im(this);
8598   prefixq(dst);
8599   emit_arith_operand(0x81, rbx, dst, imm32);
8600 }
8601 
8602 void Assembler::sbbq(Register dst, int32_t imm32) {
8603   (void) prefixq_and_encode(dst->encoding());
8604   emit_arith(0x81, 0xD8, dst, imm32);
8605 }
8606 
8607 void Assembler::sbbq(Register dst, Address src) {
8608   InstructionMark im(this);
8609   prefixq(src, dst);
8610   emit_int8(0x1B);
8611   emit_operand(dst, src);
8612 }
8613 
8614 void Assembler::sbbq(Register dst, Register src) {
8615   (void) prefixq_and_encode(dst->encoding(), src->encoding());
8616   emit_arith(0x1B, 0xC0, dst, src);
8617 }
8618 
8619 void Assembler::shlq(Register dst, int imm8) {
8620   assert(isShiftCount(imm8 >> 1), "illegal shift count");
8621   int encode = prefixq_and_encode(dst->encoding());
8622   if (imm8 == 1) {
8623     emit_int8((unsigned char)0xD1);
8624     emit_int8((unsigned char)(0xE0 | encode));
8625   } else {
8626     emit_int8((unsigned char)0xC1);
8627     emit_int8((unsigned char)(0xE0 | encode));
8628     emit_int8(imm8);
8629   }
8630 }
8631 
8632 void Assembler::shlq(Register dst) {
8633   int encode = prefixq_and_encode(dst->encoding());
8634   emit_int8((unsigned char)0xD3);
8635   emit_int8((unsigned char)(0xE0 | encode));
8636 }
8637 
8638 void Assembler::shrq(Register dst, int imm8) {
8639   assert(isShiftCount(imm8 >> 1), "illegal shift count");
8640   int encode = prefixq_and_encode(dst->encoding());
8641   emit_int8((unsigned char)0xC1);
8642   emit_int8((unsigned char)(0xE8 | encode));
8643   emit_int8(imm8);
8644 }
8645 
8646 void Assembler::shrq(Register dst) {
8647   int encode = prefixq_and_encode(dst->encoding());
8648   emit_int8((unsigned char)0xD3);
8649   emit_int8(0xE8 | encode);
8650 }
8651 
8652 void Assembler::subq(Address dst, int32_t imm32) {
8653   InstructionMark im(this);
8654   prefixq(dst);
8655   emit_arith_operand(0x81, rbp, dst, imm32);
8656 }
8657 
8658 void Assembler::subq(Address dst, Register src) {
8659   InstructionMark im(this);
8660   prefixq(dst, src);
8661   emit_int8(0x29);
8662   emit_operand(src, dst);
8663 }
8664 
8665 void Assembler::subq(Register dst, int32_t imm32) {
8666   (void) prefixq_and_encode(dst->encoding());
8667   emit_arith(0x81, 0xE8, dst, imm32);
8668 }
8669 
8670 // Force generation of a 4 byte immediate value even if it fits into 8bit
8671 void Assembler::subq_imm32(Register dst, int32_t imm32) {
8672   (void) prefixq_and_encode(dst->encoding());
8673   emit_arith_imm32(0x81, 0xE8, dst, imm32);
8674 }
8675 
8676 void Assembler::subq(Register dst, Address src) {
8677   InstructionMark im(this);
8678   prefixq(src, dst);
8679   emit_int8(0x2B);
8680   emit_operand(dst, src);
8681 }
8682 
8683 void Assembler::subq(Register dst, Register src) {
8684   (void) prefixq_and_encode(dst->encoding(), src->encoding());
8685   emit_arith(0x2B, 0xC0, dst, src);
8686 }
8687 
8688 void Assembler::testq(Register dst, int32_t imm32) {
8689   // not using emit_arith because test
8690   // doesn't support sign-extension of
8691   // 8bit operands
8692   int encode = dst->encoding();
8693   if (encode == 0) {
8694     prefix(REX_W);
8695     emit_int8((unsigned char)0xA9);
8696   } else {
8697     encode = prefixq_and_encode(encode);
8698     emit_int8((unsigned char)0xF7);
8699     emit_int8((unsigned char)(0xC0 | encode));
8700   }
8701   emit_int32(imm32);
8702 }
8703 
8704 void Assembler::testq(Register dst, Register src) {
8705   (void) prefixq_and_encode(dst->encoding(), src->encoding());
8706   emit_arith(0x85, 0xC0, dst, src);
8707 }
8708 
8709 void Assembler::xaddq(Address dst, Register src) {
8710   InstructionMark im(this);
8711   prefixq(dst, src);
8712   emit_int8(0x0F);
8713   emit_int8((unsigned char)0xC1);
8714   emit_operand(src, dst);
8715 }
8716 
8717 void Assembler::xchgq(Register dst, Address src) {
8718   InstructionMark im(this);
8719   prefixq(src, dst);
8720   emit_int8((unsigned char)0x87);
8721   emit_operand(dst, src);
8722 }
8723 
8724 void Assembler::xchgq(Register dst, Register src) {
8725   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
8726   emit_int8((unsigned char)0x87);
8727   emit_int8((unsigned char)(0xc0 | encode));
8728 }
8729 
8730 void Assembler::xorq(Register dst, Register src) {
8731   (void) prefixq_and_encode(dst->encoding(), src->encoding());
8732   emit_arith(0x33, 0xC0, dst, src);
8733 }
8734 
8735 void Assembler::xorq(Register dst, Address src) {
8736   InstructionMark im(this);
8737   prefixq(src, dst);
8738   emit_int8(0x33);
8739   emit_operand(dst, src);
8740 }
8741 
8742 #endif // !LP64