1 /*
   2  * Copyright (c) 1997, 2016, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #include "precompiled.hpp"
  26 #include "asm/assembler.hpp"
  27 #include "asm/assembler.inline.hpp"
  28 #include "gc/shared/cardTableModRefBS.hpp"
  29 #include "gc/shared/collectedHeap.inline.hpp"
  30 #include "interpreter/interpreter.hpp"
  31 #include "memory/resourceArea.hpp"
  32 #include "prims/methodHandles.hpp"
  33 #include "runtime/biasedLocking.hpp"
  34 #include "runtime/interfaceSupport.hpp"
  35 #include "runtime/objectMonitor.hpp"
  36 #include "runtime/os.hpp"
  37 #include "runtime/sharedRuntime.hpp"
  38 #include "runtime/stubRoutines.hpp"
  39 #include "utilities/macros.hpp"
  40 #if INCLUDE_ALL_GCS
  41 #include "gc/g1/g1CollectedHeap.inline.hpp"
  42 #include "gc/g1/g1SATBCardTableModRefBS.hpp"
  43 #include "gc/g1/heapRegion.hpp"
  44 #endif // INCLUDE_ALL_GCS
  45 
  46 #ifdef PRODUCT
  47 #define BLOCK_COMMENT(str) /* nothing */
  48 #define STOP(error) stop(error)
  49 #else
  50 #define BLOCK_COMMENT(str) block_comment(str)
  51 #define STOP(error) block_comment(error); stop(error)
  52 #endif
  53 
  54 #define BIND(label) bind(label); BLOCK_COMMENT(#label ":")
  55 // Implementation of AddressLiteral
  56 
  57 // A 2-D table for managing compressed displacement(disp8) on EVEX enabled platforms.
  58 unsigned char tuple_table[Assembler::EVEX_ETUP + 1][Assembler::AVX_512bit + 1] = {
  59   // -----------------Table 4.5 -------------------- //
  60   16, 32, 64,  // EVEX_FV(0)
  61   4,  4,  4,   // EVEX_FV(1) - with Evex.b
  62   16, 32, 64,  // EVEX_FV(2) - with Evex.w
  63   8,  8,  8,   // EVEX_FV(3) - with Evex.w and Evex.b
  64   8,  16, 32,  // EVEX_HV(0)
  65   4,  4,  4,   // EVEX_HV(1) - with Evex.b
  66   // -----------------Table 4.6 -------------------- //
  67   16, 32, 64,  // EVEX_FVM(0)
  68   1,  1,  1,   // EVEX_T1S(0)
  69   2,  2,  2,   // EVEX_T1S(1)
  70   4,  4,  4,   // EVEX_T1S(2)
  71   8,  8,  8,   // EVEX_T1S(3)
  72   4,  4,  4,   // EVEX_T1F(0)
  73   8,  8,  8,   // EVEX_T1F(1)
  74   8,  8,  8,   // EVEX_T2(0)
  75   0,  16, 16,  // EVEX_T2(1)
  76   0,  16, 16,  // EVEX_T4(0)
  77   0,  0,  32,  // EVEX_T4(1)
  78   0,  0,  32,  // EVEX_T8(0)
  79   8,  16, 32,  // EVEX_HVM(0)
  80   4,  8,  16,  // EVEX_QVM(0)
  81   2,  4,  8,   // EVEX_OVM(0)
  82   16, 16, 16,  // EVEX_M128(0)
  83   8,  32, 64,  // EVEX_DUP(0)
  84   0,  0,  0    // EVEX_NTUP
  85 };
  86 
  87 AddressLiteral::AddressLiteral(address target, relocInfo::relocType rtype) {
  88   _is_lval = false;
  89   _target = target;
  90   switch (rtype) {
  91   case relocInfo::oop_type:
  92   case relocInfo::metadata_type:
  93     // Oops are a special case. Normally they would be their own section
  94     // but in cases like icBuffer they are literals in the code stream that
  95     // we don't have a section for. We use none so that we get a literal address
  96     // which is always patchable.
  97     break;
  98   case relocInfo::external_word_type:
  99     _rspec = external_word_Relocation::spec(target);
 100     break;
 101   case relocInfo::internal_word_type:
 102     _rspec = internal_word_Relocation::spec(target);
 103     break;
 104   case relocInfo::opt_virtual_call_type:
 105     _rspec = opt_virtual_call_Relocation::spec();
 106     break;
 107   case relocInfo::static_call_type:
 108     _rspec = static_call_Relocation::spec();
 109     break;
 110   case relocInfo::runtime_call_type:
 111     _rspec = runtime_call_Relocation::spec();
 112     break;
 113   case relocInfo::poll_type:
 114   case relocInfo::poll_return_type:
 115     _rspec = Relocation::spec_simple(rtype);
 116     break;
 117   case relocInfo::none:
 118     break;
 119   default:
 120     ShouldNotReachHere();
 121     break;
 122   }
 123 }
 124 
 125 // Implementation of Address
 126 
 127 #ifdef _LP64
 128 
 129 Address Address::make_array(ArrayAddress adr) {
 130   // Not implementable on 64bit machines
 131   // Should have been handled higher up the call chain.
 132   ShouldNotReachHere();
 133   return Address();
 134 }
 135 
 136 // exceedingly dangerous constructor
 137 Address::Address(int disp, address loc, relocInfo::relocType rtype) {
 138   _base  = noreg;
 139   _index = noreg;
 140   _scale = no_scale;
 141   _disp  = disp;
 142   switch (rtype) {
 143     case relocInfo::external_word_type:
 144       _rspec = external_word_Relocation::spec(loc);
 145       break;
 146     case relocInfo::internal_word_type:
 147       _rspec = internal_word_Relocation::spec(loc);
 148       break;
 149     case relocInfo::runtime_call_type:
 150       // HMM
 151       _rspec = runtime_call_Relocation::spec();
 152       break;
 153     case relocInfo::poll_type:
 154     case relocInfo::poll_return_type:
 155       _rspec = Relocation::spec_simple(rtype);
 156       break;
 157     case relocInfo::none:
 158       break;
 159     default:
 160       ShouldNotReachHere();
 161   }
 162 }
 163 #else // LP64
 164 
 165 Address Address::make_array(ArrayAddress adr) {
 166   AddressLiteral base = adr.base();
 167   Address index = adr.index();
 168   assert(index._disp == 0, "must not have disp"); // maybe it can?
 169   Address array(index._base, index._index, index._scale, (intptr_t) base.target());
 170   array._rspec = base._rspec;
 171   return array;
 172 }
 173 
 174 // exceedingly dangerous constructor
 175 Address::Address(address loc, RelocationHolder spec) {
 176   _base  = noreg;
 177   _index = noreg;
 178   _scale = no_scale;
 179   _disp  = (intptr_t) loc;
 180   _rspec = spec;
 181 }
 182 
 183 #endif // _LP64
 184 
 185 
 186 
 187 // Convert the raw encoding form into the form expected by the constructor for
 188 // Address.  An index of 4 (rsp) corresponds to having no index, so convert
 189 // that to noreg for the Address constructor.
 190 Address Address::make_raw(int base, int index, int scale, int disp, relocInfo::relocType disp_reloc) {
 191   RelocationHolder rspec;
 192   if (disp_reloc != relocInfo::none) {
 193     rspec = Relocation::spec_simple(disp_reloc);
 194   }
 195   bool valid_index = index != rsp->encoding();
 196   if (valid_index) {
 197     Address madr(as_Register(base), as_Register(index), (Address::ScaleFactor)scale, in_ByteSize(disp));
 198     madr._rspec = rspec;
 199     return madr;
 200   } else {
 201     Address madr(as_Register(base), noreg, Address::no_scale, in_ByteSize(disp));
 202     madr._rspec = rspec;
 203     return madr;
 204   }
 205 }
 206 
 207 // Implementation of Assembler
 208 
 209 int AbstractAssembler::code_fill_byte() {
 210   return (u_char)'\xF4'; // hlt
 211 }
 212 
 213 // make this go away someday
 214 void Assembler::emit_data(jint data, relocInfo::relocType rtype, int format) {
 215   if (rtype == relocInfo::none)
 216     emit_int32(data);
 217   else
 218     emit_data(data, Relocation::spec_simple(rtype), format);
 219 }
 220 
 221 void Assembler::emit_data(jint data, RelocationHolder const& rspec, int format) {
 222   assert(imm_operand == 0, "default format must be immediate in this file");
 223   assert(inst_mark() != NULL, "must be inside InstructionMark");
 224   if (rspec.type() !=  relocInfo::none) {
 225     #ifdef ASSERT
 226       check_relocation(rspec, format);
 227     #endif
 228     // Do not use AbstractAssembler::relocate, which is not intended for
 229     // embedded words.  Instead, relocate to the enclosing instruction.
 230 
 231     // hack. call32 is too wide for mask so use disp32
 232     if (format == call32_operand)
 233       code_section()->relocate(inst_mark(), rspec, disp32_operand);
 234     else
 235       code_section()->relocate(inst_mark(), rspec, format);
 236   }
 237   emit_int32(data);
 238 }
 239 
 240 static int encode(Register r) {
 241   int enc = r->encoding();
 242   if (enc >= 8) {
 243     enc -= 8;
 244   }
 245   return enc;
 246 }
 247 
 248 void Assembler::emit_arith_b(int op1, int op2, Register dst, int imm8) {
 249   assert(dst->has_byte_register(), "must have byte register");
 250   assert(isByte(op1) && isByte(op2), "wrong opcode");
 251   assert(isByte(imm8), "not a byte");
 252   assert((op1 & 0x01) == 0, "should be 8bit operation");
 253   emit_int8(op1);
 254   emit_int8(op2 | encode(dst));
 255   emit_int8(imm8);
 256 }
 257 
 258 
 259 void Assembler::emit_arith(int op1, int op2, Register dst, int32_t imm32) {
 260   assert(isByte(op1) && isByte(op2), "wrong opcode");
 261   assert((op1 & 0x01) == 1, "should be 32bit operation");
 262   assert((op1 & 0x02) == 0, "sign-extension bit should not be set");
 263   if (is8bit(imm32)) {
 264     emit_int8(op1 | 0x02); // set sign bit
 265     emit_int8(op2 | encode(dst));
 266     emit_int8(imm32 & 0xFF);
 267   } else {
 268     emit_int8(op1);
 269     emit_int8(op2 | encode(dst));
 270     emit_int32(imm32);
 271   }
 272 }
 273 
 274 // Force generation of a 4 byte immediate value even if it fits into 8bit
 275 void Assembler::emit_arith_imm32(int op1, int op2, Register dst, int32_t imm32) {
 276   assert(isByte(op1) && isByte(op2), "wrong opcode");
 277   assert((op1 & 0x01) == 1, "should be 32bit operation");
 278   assert((op1 & 0x02) == 0, "sign-extension bit should not be set");
 279   emit_int8(op1);
 280   emit_int8(op2 | encode(dst));
 281   emit_int32(imm32);
 282 }
 283 
 284 // immediate-to-memory forms
 285 void Assembler::emit_arith_operand(int op1, Register rm, Address adr, int32_t imm32) {
 286   assert((op1 & 0x01) == 1, "should be 32bit operation");
 287   assert((op1 & 0x02) == 0, "sign-extension bit should not be set");
 288   if (is8bit(imm32)) {
 289     emit_int8(op1 | 0x02); // set sign bit
 290     emit_operand(rm, adr, 1);
 291     emit_int8(imm32 & 0xFF);
 292   } else {
 293     emit_int8(op1);
 294     emit_operand(rm, adr, 4);
 295     emit_int32(imm32);
 296   }
 297 }
 298 
 299 
 300 void Assembler::emit_arith(int op1, int op2, Register dst, Register src) {
 301   assert(isByte(op1) && isByte(op2), "wrong opcode");
 302   emit_int8(op1);
 303   emit_int8(op2 | encode(dst) << 3 | encode(src));
 304 }
 305 
 306 
 307 bool Assembler::query_compressed_disp_byte(int disp, bool is_evex_inst, int vector_len,
 308                                            int cur_tuple_type, int in_size_in_bits, int cur_encoding) {
 309   int mod_idx = 0;
 310   // We will test if the displacement fits the compressed format and if so
 311   // apply the compression to the displacment iff the result is8bit.
 312   if (VM_Version::supports_evex() && is_evex_inst) {
 313     switch (cur_tuple_type) {
 314     case EVEX_FV:
 315       if ((cur_encoding & VEX_W) == VEX_W) {
 316         mod_idx = ((cur_encoding & EVEX_Rb) == EVEX_Rb) ? 3 : 2;
 317       } else {
 318         mod_idx = ((cur_encoding & EVEX_Rb) == EVEX_Rb) ? 1 : 0;
 319       }
 320       break;
 321 
 322     case EVEX_HV:
 323       mod_idx = ((cur_encoding & EVEX_Rb) == EVEX_Rb) ? 1 : 0;
 324       break;
 325 
 326     case EVEX_FVM:
 327       break;
 328 
 329     case EVEX_T1S:
 330       switch (in_size_in_bits) {
 331       case EVEX_8bit:
 332         break;
 333 
 334       case EVEX_16bit:
 335         mod_idx = 1;
 336         break;
 337 
 338       case EVEX_32bit:
 339         mod_idx = 2;
 340         break;
 341 
 342       case EVEX_64bit:
 343         mod_idx = 3;
 344         break;
 345       }
 346       break;
 347 
 348     case EVEX_T1F:
 349     case EVEX_T2:
 350     case EVEX_T4:
 351       mod_idx = (in_size_in_bits == EVEX_64bit) ? 1 : 0;
 352       break;
 353 
 354     case EVEX_T8:
 355       break;
 356 
 357     case EVEX_HVM:
 358       break;
 359 
 360     case EVEX_QVM:
 361       break;
 362 
 363     case EVEX_OVM:
 364       break;
 365 
 366     case EVEX_M128:
 367       break;
 368 
 369     case EVEX_DUP:
 370       break;
 371 
 372     default:
 373       assert(0, "no valid evex tuple_table entry");
 374       break;
 375     }
 376 
 377     if (vector_len >= AVX_128bit && vector_len <= AVX_512bit) {
 378       int disp_factor = tuple_table[cur_tuple_type + mod_idx][vector_len];
 379       if ((disp % disp_factor) == 0) {
 380         int new_disp = disp / disp_factor;
 381         if ((-0x80 <= new_disp && new_disp < 0x80)) {
 382           disp = new_disp;
 383         }
 384       } else {
 385         return false;
 386       }
 387     }
 388   }
 389   return (-0x80 <= disp && disp < 0x80);
 390 }
 391 
 392 
 393 bool Assembler::emit_compressed_disp_byte(int &disp) {
 394   int mod_idx = 0;
 395   // We will test if the displacement fits the compressed format and if so
 396   // apply the compression to the displacment iff the result is8bit.
 397   if (VM_Version::supports_evex() && _attributes && _attributes->is_evex_instruction()) {
 398     int evex_encoding = _attributes->get_evex_encoding();
 399     int tuple_type = _attributes->get_tuple_type();
 400     switch (tuple_type) {
 401     case EVEX_FV:
 402       if ((evex_encoding & VEX_W) == VEX_W) {
 403         mod_idx = ((evex_encoding & EVEX_Rb) == EVEX_Rb) ? 3 : 2;
 404       } else {
 405         mod_idx = ((evex_encoding & EVEX_Rb) == EVEX_Rb) ? 1 : 0;
 406       }
 407       break;
 408 
 409     case EVEX_HV:
 410       mod_idx = ((evex_encoding & EVEX_Rb) == EVEX_Rb) ? 1 : 0;
 411       break;
 412 
 413     case EVEX_FVM:
 414       break;
 415 
 416     case EVEX_T1S:
 417       switch (_attributes->get_input_size()) {
 418       case EVEX_8bit:
 419         break;
 420 
 421       case EVEX_16bit:
 422         mod_idx = 1;
 423         break;
 424 
 425       case EVEX_32bit:
 426         mod_idx = 2;
 427         break;
 428 
 429       case EVEX_64bit:
 430         mod_idx = 3;
 431         break;
 432       }
 433       break;
 434 
 435     case EVEX_T1F:
 436     case EVEX_T2:
 437     case EVEX_T4:
 438       mod_idx = (_attributes->get_input_size() == EVEX_64bit) ? 1 : 0;
 439       break;
 440 
 441     case EVEX_T8:
 442       break;
 443 
 444     case EVEX_HVM:
 445       break;
 446 
 447     case EVEX_QVM:
 448       break;
 449 
 450     case EVEX_OVM:
 451       break;
 452 
 453     case EVEX_M128:
 454       break;
 455 
 456     case EVEX_DUP:
 457       break;
 458 
 459     default:
 460       assert(0, "no valid evex tuple_table entry");
 461       break;
 462     }
 463 
 464     int vector_len = _attributes->get_vector_len();
 465     if (vector_len >= AVX_128bit && vector_len <= AVX_512bit) {
 466       int disp_factor = tuple_table[tuple_type + mod_idx][vector_len];
 467       if ((disp % disp_factor) == 0) {
 468         int new_disp = disp / disp_factor;
 469         if (is8bit(new_disp)) {
 470           disp = new_disp;
 471         }
 472       } else {
 473         return false;
 474       }
 475     }
 476   }
 477   return is8bit(disp);
 478 }
 479 
 480 
 481 void Assembler::emit_operand(Register reg, Register base, Register index,
 482                              Address::ScaleFactor scale, int disp,
 483                              RelocationHolder const& rspec,
 484                              int rip_relative_correction) {
 485   relocInfo::relocType rtype = (relocInfo::relocType) rspec.type();
 486 
 487   // Encode the registers as needed in the fields they are used in
 488 
 489   int regenc = encode(reg) << 3;
 490   int indexenc = index->is_valid() ? encode(index) << 3 : 0;
 491   int baseenc = base->is_valid() ? encode(base) : 0;
 492 
 493   if (base->is_valid()) {
 494     if (index->is_valid()) {
 495       assert(scale != Address::no_scale, "inconsistent address");
 496       // [base + index*scale + disp]
 497       if (disp == 0 && rtype == relocInfo::none  &&
 498           base != rbp LP64_ONLY(&& base != r13)) {
 499         // [base + index*scale]
 500         // [00 reg 100][ss index base]
 501         assert(index != rsp, "illegal addressing mode");
 502         emit_int8(0x04 | regenc);
 503         emit_int8(scale << 6 | indexenc | baseenc);
 504       } else if (emit_compressed_disp_byte(disp) && rtype == relocInfo::none) {
 505         // [base + index*scale + imm8]
 506         // [01 reg 100][ss index base] imm8
 507         assert(index != rsp, "illegal addressing mode");
 508         emit_int8(0x44 | regenc);
 509         emit_int8(scale << 6 | indexenc | baseenc);
 510         emit_int8(disp & 0xFF);
 511       } else {
 512         // [base + index*scale + disp32]
 513         // [10 reg 100][ss index base] disp32
 514         assert(index != rsp, "illegal addressing mode");
 515         emit_int8(0x84 | regenc);
 516         emit_int8(scale << 6 | indexenc | baseenc);
 517         emit_data(disp, rspec, disp32_operand);
 518       }
 519     } else if (base == rsp LP64_ONLY(|| base == r12)) {
 520       // [rsp + disp]
 521       if (disp == 0 && rtype == relocInfo::none) {
 522         // [rsp]
 523         // [00 reg 100][00 100 100]
 524         emit_int8(0x04 | regenc);
 525         emit_int8(0x24);
 526       } else if (emit_compressed_disp_byte(disp) && rtype == relocInfo::none) {
 527         // [rsp + imm8]
 528         // [01 reg 100][00 100 100] disp8
 529         emit_int8(0x44 | regenc);
 530         emit_int8(0x24);
 531         emit_int8(disp & 0xFF);
 532       } else {
 533         // [rsp + imm32]
 534         // [10 reg 100][00 100 100] disp32
 535         emit_int8(0x84 | regenc);
 536         emit_int8(0x24);
 537         emit_data(disp, rspec, disp32_operand);
 538       }
 539     } else {
 540       // [base + disp]
 541       assert(base != rsp LP64_ONLY(&& base != r12), "illegal addressing mode");
 542       if (disp == 0 && rtype == relocInfo::none &&
 543           base != rbp LP64_ONLY(&& base != r13)) {
 544         // [base]
 545         // [00 reg base]
 546         emit_int8(0x00 | regenc | baseenc);
 547       } else if (emit_compressed_disp_byte(disp) && rtype == relocInfo::none) {
 548         // [base + disp8]
 549         // [01 reg base] disp8
 550         emit_int8(0x40 | regenc | baseenc);
 551         emit_int8(disp & 0xFF);
 552       } else {
 553         // [base + disp32]
 554         // [10 reg base] disp32
 555         emit_int8(0x80 | regenc | baseenc);
 556         emit_data(disp, rspec, disp32_operand);
 557       }
 558     }
 559   } else {
 560     if (index->is_valid()) {
 561       assert(scale != Address::no_scale, "inconsistent address");
 562       // [index*scale + disp]
 563       // [00 reg 100][ss index 101] disp32
 564       assert(index != rsp, "illegal addressing mode");
 565       emit_int8(0x04 | regenc);
 566       emit_int8(scale << 6 | indexenc | 0x05);
 567       emit_data(disp, rspec, disp32_operand);
 568     } else if (rtype != relocInfo::none ) {
 569       // [disp] (64bit) RIP-RELATIVE (32bit) abs
 570       // [00 000 101] disp32
 571 
 572       emit_int8(0x05 | regenc);
 573       // Note that the RIP-rel. correction applies to the generated
 574       // disp field, but _not_ to the target address in the rspec.
 575 
 576       // disp was created by converting the target address minus the pc
 577       // at the start of the instruction. That needs more correction here.
 578       // intptr_t disp = target - next_ip;
 579       assert(inst_mark() != NULL, "must be inside InstructionMark");
 580       address next_ip = pc() + sizeof(int32_t) + rip_relative_correction;
 581       int64_t adjusted = disp;
 582       // Do rip-rel adjustment for 64bit
 583       LP64_ONLY(adjusted -=  (next_ip - inst_mark()));
 584       assert(is_simm32(adjusted),
 585              "must be 32bit offset (RIP relative address)");
 586       emit_data((int32_t) adjusted, rspec, disp32_operand);
 587 
 588     } else {
 589       // 32bit never did this, did everything as the rip-rel/disp code above
 590       // [disp] ABSOLUTE
 591       // [00 reg 100][00 100 101] disp32
 592       emit_int8(0x04 | regenc);
 593       emit_int8(0x25);
 594       emit_data(disp, rspec, disp32_operand);
 595     }
 596   }
 597 }
 598 
 599 void Assembler::emit_operand(XMMRegister reg, Register base, Register index,
 600                              Address::ScaleFactor scale, int disp,
 601                              RelocationHolder const& rspec) {
 602   if (UseAVX > 2) {
 603     int xreg_enc = reg->encoding();
 604     if (xreg_enc > 15) {
 605       XMMRegister new_reg = as_XMMRegister(xreg_enc & 0xf);
 606       emit_operand((Register)new_reg, base, index, scale, disp, rspec);
 607       return;
 608     }
 609   }
 610   emit_operand((Register)reg, base, index, scale, disp, rspec);
 611 }
 612 
 613 // Secret local extension to Assembler::WhichOperand:
 614 #define end_pc_operand (_WhichOperand_limit)
 615 
 616 address Assembler::locate_operand(address inst, WhichOperand which) {
 617   // Decode the given instruction, and return the address of
 618   // an embedded 32-bit operand word.
 619 
 620   // If "which" is disp32_operand, selects the displacement portion
 621   // of an effective address specifier.
 622   // If "which" is imm64_operand, selects the trailing immediate constant.
 623   // If "which" is call32_operand, selects the displacement of a call or jump.
 624   // Caller is responsible for ensuring that there is such an operand,
 625   // and that it is 32/64 bits wide.
 626 
 627   // If "which" is end_pc_operand, find the end of the instruction.
 628 
 629   address ip = inst;
 630   bool is_64bit = false;
 631 
 632   debug_only(bool has_disp32 = false);
 633   int tail_size = 0; // other random bytes (#32, #16, etc.) at end of insn
 634 
 635   again_after_prefix:
 636   switch (0xFF & *ip++) {
 637 
 638   // These convenience macros generate groups of "case" labels for the switch.
 639 #define REP4(x) (x)+0: case (x)+1: case (x)+2: case (x)+3
 640 #define REP8(x) (x)+0: case (x)+1: case (x)+2: case (x)+3: \
 641              case (x)+4: case (x)+5: case (x)+6: case (x)+7
 642 #define REP16(x) REP8((x)+0): \
 643               case REP8((x)+8)
 644 
 645   case CS_segment:
 646   case SS_segment:
 647   case DS_segment:
 648   case ES_segment:
 649   case FS_segment:
 650   case GS_segment:
 651     // Seems dubious
 652     LP64_ONLY(assert(false, "shouldn't have that prefix"));
 653     assert(ip == inst+1, "only one prefix allowed");
 654     goto again_after_prefix;
 655 
 656   case 0x67:
 657   case REX:
 658   case REX_B:
 659   case REX_X:
 660   case REX_XB:
 661   case REX_R:
 662   case REX_RB:
 663   case REX_RX:
 664   case REX_RXB:
 665     NOT_LP64(assert(false, "64bit prefixes"));
 666     goto again_after_prefix;
 667 
 668   case REX_W:
 669   case REX_WB:
 670   case REX_WX:
 671   case REX_WXB:
 672   case REX_WR:
 673   case REX_WRB:
 674   case REX_WRX:
 675   case REX_WRXB:
 676     NOT_LP64(assert(false, "64bit prefixes"));
 677     is_64bit = true;
 678     goto again_after_prefix;
 679 
 680   case 0xFF: // pushq a; decl a; incl a; call a; jmp a
 681   case 0x88: // movb a, r
 682   case 0x89: // movl a, r
 683   case 0x8A: // movb r, a
 684   case 0x8B: // movl r, a
 685   case 0x8F: // popl a
 686     debug_only(has_disp32 = true);
 687     break;
 688 
 689   case 0x68: // pushq #32
 690     if (which == end_pc_operand) {
 691       return ip + 4;
 692     }
 693     assert(which == imm_operand && !is_64bit, "pushl has no disp32 or 64bit immediate");
 694     return ip;                  // not produced by emit_operand
 695 
 696   case 0x66: // movw ... (size prefix)
 697     again_after_size_prefix2:
 698     switch (0xFF & *ip++) {
 699     case REX:
 700     case REX_B:
 701     case REX_X:
 702     case REX_XB:
 703     case REX_R:
 704     case REX_RB:
 705     case REX_RX:
 706     case REX_RXB:
 707     case REX_W:
 708     case REX_WB:
 709     case REX_WX:
 710     case REX_WXB:
 711     case REX_WR:
 712     case REX_WRB:
 713     case REX_WRX:
 714     case REX_WRXB:
 715       NOT_LP64(assert(false, "64bit prefix found"));
 716       goto again_after_size_prefix2;
 717     case 0x8B: // movw r, a
 718     case 0x89: // movw a, r
 719       debug_only(has_disp32 = true);
 720       break;
 721     case 0xC7: // movw a, #16
 722       debug_only(has_disp32 = true);
 723       tail_size = 2;  // the imm16
 724       break;
 725     case 0x0F: // several SSE/SSE2 variants
 726       ip--;    // reparse the 0x0F
 727       goto again_after_prefix;
 728     default:
 729       ShouldNotReachHere();
 730     }
 731     break;
 732 
 733   case REP8(0xB8): // movl/q r, #32/#64(oop?)
 734     if (which == end_pc_operand)  return ip + (is_64bit ? 8 : 4);
 735     // these asserts are somewhat nonsensical
 736 #ifndef _LP64
 737     assert(which == imm_operand || which == disp32_operand,
 738            "which %d is_64_bit %d ip " INTPTR_FORMAT, which, is_64bit, p2i(ip));
 739 #else
 740     assert((which == call32_operand || which == imm_operand) && is_64bit ||
 741            which == narrow_oop_operand && !is_64bit,
 742            "which %d is_64_bit %d ip " INTPTR_FORMAT, which, is_64bit, p2i(ip));
 743 #endif // _LP64
 744     return ip;
 745 
 746   case 0x69: // imul r, a, #32
 747   case 0xC7: // movl a, #32(oop?)
 748     tail_size = 4;
 749     debug_only(has_disp32 = true); // has both kinds of operands!
 750     break;
 751 
 752   case 0x0F: // movx..., etc.
 753     switch (0xFF & *ip++) {
 754     case 0x3A: // pcmpestri
 755       tail_size = 1;
 756     case 0x38: // ptest, pmovzxbw
 757       ip++; // skip opcode
 758       debug_only(has_disp32 = true); // has both kinds of operands!
 759       break;
 760 
 761     case 0x70: // pshufd r, r/a, #8
 762       debug_only(has_disp32 = true); // has both kinds of operands!
 763     case 0x73: // psrldq r, #8
 764       tail_size = 1;
 765       break;
 766 
 767     case 0x12: // movlps
 768     case 0x28: // movaps
 769     case 0x2E: // ucomiss
 770     case 0x2F: // comiss
 771     case 0x54: // andps
 772     case 0x55: // andnps
 773     case 0x56: // orps
 774     case 0x57: // xorps
 775     case 0x58: // addpd
 776     case 0x59: // mulpd
 777     case 0x6E: // movd
 778     case 0x7E: // movd
 779     case 0xAE: // ldmxcsr, stmxcsr, fxrstor, fxsave, clflush
 780     case 0xFE: // paddd
 781       debug_only(has_disp32 = true);
 782       break;
 783 
 784     case 0xAD: // shrd r, a, %cl
 785     case 0xAF: // imul r, a
 786     case 0xBE: // movsbl r, a (movsxb)
 787     case 0xBF: // movswl r, a (movsxw)
 788     case 0xB6: // movzbl r, a (movzxb)
 789     case 0xB7: // movzwl r, a (movzxw)
 790     case REP16(0x40): // cmovl cc, r, a
 791     case 0xB0: // cmpxchgb
 792     case 0xB1: // cmpxchg
 793     case 0xC1: // xaddl
 794     case 0xC7: // cmpxchg8
 795     case REP16(0x90): // setcc a
 796       debug_only(has_disp32 = true);
 797       // fall out of the switch to decode the address
 798       break;
 799 
 800     case 0xC4: // pinsrw r, a, #8
 801       debug_only(has_disp32 = true);
 802     case 0xC5: // pextrw r, r, #8
 803       tail_size = 1;  // the imm8
 804       break;
 805 
 806     case 0xAC: // shrd r, a, #8
 807       debug_only(has_disp32 = true);
 808       tail_size = 1;  // the imm8
 809       break;
 810 
 811     case REP16(0x80): // jcc rdisp32
 812       if (which == end_pc_operand)  return ip + 4;
 813       assert(which == call32_operand, "jcc has no disp32 or imm");
 814       return ip;
 815     default:
 816       ShouldNotReachHere();
 817     }
 818     break;
 819 
 820   case 0x81: // addl a, #32; addl r, #32
 821     // also: orl, adcl, sbbl, andl, subl, xorl, cmpl
 822     // on 32bit in the case of cmpl, the imm might be an oop
 823     tail_size = 4;
 824     debug_only(has_disp32 = true); // has both kinds of operands!
 825     break;
 826 
 827   case 0x83: // addl a, #8; addl r, #8
 828     // also: orl, adcl, sbbl, andl, subl, xorl, cmpl
 829     debug_only(has_disp32 = true); // has both kinds of operands!
 830     tail_size = 1;
 831     break;
 832 
 833   case 0x9B:
 834     switch (0xFF & *ip++) {
 835     case 0xD9: // fnstcw a
 836       debug_only(has_disp32 = true);
 837       break;
 838     default:
 839       ShouldNotReachHere();
 840     }
 841     break;
 842 
 843   case REP4(0x00): // addb a, r; addl a, r; addb r, a; addl r, a
 844   case REP4(0x10): // adc...
 845   case REP4(0x20): // and...
 846   case REP4(0x30): // xor...
 847   case REP4(0x08): // or...
 848   case REP4(0x18): // sbb...
 849   case REP4(0x28): // sub...
 850   case 0xF7: // mull a
 851   case 0x8D: // lea r, a
 852   case 0x87: // xchg r, a
 853   case REP4(0x38): // cmp...
 854   case 0x85: // test r, a
 855     debug_only(has_disp32 = true); // has both kinds of operands!
 856     break;
 857 
 858   case 0xC1: // sal a, #8; sar a, #8; shl a, #8; shr a, #8
 859   case 0xC6: // movb a, #8
 860   case 0x80: // cmpb a, #8
 861   case 0x6B: // imul r, a, #8
 862     debug_only(has_disp32 = true); // has both kinds of operands!
 863     tail_size = 1; // the imm8
 864     break;
 865 
 866   case 0xC4: // VEX_3bytes
 867   case 0xC5: // VEX_2bytes
 868     assert((UseAVX > 0), "shouldn't have VEX prefix");
 869     assert(ip == inst+1, "no prefixes allowed");
 870     // C4 and C5 are also used as opcodes for PINSRW and PEXTRW instructions
 871     // but they have prefix 0x0F and processed when 0x0F processed above.
 872     //
 873     // In 32-bit mode the VEX first byte C4 and C5 alias onto LDS and LES
 874     // instructions (these instructions are not supported in 64-bit mode).
 875     // To distinguish them bits [7:6] are set in the VEX second byte since
 876     // ModRM byte can not be of the form 11xxxxxx in 32-bit mode. To set
 877     // those VEX bits REX and vvvv bits are inverted.
 878     //
 879     // Fortunately C2 doesn't generate these instructions so we don't need
 880     // to check for them in product version.
 881 
 882     // Check second byte
 883     NOT_LP64(assert((0xC0 & *ip) == 0xC0, "shouldn't have LDS and LES instructions"));
 884 
 885     int vex_opcode;
 886     // First byte
 887     if ((0xFF & *inst) == VEX_3bytes) {
 888       vex_opcode = VEX_OPCODE_MASK & *ip;
 889       ip++; // third byte
 890       is_64bit = ((VEX_W & *ip) == VEX_W);
 891     } else {
 892       vex_opcode = VEX_OPCODE_0F;
 893     }
 894     ip++; // opcode
 895     // To find the end of instruction (which == end_pc_operand).
 896     switch (vex_opcode) {
 897       case VEX_OPCODE_0F:
 898         switch (0xFF & *ip) {
 899         case 0x70: // pshufd r, r/a, #8
 900         case 0x71: // ps[rl|ra|ll]w r, #8
 901         case 0x72: // ps[rl|ra|ll]d r, #8
 902         case 0x73: // ps[rl|ra|ll]q r, #8
 903         case 0xC2: // cmp[ps|pd|ss|sd] r, r, r/a, #8
 904         case 0xC4: // pinsrw r, r, r/a, #8
 905         case 0xC5: // pextrw r/a, r, #8
 906         case 0xC6: // shufp[s|d] r, r, r/a, #8
 907           tail_size = 1;  // the imm8
 908           break;
 909         }
 910         break;
 911       case VEX_OPCODE_0F_3A:
 912         tail_size = 1;
 913         break;
 914     }
 915     ip++; // skip opcode
 916     debug_only(has_disp32 = true); // has both kinds of operands!
 917     break;
 918 
 919   case 0x62: // EVEX_4bytes
 920     assert((UseAVX > 0), "shouldn't have EVEX prefix");
 921     assert(ip == inst+1, "no prefixes allowed");
 922     // no EVEX collisions, all instructions that have 0x62 opcodes
 923     // have EVEX versions and are subopcodes of 0x66
 924     ip++; // skip P0 and exmaine W in P1
 925     is_64bit = ((VEX_W & *ip) == VEX_W);
 926     ip++; // move to P2
 927     ip++; // skip P2, move to opcode
 928     // To find the end of instruction (which == end_pc_operand).
 929     switch (0xFF & *ip) {
 930     case 0x22: // pinsrd r, r/a, #8
 931     case 0x61: // pcmpestri r, r/a, #8
 932     case 0x70: // pshufd r, r/a, #8
 933     case 0x73: // psrldq r, #8
 934       tail_size = 1;  // the imm8
 935       break;
 936     default:
 937       break;
 938     }
 939     ip++; // skip opcode
 940     debug_only(has_disp32 = true); // has both kinds of operands!
 941     break;
 942 
 943   case 0xD1: // sal a, 1; sar a, 1; shl a, 1; shr a, 1
 944   case 0xD3: // sal a, %cl; sar a, %cl; shl a, %cl; shr a, %cl
 945   case 0xD9: // fld_s a; fst_s a; fstp_s a; fldcw a
 946   case 0xDD: // fld_d a; fst_d a; fstp_d a
 947   case 0xDB: // fild_s a; fistp_s a; fld_x a; fstp_x a
 948   case 0xDF: // fild_d a; fistp_d a
 949   case 0xD8: // fadd_s a; fsubr_s a; fmul_s a; fdivr_s a; fcomp_s a
 950   case 0xDC: // fadd_d a; fsubr_d a; fmul_d a; fdivr_d a; fcomp_d a
 951   case 0xDE: // faddp_d a; fsubrp_d a; fmulp_d a; fdivrp_d a; fcompp_d a
 952     debug_only(has_disp32 = true);
 953     break;
 954 
 955   case 0xE8: // call rdisp32
 956   case 0xE9: // jmp  rdisp32
 957     if (which == end_pc_operand)  return ip + 4;
 958     assert(which == call32_operand, "call has no disp32 or imm");
 959     return ip;
 960 
 961   case 0xF0:                    // Lock
 962     assert(os::is_MP(), "only on MP");
 963     goto again_after_prefix;
 964 
 965   case 0xF3:                    // For SSE
 966   case 0xF2:                    // For SSE2
 967     switch (0xFF & *ip++) {
 968     case REX:
 969     case REX_B:
 970     case REX_X:
 971     case REX_XB:
 972     case REX_R:
 973     case REX_RB:
 974     case REX_RX:
 975     case REX_RXB:
 976     case REX_W:
 977     case REX_WB:
 978     case REX_WX:
 979     case REX_WXB:
 980     case REX_WR:
 981     case REX_WRB:
 982     case REX_WRX:
 983     case REX_WRXB:
 984       NOT_LP64(assert(false, "found 64bit prefix"));
 985       ip++;
 986     default:
 987       ip++;
 988     }
 989     debug_only(has_disp32 = true); // has both kinds of operands!
 990     break;
 991 
 992   default:
 993     ShouldNotReachHere();
 994 
 995 #undef REP8
 996 #undef REP16
 997   }
 998 
 999   assert(which != call32_operand, "instruction is not a call, jmp, or jcc");
1000 #ifdef _LP64
1001   assert(which != imm_operand, "instruction is not a movq reg, imm64");
1002 #else
1003   // assert(which != imm_operand || has_imm32, "instruction has no imm32 field");
1004   assert(which != imm_operand || has_disp32, "instruction has no imm32 field");
1005 #endif // LP64
1006   assert(which != disp32_operand || has_disp32, "instruction has no disp32 field");
1007 
1008   // parse the output of emit_operand
1009   int op2 = 0xFF & *ip++;
1010   int base = op2 & 0x07;
1011   int op3 = -1;
1012   const int b100 = 4;
1013   const int b101 = 5;
1014   if (base == b100 && (op2 >> 6) != 3) {
1015     op3 = 0xFF & *ip++;
1016     base = op3 & 0x07;   // refetch the base
1017   }
1018   // now ip points at the disp (if any)
1019 
1020   switch (op2 >> 6) {
1021   case 0:
1022     // [00 reg  100][ss index base]
1023     // [00 reg  100][00   100  esp]
1024     // [00 reg base]
1025     // [00 reg  100][ss index  101][disp32]
1026     // [00 reg  101]               [disp32]
1027 
1028     if (base == b101) {
1029       if (which == disp32_operand)
1030         return ip;              // caller wants the disp32
1031       ip += 4;                  // skip the disp32
1032     }
1033     break;
1034 
1035   case 1:
1036     // [01 reg  100][ss index base][disp8]
1037     // [01 reg  100][00   100  esp][disp8]
1038     // [01 reg base]               [disp8]
1039     ip += 1;                    // skip the disp8
1040     break;
1041 
1042   case 2:
1043     // [10 reg  100][ss index base][disp32]
1044     // [10 reg  100][00   100  esp][disp32]
1045     // [10 reg base]               [disp32]
1046     if (which == disp32_operand)
1047       return ip;                // caller wants the disp32
1048     ip += 4;                    // skip the disp32
1049     break;
1050 
1051   case 3:
1052     // [11 reg base]  (not a memory addressing mode)
1053     break;
1054   }
1055 
1056   if (which == end_pc_operand) {
1057     return ip + tail_size;
1058   }
1059 
1060 #ifdef _LP64
1061   assert(which == narrow_oop_operand && !is_64bit, "instruction is not a movl adr, imm32");
1062 #else
1063   assert(which == imm_operand, "instruction has only an imm field");
1064 #endif // LP64
1065   return ip;
1066 }
1067 
1068 address Assembler::locate_next_instruction(address inst) {
1069   // Secretly share code with locate_operand:
1070   return locate_operand(inst, end_pc_operand);
1071 }
1072 
1073 
1074 #ifdef ASSERT
1075 void Assembler::check_relocation(RelocationHolder const& rspec, int format) {
1076   address inst = inst_mark();
1077   assert(inst != NULL && inst < pc(), "must point to beginning of instruction");
1078   address opnd;
1079 
1080   Relocation* r = rspec.reloc();
1081   if (r->type() == relocInfo::none) {
1082     return;
1083   } else if (r->is_call() || format == call32_operand) {
1084     // assert(format == imm32_operand, "cannot specify a nonzero format");
1085     opnd = locate_operand(inst, call32_operand);
1086   } else if (r->is_data()) {
1087     assert(format == imm_operand || format == disp32_operand
1088            LP64_ONLY(|| format == narrow_oop_operand), "format ok");
1089     opnd = locate_operand(inst, (WhichOperand)format);
1090   } else {
1091     assert(format == imm_operand, "cannot specify a format");
1092     return;
1093   }
1094   assert(opnd == pc(), "must put operand where relocs can find it");
1095 }
1096 #endif // ASSERT
1097 
1098 void Assembler::emit_operand32(Register reg, Address adr) {
1099   assert(reg->encoding() < 8, "no extended registers");
1100   assert(!adr.base_needs_rex() && !adr.index_needs_rex(), "no extended registers");
1101   emit_operand(reg, adr._base, adr._index, adr._scale, adr._disp,
1102                adr._rspec);
1103 }
1104 
1105 void Assembler::emit_operand(Register reg, Address adr,
1106                              int rip_relative_correction) {
1107   emit_operand(reg, adr._base, adr._index, adr._scale, adr._disp,
1108                adr._rspec,
1109                rip_relative_correction);
1110 }
1111 
1112 void Assembler::emit_operand(XMMRegister reg, Address adr) {
1113   emit_operand(reg, adr._base, adr._index, adr._scale, adr._disp,
1114                adr._rspec);
1115 }
1116 
1117 // MMX operations
1118 void Assembler::emit_operand(MMXRegister reg, Address adr) {
1119   assert(!adr.base_needs_rex() && !adr.index_needs_rex(), "no extended registers");
1120   emit_operand((Register)reg, adr._base, adr._index, adr._scale, adr._disp, adr._rspec);
1121 }
1122 
1123 // work around gcc (3.2.1-7a) bug
1124 void Assembler::emit_operand(Address adr, MMXRegister reg) {
1125   assert(!adr.base_needs_rex() && !adr.index_needs_rex(), "no extended registers");
1126   emit_operand((Register)reg, adr._base, adr._index, adr._scale, adr._disp, adr._rspec);
1127 }
1128 
1129 
1130 void Assembler::emit_farith(int b1, int b2, int i) {
1131   assert(isByte(b1) && isByte(b2), "wrong opcode");
1132   assert(0 <= i &&  i < 8, "illegal stack offset");
1133   emit_int8(b1);
1134   emit_int8(b2 + i);
1135 }
1136 
1137 
1138 // Now the Assembler instructions (identical for 32/64 bits)
1139 
1140 void Assembler::adcl(Address dst, int32_t imm32) {
1141   InstructionMark im(this);
1142   prefix(dst);
1143   emit_arith_operand(0x81, rdx, dst, imm32);
1144 }
1145 
1146 void Assembler::adcl(Address dst, Register src) {
1147   InstructionMark im(this);
1148   prefix(dst, src);
1149   emit_int8(0x11);
1150   emit_operand(src, dst);
1151 }
1152 
1153 void Assembler::adcl(Register dst, int32_t imm32) {
1154   prefix(dst);
1155   emit_arith(0x81, 0xD0, dst, imm32);
1156 }
1157 
1158 void Assembler::adcl(Register dst, Address src) {
1159   InstructionMark im(this);
1160   prefix(src, dst);
1161   emit_int8(0x13);
1162   emit_operand(dst, src);
1163 }
1164 
1165 void Assembler::adcl(Register dst, Register src) {
1166   (void) prefix_and_encode(dst->encoding(), src->encoding());
1167   emit_arith(0x13, 0xC0, dst, src);
1168 }
1169 
1170 void Assembler::addl(Address dst, int32_t imm32) {
1171   InstructionMark im(this);
1172   prefix(dst);
1173   emit_arith_operand(0x81, rax, dst, imm32);
1174 }
1175 
1176 void Assembler::addl(Address dst, Register src) {
1177   InstructionMark im(this);
1178   prefix(dst, src);
1179   emit_int8(0x01);
1180   emit_operand(src, dst);
1181 }
1182 
1183 void Assembler::addl(Register dst, int32_t imm32) {
1184   prefix(dst);
1185   emit_arith(0x81, 0xC0, dst, imm32);
1186 }
1187 
1188 void Assembler::addl(Register dst, Address src) {
1189   InstructionMark im(this);
1190   prefix(src, dst);
1191   emit_int8(0x03);
1192   emit_operand(dst, src);
1193 }
1194 
1195 void Assembler::addl(Register dst, Register src) {
1196   (void) prefix_and_encode(dst->encoding(), src->encoding());
1197   emit_arith(0x03, 0xC0, dst, src);
1198 }
1199 
1200 void Assembler::addr_nop_4() {
1201   assert(UseAddressNop, "no CPU support");
1202   // 4 bytes: NOP DWORD PTR [EAX+0]
1203   emit_int8(0x0F);
1204   emit_int8(0x1F);
1205   emit_int8(0x40); // emit_rm(cbuf, 0x1, EAX_enc, EAX_enc);
1206   emit_int8(0);    // 8-bits offset (1 byte)
1207 }
1208 
1209 void Assembler::addr_nop_5() {
1210   assert(UseAddressNop, "no CPU support");
1211   // 5 bytes: NOP DWORD PTR [EAX+EAX*0+0] 8-bits offset
1212   emit_int8(0x0F);
1213   emit_int8(0x1F);
1214   emit_int8(0x44); // emit_rm(cbuf, 0x1, EAX_enc, 0x4);
1215   emit_int8(0x00); // emit_rm(cbuf, 0x0, EAX_enc, EAX_enc);
1216   emit_int8(0);    // 8-bits offset (1 byte)
1217 }
1218 
1219 void Assembler::addr_nop_7() {
1220   assert(UseAddressNop, "no CPU support");
1221   // 7 bytes: NOP DWORD PTR [EAX+0] 32-bits offset
1222   emit_int8(0x0F);
1223   emit_int8(0x1F);
1224   emit_int8((unsigned char)0x80);
1225                    // emit_rm(cbuf, 0x2, EAX_enc, EAX_enc);
1226   emit_int32(0);   // 32-bits offset (4 bytes)
1227 }
1228 
1229 void Assembler::addr_nop_8() {
1230   assert(UseAddressNop, "no CPU support");
1231   // 8 bytes: NOP DWORD PTR [EAX+EAX*0+0] 32-bits offset
1232   emit_int8(0x0F);
1233   emit_int8(0x1F);
1234   emit_int8((unsigned char)0x84);
1235                    // emit_rm(cbuf, 0x2, EAX_enc, 0x4);
1236   emit_int8(0x00); // emit_rm(cbuf, 0x0, EAX_enc, EAX_enc);
1237   emit_int32(0);   // 32-bits offset (4 bytes)
1238 }
1239 
1240 void Assembler::addsd(XMMRegister dst, XMMRegister src) {
1241   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1242   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
1243   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
1244   emit_int8(0x58);
1245   emit_int8((unsigned char)(0xC0 | encode));
1246 }
1247 
1248 void Assembler::addsd(XMMRegister dst, Address src) {
1249   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1250   InstructionMark im(this);
1251   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
1252   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
1253   simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
1254   emit_int8(0x58);
1255   emit_operand(dst, src);
1256 }
1257 
1258 void Assembler::addss(XMMRegister dst, XMMRegister src) {
1259   NOT_LP64(assert(VM_Version::supports_sse(), ""));
1260   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
1261   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
1262   emit_int8(0x58);
1263   emit_int8((unsigned char)(0xC0 | encode));
1264 }
1265 
1266 void Assembler::addss(XMMRegister dst, Address src) {
1267   NOT_LP64(assert(VM_Version::supports_sse(), ""));
1268   InstructionMark im(this);
1269   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
1270   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
1271   simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
1272   emit_int8(0x58);
1273   emit_operand(dst, src);
1274 }
1275 
1276 void Assembler::aesdec(XMMRegister dst, Address src) {
1277   assert(VM_Version::supports_aes(), "");
1278   InstructionMark im(this);
1279   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
1280   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
1281   emit_int8((unsigned char)0xDE);
1282   emit_operand(dst, src);
1283 }
1284 
1285 void Assembler::aesdec(XMMRegister dst, XMMRegister src) {
1286   assert(VM_Version::supports_aes(), "");
1287   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
1288   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
1289   emit_int8((unsigned char)0xDE);
1290   emit_int8(0xC0 | encode);
1291 }
1292 
1293 void Assembler::aesdeclast(XMMRegister dst, Address src) {
1294   assert(VM_Version::supports_aes(), "");
1295   InstructionMark im(this);
1296   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
1297   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
1298   emit_int8((unsigned char)0xDF);
1299   emit_operand(dst, src);
1300 }
1301 
1302 void Assembler::aesdeclast(XMMRegister dst, XMMRegister src) {
1303   assert(VM_Version::supports_aes(), "");
1304   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
1305   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
1306   emit_int8((unsigned char)0xDF);
1307   emit_int8((unsigned char)(0xC0 | encode));
1308 }
1309 
1310 void Assembler::aesenc(XMMRegister dst, Address src) {
1311   assert(VM_Version::supports_aes(), "");
1312   InstructionMark im(this);
1313   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
1314   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
1315   emit_int8((unsigned char)0xDC);
1316   emit_operand(dst, src);
1317 }
1318 
1319 void Assembler::aesenc(XMMRegister dst, XMMRegister src) {
1320   assert(VM_Version::supports_aes(), "");
1321   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
1322   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
1323   emit_int8((unsigned char)0xDC);
1324   emit_int8(0xC0 | encode);
1325 }
1326 
1327 void Assembler::aesenclast(XMMRegister dst, Address src) {
1328   assert(VM_Version::supports_aes(), "");
1329   InstructionMark im(this);
1330   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
1331   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
1332   emit_int8((unsigned char)0xDD);
1333   emit_operand(dst, src);
1334 }
1335 
1336 void Assembler::aesenclast(XMMRegister dst, XMMRegister src) {
1337   assert(VM_Version::supports_aes(), "");
1338   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
1339   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
1340   emit_int8((unsigned char)0xDD);
1341   emit_int8((unsigned char)(0xC0 | encode));
1342 }
1343 
1344 void Assembler::andl(Address dst, int32_t imm32) {
1345   InstructionMark im(this);
1346   prefix(dst);
1347   emit_int8((unsigned char)0x81);
1348   emit_operand(rsp, dst, 4);
1349   emit_int32(imm32);
1350 }
1351 
1352 void Assembler::andl(Register dst, int32_t imm32) {
1353   prefix(dst);
1354   emit_arith(0x81, 0xE0, dst, imm32);
1355 }
1356 
1357 void Assembler::andl(Register dst, Address src) {
1358   InstructionMark im(this);
1359   prefix(src, dst);
1360   emit_int8(0x23);
1361   emit_operand(dst, src);
1362 }
1363 
1364 void Assembler::andl(Register dst, Register src) {
1365   (void) prefix_and_encode(dst->encoding(), src->encoding());
1366   emit_arith(0x23, 0xC0, dst, src);
1367 }
1368 
1369 void Assembler::andnl(Register dst, Register src1, Register src2) {
1370   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1371   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
1372   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
1373   emit_int8((unsigned char)0xF2);
1374   emit_int8((unsigned char)(0xC0 | encode));
1375 }
1376 
1377 void Assembler::andnl(Register dst, Register src1, Address src2) {
1378   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1379   InstructionMark im(this);
1380   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
1381   vex_prefix(src2, src1->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
1382   emit_int8((unsigned char)0xF2);
1383   emit_operand(dst, src2);
1384 }
1385 
1386 void Assembler::bsfl(Register dst, Register src) {
1387   int encode = prefix_and_encode(dst->encoding(), src->encoding());
1388   emit_int8(0x0F);
1389   emit_int8((unsigned char)0xBC);
1390   emit_int8((unsigned char)(0xC0 | encode));
1391 }
1392 
1393 void Assembler::bsrl(Register dst, Register src) {
1394   int encode = prefix_and_encode(dst->encoding(), src->encoding());
1395   emit_int8(0x0F);
1396   emit_int8((unsigned char)0xBD);
1397   emit_int8((unsigned char)(0xC0 | encode));
1398 }
1399 
1400 void Assembler::bswapl(Register reg) { // bswap
1401   int encode = prefix_and_encode(reg->encoding());
1402   emit_int8(0x0F);
1403   emit_int8((unsigned char)(0xC8 | encode));
1404 }
1405 
1406 void Assembler::blsil(Register dst, Register src) {
1407   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1408   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
1409   int encode = vex_prefix_and_encode(rbx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
1410   emit_int8((unsigned char)0xF3);
1411   emit_int8((unsigned char)(0xC0 | encode));
1412 }
1413 
1414 void Assembler::blsil(Register dst, Address src) {
1415   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1416   InstructionMark im(this);
1417   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
1418   vex_prefix(src, dst->encoding(), rbx->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
1419   emit_int8((unsigned char)0xF3);
1420   emit_operand(rbx, src);
1421 }
1422 
1423 void Assembler::blsmskl(Register dst, Register src) {
1424   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1425   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
1426   int encode = vex_prefix_and_encode(rdx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
1427   emit_int8((unsigned char)0xF3);
1428   emit_int8((unsigned char)(0xC0 | encode));
1429 }
1430 
1431 void Assembler::blsmskl(Register dst, Address src) {
1432   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1433   InstructionMark im(this);
1434   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
1435   vex_prefix(src, dst->encoding(), rdx->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
1436   emit_int8((unsigned char)0xF3);
1437   emit_operand(rdx, src);
1438 }
1439 
1440 void Assembler::blsrl(Register dst, Register src) {
1441   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1442   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
1443   int encode = vex_prefix_and_encode(rcx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
1444   emit_int8((unsigned char)0xF3);
1445   emit_int8((unsigned char)(0xC0 | encode));
1446 }
1447 
1448 void Assembler::blsrl(Register dst, Address src) {
1449   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1450   InstructionMark im(this);
1451   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
1452   vex_prefix(src, dst->encoding(), rcx->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
1453   emit_int8((unsigned char)0xF3);
1454   emit_operand(rcx, src);
1455 }
1456 
1457 void Assembler::call(Label& L, relocInfo::relocType rtype) {
1458   // suspect disp32 is always good
1459   int operand = LP64_ONLY(disp32_operand) NOT_LP64(imm_operand);
1460 
1461   if (L.is_bound()) {
1462     const int long_size = 5;
1463     int offs = (int)( target(L) - pc() );
1464     assert(offs <= 0, "assembler error");
1465     InstructionMark im(this);
1466     // 1110 1000 #32-bit disp
1467     emit_int8((unsigned char)0xE8);
1468     emit_data(offs - long_size, rtype, operand);
1469   } else {
1470     InstructionMark im(this);
1471     // 1110 1000 #32-bit disp
1472     L.add_patch_at(code(), locator());
1473 
1474     emit_int8((unsigned char)0xE8);
1475     emit_data(int(0), rtype, operand);
1476   }
1477 }
1478 
1479 void Assembler::call(Register dst) {
1480   int encode = prefix_and_encode(dst->encoding());
1481   emit_int8((unsigned char)0xFF);
1482   emit_int8((unsigned char)(0xD0 | encode));
1483 }
1484 
1485 
1486 void Assembler::call(Address adr) {
1487   InstructionMark im(this);
1488   prefix(adr);
1489   emit_int8((unsigned char)0xFF);
1490   emit_operand(rdx, adr);
1491 }
1492 
1493 void Assembler::call_literal(address entry, RelocationHolder const& rspec) {
1494   assert(entry != NULL, "call most probably wrong");
1495   InstructionMark im(this);
1496   emit_int8((unsigned char)0xE8);
1497   intptr_t disp = entry - (pc() + sizeof(int32_t));
1498   assert(is_simm32(disp), "must be 32bit offset (call2)");
1499   // Technically, should use call32_operand, but this format is
1500   // implied by the fact that we're emitting a call instruction.
1501 
1502   int operand = LP64_ONLY(disp32_operand) NOT_LP64(call32_operand);
1503   emit_data((int) disp, rspec, operand);
1504 }
1505 
1506 void Assembler::cdql() {
1507   emit_int8((unsigned char)0x99);
1508 }
1509 
1510 void Assembler::cld() {
1511   emit_int8((unsigned char)0xFC);
1512 }
1513 
1514 void Assembler::cmovl(Condition cc, Register dst, Register src) {
1515   NOT_LP64(guarantee(VM_Version::supports_cmov(), "illegal instruction"));
1516   int encode = prefix_and_encode(dst->encoding(), src->encoding());
1517   emit_int8(0x0F);
1518   emit_int8(0x40 | cc);
1519   emit_int8((unsigned char)(0xC0 | encode));
1520 }
1521 
1522 
1523 void Assembler::cmovl(Condition cc, Register dst, Address src) {
1524   NOT_LP64(guarantee(VM_Version::supports_cmov(), "illegal instruction"));
1525   prefix(src, dst);
1526   emit_int8(0x0F);
1527   emit_int8(0x40 | cc);
1528   emit_operand(dst, src);
1529 }
1530 
1531 void Assembler::cmpb(Address dst, int imm8) {
1532   InstructionMark im(this);
1533   prefix(dst);
1534   emit_int8((unsigned char)0x80);
1535   emit_operand(rdi, dst, 1);
1536   emit_int8(imm8);
1537 }
1538 
1539 void Assembler::cmpl(Address dst, int32_t imm32) {
1540   InstructionMark im(this);
1541   prefix(dst);
1542   emit_int8((unsigned char)0x81);
1543   emit_operand(rdi, dst, 4);
1544   emit_int32(imm32);
1545 }
1546 
1547 void Assembler::cmpl(Register dst, int32_t imm32) {
1548   prefix(dst);
1549   emit_arith(0x81, 0xF8, dst, imm32);
1550 }
1551 
1552 void Assembler::cmpl(Register dst, Register src) {
1553   (void) prefix_and_encode(dst->encoding(), src->encoding());
1554   emit_arith(0x3B, 0xC0, dst, src);
1555 }
1556 
1557 void Assembler::cmpl(Register dst, Address  src) {
1558   InstructionMark im(this);
1559   prefix(src, dst);
1560   emit_int8((unsigned char)0x3B);
1561   emit_operand(dst, src);
1562 }
1563 
1564 void Assembler::cmpw(Address dst, int imm16) {
1565   InstructionMark im(this);
1566   assert(!dst.base_needs_rex() && !dst.index_needs_rex(), "no extended registers");
1567   emit_int8(0x66);
1568   emit_int8((unsigned char)0x81);
1569   emit_operand(rdi, dst, 2);
1570   emit_int16(imm16);
1571 }
1572 
1573 // The 32-bit cmpxchg compares the value at adr with the contents of rax,
1574 // and stores reg into adr if so; otherwise, the value at adr is loaded into rax,.
1575 // The ZF is set if the compared values were equal, and cleared otherwise.
1576 void Assembler::cmpxchgl(Register reg, Address adr) { // cmpxchg
1577   InstructionMark im(this);
1578   prefix(adr, reg);
1579   emit_int8(0x0F);
1580   emit_int8((unsigned char)0xB1);
1581   emit_operand(reg, adr);
1582 }
1583 
1584 // The 8-bit cmpxchg compares the value at adr with the contents of rax,
1585 // and stores reg into adr if so; otherwise, the value at adr is loaded into rax,.
1586 // The ZF is set if the compared values were equal, and cleared otherwise.
1587 void Assembler::cmpxchgb(Register reg, Address adr) { // cmpxchg
1588   InstructionMark im(this);
1589   prefix(adr, reg, true);
1590   emit_int8(0x0F);
1591   emit_int8((unsigned char)0xB0);
1592   emit_operand(reg, adr);
1593 }
1594 
1595 void Assembler::comisd(XMMRegister dst, Address src) {
1596   // NOTE: dbx seems to decode this as comiss even though the
1597   // 0x66 is there. Strangly ucomisd comes out correct
1598   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1599   InstructionMark im(this);
1600   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);;
1601   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
1602   simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
1603   emit_int8(0x2F);
1604   emit_operand(dst, src);
1605 }
1606 
1607 void Assembler::comisd(XMMRegister dst, XMMRegister src) {
1608   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1609   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1610   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
1611   emit_int8(0x2F);
1612   emit_int8((unsigned char)(0xC0 | encode));
1613 }
1614 
1615 void Assembler::comiss(XMMRegister dst, Address src) {
1616   NOT_LP64(assert(VM_Version::supports_sse(), ""));
1617   InstructionMark im(this);
1618   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1619   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
1620   simd_prefix(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
1621   emit_int8(0x2F);
1622   emit_operand(dst, src);
1623 }
1624 
1625 void Assembler::comiss(XMMRegister dst, XMMRegister src) {
1626   NOT_LP64(assert(VM_Version::supports_sse(), ""));
1627   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1628   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
1629   emit_int8(0x2F);
1630   emit_int8((unsigned char)(0xC0 | encode));
1631 }
1632 
1633 void Assembler::cpuid() {
1634   emit_int8(0x0F);
1635   emit_int8((unsigned char)0xA2);
1636 }
1637 
1638 // Opcode / Instruction                      Op /  En  64 - Bit Mode     Compat / Leg Mode Description                  Implemented
1639 // F2 0F 38 F0 / r       CRC32 r32, r / m8   RM        Valid             Valid             Accumulate CRC32 on r / m8.  v
1640 // F2 REX 0F 38 F0 / r   CRC32 r32, r / m8*  RM        Valid             N.E.              Accumulate CRC32 on r / m8.  -
1641 // F2 REX.W 0F 38 F0 / r CRC32 r64, r / m8   RM        Valid             N.E.              Accumulate CRC32 on r / m8.  -
1642 //
1643 // F2 0F 38 F1 / r       CRC32 r32, r / m16  RM        Valid             Valid             Accumulate CRC32 on r / m16. v
1644 //
1645 // F2 0F 38 F1 / r       CRC32 r32, r / m32  RM        Valid             Valid             Accumulate CRC32 on r / m32. v
1646 //
1647 // F2 REX.W 0F 38 F1 / r CRC32 r64, r / m64  RM        Valid             N.E.              Accumulate CRC32 on r / m64. v
1648 void Assembler::crc32(Register crc, Register v, int8_t sizeInBytes) {
1649   assert(VM_Version::supports_sse4_2(), "");
1650   int8_t w = 0x01;
1651   Prefix p = Prefix_EMPTY;
1652 
1653   emit_int8((int8_t)0xF2);
1654   switch (sizeInBytes) {
1655   case 1:
1656     w = 0;
1657     break;
1658   case 2:
1659   case 4:
1660     break;
1661   LP64_ONLY(case 8:)
1662     // This instruction is not valid in 32 bits
1663     // Note:
1664     // http://www.intel.com/content/dam/www/public/us/en/documents/manuals/64-ia-32-architectures-software-developer-instruction-set-reference-manual-325383.pdf
1665     //
1666     // Page B - 72   Vol. 2C says
1667     // qwreg2 to qwreg            1111 0010 : 0100 1R0B : 0000 1111 : 0011 1000 : 1111 0000 : 11 qwreg1 qwreg2
1668     // mem64 to qwreg             1111 0010 : 0100 1R0B : 0000 1111 : 0011 1000 : 1111 0000 : mod qwreg r / m
1669     //                                                                            F0!!!
1670     // while 3 - 208 Vol. 2A
1671     // F2 REX.W 0F 38 F1 / r       CRC32 r64, r / m64             RM         Valid      N.E.Accumulate CRC32 on r / m64.
1672     //
1673     // the 0 on a last bit is reserved for a different flavor of this instruction :
1674     // F2 REX.W 0F 38 F0 / r       CRC32 r64, r / m8              RM         Valid      N.E.Accumulate CRC32 on r / m8.
1675     p = REX_W;
1676     break;
1677   default:
1678     assert(0, "Unsupported value for a sizeInBytes argument");
1679     break;
1680   }
1681   LP64_ONLY(prefix(crc, v, p);)
1682   emit_int8((int8_t)0x0F);
1683   emit_int8(0x38);
1684   emit_int8((int8_t)(0xF0 | w));
1685   emit_int8(0xC0 | ((crc->encoding() & 0x7) << 3) | (v->encoding() & 7));
1686 }
1687 
1688 void Assembler::crc32(Register crc, Address adr, int8_t sizeInBytes) {
1689   assert(VM_Version::supports_sse4_2(), "");
1690   InstructionMark im(this);
1691   int8_t w = 0x01;
1692   Prefix p = Prefix_EMPTY;
1693 
1694   emit_int8((int8_t)0xF2);
1695   switch (sizeInBytes) {
1696   case 1:
1697     w = 0;
1698     break;
1699   case 2:
1700   case 4:
1701     break;
1702   LP64_ONLY(case 8:)
1703     // This instruction is not valid in 32 bits
1704     p = REX_W;
1705     break;
1706   default:
1707     assert(0, "Unsupported value for a sizeInBytes argument");
1708     break;
1709   }
1710   LP64_ONLY(prefix(crc, adr, p);)
1711   emit_int8((int8_t)0x0F);
1712   emit_int8(0x38);
1713   emit_int8((int8_t)(0xF0 | w));
1714   emit_operand(crc, adr);
1715 }
1716 
1717 void Assembler::cvtdq2pd(XMMRegister dst, XMMRegister src) {
1718   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1719   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
1720   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
1721   emit_int8((unsigned char)0xE6);
1722   emit_int8((unsigned char)(0xC0 | encode));
1723 }
1724 
1725 void Assembler::cvtdq2ps(XMMRegister dst, XMMRegister src) {
1726   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1727   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
1728   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
1729   emit_int8(0x5B);
1730   emit_int8((unsigned char)(0xC0 | encode));
1731 }
1732 
1733 void Assembler::cvtsd2ss(XMMRegister dst, XMMRegister src) {
1734   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1735   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
1736   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
1737   emit_int8(0x5A);
1738   emit_int8((unsigned char)(0xC0 | encode));
1739 }
1740 
1741 void Assembler::cvtsd2ss(XMMRegister dst, Address src) {
1742   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1743   InstructionMark im(this);
1744   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
1745   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
1746   simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
1747   emit_int8(0x5A);
1748   emit_operand(dst, src);
1749 }
1750 
1751 void Assembler::cvtsi2sdl(XMMRegister dst, Register src) {
1752   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1753   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1754   int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
1755   emit_int8(0x2A);
1756   emit_int8((unsigned char)(0xC0 | encode));
1757 }
1758 
1759 void Assembler::cvtsi2sdl(XMMRegister dst, Address src) {
1760   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1761   InstructionMark im(this);
1762   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1763   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
1764   simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
1765   emit_int8(0x2A);
1766   emit_operand(dst, src);
1767 }
1768 
1769 void Assembler::cvtsi2ssl(XMMRegister dst, Register src) {
1770   NOT_LP64(assert(VM_Version::supports_sse(), ""));
1771   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1772   int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
1773   emit_int8(0x2A);
1774   emit_int8((unsigned char)(0xC0 | encode));
1775 }
1776 
1777 void Assembler::cvtsi2ssl(XMMRegister dst, Address src) {
1778   NOT_LP64(assert(VM_Version::supports_sse(), ""));
1779   InstructionMark im(this);
1780   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1781   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
1782   simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
1783   emit_int8(0x2A);
1784   emit_operand(dst, src);
1785 }
1786 
1787 void Assembler::cvtsi2ssq(XMMRegister dst, Register src) {
1788   NOT_LP64(assert(VM_Version::supports_sse(), ""));
1789   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1790   int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
1791   emit_int8(0x2A);
1792   emit_int8((unsigned char)(0xC0 | encode));
1793 }
1794 
1795 void Assembler::cvtss2sd(XMMRegister dst, XMMRegister src) {
1796   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1797   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
1798   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
1799   emit_int8(0x5A);
1800   emit_int8((unsigned char)(0xC0 | encode));
1801 }
1802 
1803 void Assembler::cvtss2sd(XMMRegister dst, Address src) {
1804   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1805   InstructionMark im(this);
1806   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
1807   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
1808   simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
1809   emit_int8(0x5A);
1810   emit_operand(dst, src);
1811 }
1812 
1813 
1814 void Assembler::cvttsd2sil(Register dst, XMMRegister src) {
1815   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1816   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1817   int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
1818   emit_int8(0x2C);
1819   emit_int8((unsigned char)(0xC0 | encode));
1820 }
1821 
1822 void Assembler::cvttss2sil(Register dst, XMMRegister src) {
1823   NOT_LP64(assert(VM_Version::supports_sse(), ""));
1824   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1825   int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
1826   emit_int8(0x2C);
1827   emit_int8((unsigned char)(0xC0 | encode));
1828 }
1829 
1830 void Assembler::cvttpd2dq(XMMRegister dst, XMMRegister src) {
1831   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1832   int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_128bit;
1833   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
1834   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
1835   emit_int8((unsigned char)0xE6);
1836   emit_int8((unsigned char)(0xC0 | encode));
1837 }
1838 
1839 void Assembler::decl(Address dst) {
1840   // Don't use it directly. Use MacroAssembler::decrement() instead.
1841   InstructionMark im(this);
1842   prefix(dst);
1843   emit_int8((unsigned char)0xFF);
1844   emit_operand(rcx, dst);
1845 }
1846 
1847 void Assembler::divsd(XMMRegister dst, Address src) {
1848   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1849   InstructionMark im(this);
1850   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
1851   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
1852   simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
1853   emit_int8(0x5E);
1854   emit_operand(dst, src);
1855 }
1856 
1857 void Assembler::divsd(XMMRegister dst, XMMRegister src) {
1858   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1859   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
1860   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
1861   emit_int8(0x5E);
1862   emit_int8((unsigned char)(0xC0 | encode));
1863 }
1864 
1865 void Assembler::divss(XMMRegister dst, Address src) {
1866   NOT_LP64(assert(VM_Version::supports_sse(), ""));
1867   InstructionMark im(this);
1868   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
1869   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
1870   simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
1871   emit_int8(0x5E);
1872   emit_operand(dst, src);
1873 }
1874 
1875 void Assembler::divss(XMMRegister dst, XMMRegister src) {
1876   NOT_LP64(assert(VM_Version::supports_sse(), ""));
1877   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
1878   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
1879   emit_int8(0x5E);
1880   emit_int8((unsigned char)(0xC0 | encode));
1881 }
1882 
1883 void Assembler::emms() {
1884   NOT_LP64(assert(VM_Version::supports_mmx(), ""));
1885   emit_int8(0x0F);
1886   emit_int8(0x77);
1887 }
1888 
1889 void Assembler::hlt() {
1890   emit_int8((unsigned char)0xF4);
1891 }
1892 
1893 void Assembler::idivl(Register src) {
1894   int encode = prefix_and_encode(src->encoding());
1895   emit_int8((unsigned char)0xF7);
1896   emit_int8((unsigned char)(0xF8 | encode));
1897 }
1898 
1899 void Assembler::divl(Register src) { // Unsigned
1900   int encode = prefix_and_encode(src->encoding());
1901   emit_int8((unsigned char)0xF7);
1902   emit_int8((unsigned char)(0xF0 | encode));
1903 }
1904 
1905 void Assembler::imull(Register src) {
1906   int encode = prefix_and_encode(src->encoding());
1907   emit_int8((unsigned char)0xF7);
1908   emit_int8((unsigned char)(0xE8 | encode));
1909 }
1910 
1911 void Assembler::imull(Register dst, Register src) {
1912   int encode = prefix_and_encode(dst->encoding(), src->encoding());
1913   emit_int8(0x0F);
1914   emit_int8((unsigned char)0xAF);
1915   emit_int8((unsigned char)(0xC0 | encode));
1916 }
1917 
1918 
1919 void Assembler::imull(Register dst, Register src, int value) {
1920   int encode = prefix_and_encode(dst->encoding(), src->encoding());
1921   if (is8bit(value)) {
1922     emit_int8(0x6B);
1923     emit_int8((unsigned char)(0xC0 | encode));
1924     emit_int8(value & 0xFF);
1925   } else {
1926     emit_int8(0x69);
1927     emit_int8((unsigned char)(0xC0 | encode));
1928     emit_int32(value);
1929   }
1930 }
1931 
1932 void Assembler::imull(Register dst, Address src) {
1933   InstructionMark im(this);
1934   prefix(src, dst);
1935   emit_int8(0x0F);
1936   emit_int8((unsigned char) 0xAF);
1937   emit_operand(dst, src);
1938 }
1939 
1940 
1941 void Assembler::incl(Address dst) {
1942   // Don't use it directly. Use MacroAssembler::increment() instead.
1943   InstructionMark im(this);
1944   prefix(dst);
1945   emit_int8((unsigned char)0xFF);
1946   emit_operand(rax, dst);
1947 }
1948 
1949 void Assembler::jcc(Condition cc, Label& L, bool maybe_short) {
1950   InstructionMark im(this);
1951   assert((0 <= cc) && (cc < 16), "illegal cc");
1952   if (L.is_bound()) {
1953     address dst = target(L);
1954     assert(dst != NULL, "jcc most probably wrong");
1955 
1956     const int short_size = 2;
1957     const int long_size = 6;
1958     intptr_t offs = (intptr_t)dst - (intptr_t)pc();
1959     if (maybe_short && is8bit(offs - short_size)) {
1960       // 0111 tttn #8-bit disp
1961       emit_int8(0x70 | cc);
1962       emit_int8((offs - short_size) & 0xFF);
1963     } else {
1964       // 0000 1111 1000 tttn #32-bit disp
1965       assert(is_simm32(offs - long_size),
1966              "must be 32bit offset (call4)");
1967       emit_int8(0x0F);
1968       emit_int8((unsigned char)(0x80 | cc));
1969       emit_int32(offs - long_size);
1970     }
1971   } else {
1972     // Note: could eliminate cond. jumps to this jump if condition
1973     //       is the same however, seems to be rather unlikely case.
1974     // Note: use jccb() if label to be bound is very close to get
1975     //       an 8-bit displacement
1976     L.add_patch_at(code(), locator());
1977     emit_int8(0x0F);
1978     emit_int8((unsigned char)(0x80 | cc));
1979     emit_int32(0);
1980   }
1981 }
1982 
1983 void Assembler::jccb(Condition cc, Label& L) {
1984   if (L.is_bound()) {
1985     const int short_size = 2;
1986     address entry = target(L);
1987 #ifdef ASSERT
1988     intptr_t dist = (intptr_t)entry - ((intptr_t)pc() + short_size);
1989     intptr_t delta = short_branch_delta();
1990     if (delta != 0) {
1991       dist += (dist < 0 ? (-delta) :delta);
1992     }
1993     assert(is8bit(dist), "Dispacement too large for a short jmp");
1994 #endif
1995     intptr_t offs = (intptr_t)entry - (intptr_t)pc();
1996     // 0111 tttn #8-bit disp
1997     emit_int8(0x70 | cc);
1998     emit_int8((offs - short_size) & 0xFF);
1999   } else {
2000     InstructionMark im(this);
2001     L.add_patch_at(code(), locator());
2002     emit_int8(0x70 | cc);
2003     emit_int8(0);
2004   }
2005 }
2006 
2007 void Assembler::jmp(Address adr) {
2008   InstructionMark im(this);
2009   prefix(adr);
2010   emit_int8((unsigned char)0xFF);
2011   emit_operand(rsp, adr);
2012 }
2013 
2014 void Assembler::jmp(Label& L, bool maybe_short) {
2015   if (L.is_bound()) {
2016     address entry = target(L);
2017     assert(entry != NULL, "jmp most probably wrong");
2018     InstructionMark im(this);
2019     const int short_size = 2;
2020     const int long_size = 5;
2021     intptr_t offs = entry - pc();
2022     if (maybe_short && is8bit(offs - short_size)) {
2023       emit_int8((unsigned char)0xEB);
2024       emit_int8((offs - short_size) & 0xFF);
2025     } else {
2026       emit_int8((unsigned char)0xE9);
2027       emit_int32(offs - long_size);
2028     }
2029   } else {
2030     // By default, forward jumps are always 32-bit displacements, since
2031     // we can't yet know where the label will be bound.  If you're sure that
2032     // the forward jump will not run beyond 256 bytes, use jmpb to
2033     // force an 8-bit displacement.
2034     InstructionMark im(this);
2035     L.add_patch_at(code(), locator());
2036     emit_int8((unsigned char)0xE9);
2037     emit_int32(0);
2038   }
2039 }
2040 
2041 void Assembler::jmp(Register entry) {
2042   int encode = prefix_and_encode(entry->encoding());
2043   emit_int8((unsigned char)0xFF);
2044   emit_int8((unsigned char)(0xE0 | encode));
2045 }
2046 
2047 void Assembler::jmp_literal(address dest, RelocationHolder const& rspec) {
2048   InstructionMark im(this);
2049   emit_int8((unsigned char)0xE9);
2050   assert(dest != NULL, "must have a target");
2051   intptr_t disp = dest - (pc() + sizeof(int32_t));
2052   assert(is_simm32(disp), "must be 32bit offset (jmp)");
2053   emit_data(disp, rspec.reloc(), call32_operand);
2054 }
2055 
2056 void Assembler::jmpb(Label& L) {
2057   if (L.is_bound()) {
2058     const int short_size = 2;
2059     address entry = target(L);
2060     assert(entry != NULL, "jmp most probably wrong");
2061 #ifdef ASSERT
2062     intptr_t dist = (intptr_t)entry - ((intptr_t)pc() + short_size);
2063     intptr_t delta = short_branch_delta();
2064     if (delta != 0) {
2065       dist += (dist < 0 ? (-delta) :delta);
2066     }
2067     assert(is8bit(dist), "Dispacement too large for a short jmp");
2068 #endif
2069     intptr_t offs = entry - pc();
2070     emit_int8((unsigned char)0xEB);
2071     emit_int8((offs - short_size) & 0xFF);
2072   } else {
2073     InstructionMark im(this);
2074     L.add_patch_at(code(), locator());
2075     emit_int8((unsigned char)0xEB);
2076     emit_int8(0);
2077   }
2078 }
2079 
2080 void Assembler::ldmxcsr( Address src) {
2081   NOT_LP64(assert(VM_Version::supports_sse(), ""));
2082   InstructionMark im(this);
2083   prefix(src);
2084   emit_int8(0x0F);
2085   emit_int8((unsigned char)0xAE);
2086   emit_operand(as_Register(2), src);
2087 }
2088 
2089 void Assembler::leal(Register dst, Address src) {
2090   InstructionMark im(this);
2091 #ifdef _LP64
2092   emit_int8(0x67); // addr32
2093   prefix(src, dst);
2094 #endif // LP64
2095   emit_int8((unsigned char)0x8D);
2096   emit_operand(dst, src);
2097 }
2098 
2099 void Assembler::lfence() {
2100   emit_int8(0x0F);
2101   emit_int8((unsigned char)0xAE);
2102   emit_int8((unsigned char)0xE8);
2103 }
2104 
2105 void Assembler::lock() {
2106   emit_int8((unsigned char)0xF0);
2107 }
2108 
2109 void Assembler::lzcntl(Register dst, Register src) {
2110   assert(VM_Version::supports_lzcnt(), "encoding is treated as BSR");
2111   emit_int8((unsigned char)0xF3);
2112   int encode = prefix_and_encode(dst->encoding(), src->encoding());
2113   emit_int8(0x0F);
2114   emit_int8((unsigned char)0xBD);
2115   emit_int8((unsigned char)(0xC0 | encode));
2116 }
2117 
2118 // Emit mfence instruction
2119 void Assembler::mfence() {
2120   NOT_LP64(assert(VM_Version::supports_sse2(), "unsupported");)
2121   emit_int8(0x0F);
2122   emit_int8((unsigned char)0xAE);
2123   emit_int8((unsigned char)0xF0);
2124 }
2125 
2126 void Assembler::mov(Register dst, Register src) {
2127   LP64_ONLY(movq(dst, src)) NOT_LP64(movl(dst, src));
2128 }
2129 
2130 void Assembler::movapd(XMMRegister dst, XMMRegister src) {
2131   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2132   int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_128bit;
2133   InstructionAttr attributes(vector_len, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
2134   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2135   emit_int8(0x28);
2136   emit_int8((unsigned char)(0xC0 | encode));
2137 }
2138 
2139 void Assembler::movaps(XMMRegister dst, XMMRegister src) {
2140   NOT_LP64(assert(VM_Version::supports_sse(), ""));
2141   int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_128bit;
2142   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
2143   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2144   emit_int8(0x28);
2145   emit_int8((unsigned char)(0xC0 | encode));
2146 }
2147 
2148 void Assembler::movlhps(XMMRegister dst, XMMRegister src) {
2149   NOT_LP64(assert(VM_Version::supports_sse(), ""));
2150   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2151   int encode = simd_prefix_and_encode(dst, src, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2152   emit_int8(0x16);
2153   emit_int8((unsigned char)(0xC0 | encode));
2154 }
2155 
2156 void Assembler::movb(Register dst, Address src) {
2157   NOT_LP64(assert(dst->has_byte_register(), "must have byte register"));
2158   InstructionMark im(this);
2159   prefix(src, dst, true);
2160   emit_int8((unsigned char)0x8A);
2161   emit_operand(dst, src);
2162 }
2163 
2164 void Assembler::movddup(XMMRegister dst, XMMRegister src) {
2165   NOT_LP64(assert(VM_Version::supports_sse3(), ""));
2166   int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_128bit;
2167   InstructionAttr attributes(vector_len, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2168   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2169   emit_int8(0x12);
2170   emit_int8(0xC0 | encode);
2171 }
2172 
2173 void Assembler::kmovbl(KRegister dst, Register src) {
2174   assert(VM_Version::supports_avx512dq(), "");
2175   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2176   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2177   emit_int8((unsigned char)0x92);
2178   emit_int8((unsigned char)(0xC0 | encode));
2179 }
2180 
2181 void Assembler::kmovbl(Register dst, KRegister src) {
2182   assert(VM_Version::supports_avx512dq(), "");
2183   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2184   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2185   emit_int8((unsigned char)0x93);
2186   emit_int8((unsigned char)(0xC0 | encode));
2187 }
2188 
2189 void Assembler::kmovwl(KRegister dst, Register src) {
2190   assert(VM_Version::supports_evex(), "");
2191   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2192   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2193   emit_int8((unsigned char)0x92);
2194   emit_int8((unsigned char)(0xC0 | encode));
2195 }
2196 
2197 void Assembler::kmovwl(Register dst, KRegister src) {
2198   assert(VM_Version::supports_evex(), "");
2199   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2200   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2201   emit_int8((unsigned char)0x93);
2202   emit_int8((unsigned char)(0xC0 | encode));
2203 }
2204 
2205 void Assembler::kmovdl(KRegister dst, Register src) {
2206   assert(VM_Version::supports_avx512bw(), "");
2207   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2208   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2209   emit_int8((unsigned char)0x92);
2210   emit_int8((unsigned char)(0xC0 | encode));
2211 }
2212 
2213 void Assembler::kmovdl(Register dst, KRegister src) {
2214   assert(VM_Version::supports_avx512bw(), "");
2215   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2216   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2217   emit_int8((unsigned char)0x93);
2218   emit_int8((unsigned char)(0xC0 | encode));
2219 }
2220 
2221 void Assembler::kmovql(KRegister dst, KRegister src) {
2222   assert(VM_Version::supports_avx512bw(), "");
2223   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2224   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2225   emit_int8((unsigned char)0x90);
2226   emit_int8((unsigned char)(0xC0 | encode));
2227 }
2228 
2229 void Assembler::kmovql(KRegister dst, Address src) {
2230   assert(VM_Version::supports_avx512bw(), "");
2231   InstructionMark im(this);
2232   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2233   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2234   emit_int8((unsigned char)0x90);
2235   emit_operand((Register)dst, src);
2236 }
2237 
2238 void Assembler::kmovql(Address dst, KRegister src) {
2239   assert(VM_Version::supports_avx512bw(), "");
2240   InstructionMark im(this);
2241   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2242   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2243   emit_int8((unsigned char)0x90);
2244   emit_operand((Register)src, dst);
2245 }
2246 
2247 void Assembler::kmovql(KRegister dst, Register src) {
2248   assert(VM_Version::supports_avx512bw(), "");
2249   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2250   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2251   emit_int8((unsigned char)0x92);
2252   emit_int8((unsigned char)(0xC0 | encode));
2253 }
2254 
2255 void Assembler::kmovql(Register dst, KRegister src) {
2256   assert(VM_Version::supports_avx512bw(), "");
2257   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2258   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2259   emit_int8((unsigned char)0x93);
2260   emit_int8((unsigned char)(0xC0 | encode));
2261 }
2262 
2263 // This instruction produces ZF or CF flags
2264 void Assembler::kortestbl(KRegister src1, KRegister src2) {
2265   assert(VM_Version::supports_avx512dq(), "");
2266   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2267   int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2268   emit_int8((unsigned char)0x98);
2269   emit_int8((unsigned char)(0xC0 | encode));
2270 }
2271 
2272 // This instruction produces ZF or CF flags
2273 void Assembler::kortestwl(KRegister src1, KRegister src2) {
2274   assert(VM_Version::supports_evex(), "");
2275   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2276   int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2277   emit_int8((unsigned char)0x98);
2278   emit_int8((unsigned char)(0xC0 | encode));
2279 }
2280 
2281 // This instruction produces ZF or CF flags
2282 void Assembler::kortestdl(KRegister src1, KRegister src2) {
2283   assert(VM_Version::supports_avx512bw(), "");
2284   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2285   int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2286   emit_int8((unsigned char)0x98);
2287   emit_int8((unsigned char)(0xC0 | encode));
2288 }
2289 
2290 // This instruction produces ZF or CF flags
2291 void Assembler::kortestql(KRegister src1, KRegister src2) {
2292   assert(VM_Version::supports_avx512bw(), "");
2293   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2294   int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2295   emit_int8((unsigned char)0x98);
2296   emit_int8((unsigned char)(0xC0 | encode));
2297 }
2298 
2299 void Assembler::movb(Address dst, int imm8) {
2300   InstructionMark im(this);
2301    prefix(dst);
2302   emit_int8((unsigned char)0xC6);
2303   emit_operand(rax, dst, 1);
2304   emit_int8(imm8);
2305 }
2306 
2307 
2308 void Assembler::movb(Address dst, Register src) {
2309   assert(src->has_byte_register(), "must have byte register");
2310   InstructionMark im(this);
2311   prefix(dst, src, true);
2312   emit_int8((unsigned char)0x88);
2313   emit_operand(src, dst);
2314 }
2315 
2316 void Assembler::movdl(XMMRegister dst, Register src) {
2317   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2318   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2319   int encode = simd_prefix_and_encode(dst, xnoreg, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2320   emit_int8(0x6E);
2321   emit_int8((unsigned char)(0xC0 | encode));
2322 }
2323 
2324 void Assembler::movdl(Register dst, XMMRegister src) {
2325   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2326   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2327   // swap src/dst to get correct prefix
2328   int encode = simd_prefix_and_encode(src, xnoreg, as_XMMRegister(dst->encoding()), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2329   emit_int8(0x7E);
2330   emit_int8((unsigned char)(0xC0 | encode));
2331 }
2332 
2333 void Assembler::movdl(XMMRegister dst, Address src) {
2334   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2335   InstructionMark im(this);
2336   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2337   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
2338   simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2339   emit_int8(0x6E);
2340   emit_operand(dst, src);
2341 }
2342 
2343 void Assembler::movdl(Address dst, XMMRegister src) {
2344   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2345   InstructionMark im(this);
2346   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2347   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
2348   simd_prefix(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2349   emit_int8(0x7E);
2350   emit_operand(src, dst);
2351 }
2352 
2353 void Assembler::movdqa(XMMRegister dst, XMMRegister src) {
2354   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2355   int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_128bit;
2356   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2357   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2358   emit_int8(0x6F);
2359   emit_int8((unsigned char)(0xC0 | encode));
2360 }
2361 
2362 void Assembler::movdqa(XMMRegister dst, Address src) {
2363   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2364   InstructionMark im(this);
2365   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2366   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2367   simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2368   emit_int8(0x6F);
2369   emit_operand(dst, src);
2370 }
2371 
2372 void Assembler::movdqu(XMMRegister dst, Address src) {
2373   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2374   InstructionMark im(this);
2375   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2376   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2377   simd_prefix(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2378   emit_int8(0x6F);
2379   emit_operand(dst, src);
2380 }
2381 
2382 void Assembler::movdqu(XMMRegister dst, XMMRegister src) {
2383   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2384   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2385   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2386   emit_int8(0x6F);
2387   emit_int8((unsigned char)(0xC0 | encode));
2388 }
2389 
2390 void Assembler::movdqu(Address dst, XMMRegister src) {
2391   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2392   InstructionMark im(this);
2393   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2394   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2395   simd_prefix(src, xnoreg, dst, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2396   emit_int8(0x7F);
2397   emit_operand(src, dst);
2398 }
2399 
2400 // Move Unaligned 256bit Vector
2401 void Assembler::vmovdqu(XMMRegister dst, XMMRegister src) {
2402   assert(UseAVX > 0, "");
2403   InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2404   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2405   emit_int8(0x6F);
2406   emit_int8((unsigned char)(0xC0 | encode));
2407 }
2408 
2409 void Assembler::vmovdqu(XMMRegister dst, Address src) {
2410   assert(UseAVX > 0, "");
2411   InstructionMark im(this);
2412   InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2413   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2414   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2415   emit_int8(0x6F);
2416   emit_operand(dst, src);
2417 }
2418 
2419 void Assembler::vmovdqu(Address dst, XMMRegister src) {
2420   assert(UseAVX > 0, "");
2421   InstructionMark im(this);
2422   InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2423   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2424   // swap src<->dst for encoding
2425   assert(src != xnoreg, "sanity");
2426   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2427   emit_int8(0x7F);
2428   emit_operand(src, dst);
2429 }
2430 
2431 // Move Unaligned EVEX enabled Vector (programmable : 8,16,32,64)
2432 void Assembler::evmovdqub(XMMRegister dst, XMMRegister src, int vector_len) {
2433   assert(VM_Version::supports_evex(), "");
2434   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
2435   int prefix = (_legacy_mode_bw) ? VEX_SIMD_F2 : VEX_SIMD_F3;
2436   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), (Assembler::VexSimdPrefix)prefix, VEX_OPCODE_0F, &attributes);
2437   emit_int8(0x6F);
2438   emit_int8((unsigned char)(0xC0 | encode));
2439 }
2440 
2441 void Assembler::evmovdqub(XMMRegister dst, Address src, int vector_len) {
2442   assert(VM_Version::supports_evex(), "");
2443   InstructionMark im(this);
2444   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
2445   int prefix = (_legacy_mode_bw) ? VEX_SIMD_F2 : VEX_SIMD_F3;
2446   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2447   vex_prefix(src, 0, dst->encoding(), (Assembler::VexSimdPrefix)prefix, VEX_OPCODE_0F, &attributes);
2448   emit_int8(0x6F);
2449   emit_operand(dst, src);
2450 }
2451 
2452 void Assembler::evmovdqub(Address dst, XMMRegister src, int vector_len) {
2453   assert(VM_Version::supports_evex(), "");
2454   assert(src != xnoreg, "sanity");
2455   InstructionMark im(this);
2456   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
2457   int prefix = (_legacy_mode_bw) ? VEX_SIMD_F2 : VEX_SIMD_F3;
2458   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2459   vex_prefix(dst, 0, src->encoding(), (Assembler::VexSimdPrefix)prefix, VEX_OPCODE_0F, &attributes);
2460   emit_int8(0x7F);
2461   emit_operand(src, dst);
2462 }
2463 
2464 void Assembler::evmovdquw(XMMRegister dst, XMMRegister src, int vector_len) {
2465   assert(VM_Version::supports_evex(), "");
2466   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
2467   int prefix = (_legacy_mode_bw) ? VEX_SIMD_F2 : VEX_SIMD_F3;
2468   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), (Assembler::VexSimdPrefix)prefix, VEX_OPCODE_0F, &attributes);
2469   emit_int8(0x6F);
2470   emit_int8((unsigned char)(0xC0 | encode));
2471 }
2472 
2473 void Assembler::evmovdquw(XMMRegister dst, Address src, int vector_len) {
2474   assert(VM_Version::supports_evex(), "");
2475   InstructionMark im(this);
2476   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
2477   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2478   int prefix = (_legacy_mode_bw) ? VEX_SIMD_F2 : VEX_SIMD_F3;
2479   vex_prefix(src, 0, dst->encoding(), (Assembler::VexSimdPrefix)prefix, VEX_OPCODE_0F, &attributes);
2480   emit_int8(0x6F);
2481   emit_operand(dst, src);
2482 }
2483 
2484 void Assembler::evmovdquw(Address dst, XMMRegister src, int vector_len) {
2485   assert(VM_Version::supports_evex(), "");
2486   assert(src != xnoreg, "sanity");
2487   InstructionMark im(this);
2488   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
2489   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2490   int prefix = (_legacy_mode_bw) ? VEX_SIMD_F2 : VEX_SIMD_F3;
2491   vex_prefix(dst, 0, src->encoding(), (Assembler::VexSimdPrefix)prefix, VEX_OPCODE_0F, &attributes);
2492   emit_int8(0x7F);
2493   emit_operand(src, dst);
2494 }
2495 
2496 void Assembler::evmovdqul(XMMRegister dst, XMMRegister src, int vector_len) {
2497   assert(VM_Version::supports_evex(), "");
2498   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2499   attributes.set_is_evex_instruction();
2500   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2501   emit_int8(0x6F);
2502   emit_int8((unsigned char)(0xC0 | encode));
2503 }
2504 
2505 void Assembler::evmovdqul(XMMRegister dst, Address src, int vector_len) {
2506   assert(VM_Version::supports_evex(), "");
2507   InstructionMark im(this);
2508   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false , /* uses_vl */ true);
2509   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2510   attributes.set_is_evex_instruction();
2511   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2512   emit_int8(0x6F);
2513   emit_operand(dst, src);
2514 }
2515 
2516 void Assembler::evmovdqul(Address dst, XMMRegister src, int vector_len) {
2517   assert(VM_Version::supports_evex(), "");
2518   assert(src != xnoreg, "sanity");
2519   InstructionMark im(this);
2520   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2521   attributes.set_is_evex_instruction();
2522   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2523   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2524   emit_int8(0x7F);
2525   emit_operand(src, dst);
2526 }
2527 
2528 void Assembler::evmovdquq(XMMRegister dst, XMMRegister src, int vector_len) {
2529   assert(VM_Version::supports_evex(), "");
2530   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2531   attributes.set_is_evex_instruction();
2532   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2533   emit_int8(0x6F);
2534   emit_int8((unsigned char)(0xC0 | encode));
2535 }
2536 
2537 void Assembler::evmovdquq(XMMRegister dst, Address src, int vector_len) {
2538   assert(VM_Version::supports_evex(), "");
2539   InstructionMark im(this);
2540   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2541   attributes.set_is_evex_instruction();
2542   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2543   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2544   emit_int8(0x6F);
2545   emit_operand(dst, src);
2546 }
2547 
2548 void Assembler::evmovdquq(Address dst, XMMRegister src, int vector_len) {
2549   assert(VM_Version::supports_evex(), "");
2550   assert(src != xnoreg, "sanity");
2551   InstructionMark im(this);
2552   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2553   attributes.set_is_evex_instruction();
2554   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2555   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2556   emit_int8(0x7F);
2557   emit_operand(src, dst);
2558 }
2559 
2560 // Uses zero extension on 64bit
2561 
2562 void Assembler::movl(Register dst, int32_t imm32) {
2563   int encode = prefix_and_encode(dst->encoding());
2564   emit_int8((unsigned char)(0xB8 | encode));
2565   emit_int32(imm32);
2566 }
2567 
2568 void Assembler::movl(Register dst, Register src) {
2569   int encode = prefix_and_encode(dst->encoding(), src->encoding());
2570   emit_int8((unsigned char)0x8B);
2571   emit_int8((unsigned char)(0xC0 | encode));
2572 }
2573 
2574 void Assembler::movl(Register dst, Address src) {
2575   InstructionMark im(this);
2576   prefix(src, dst);
2577   emit_int8((unsigned char)0x8B);
2578   emit_operand(dst, src);
2579 }
2580 
2581 void Assembler::movl(Address dst, int32_t imm32) {
2582   InstructionMark im(this);
2583   prefix(dst);
2584   emit_int8((unsigned char)0xC7);
2585   emit_operand(rax, dst, 4);
2586   emit_int32(imm32);
2587 }
2588 
2589 void Assembler::movl(Address dst, Register src) {
2590   InstructionMark im(this);
2591   prefix(dst, src);
2592   emit_int8((unsigned char)0x89);
2593   emit_operand(src, dst);
2594 }
2595 
2596 // New cpus require to use movsd and movss to avoid partial register stall
2597 // when loading from memory. But for old Opteron use movlpd instead of movsd.
2598 // The selection is done in MacroAssembler::movdbl() and movflt().
2599 void Assembler::movlpd(XMMRegister dst, Address src) {
2600   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2601   InstructionMark im(this);
2602   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2603   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
2604   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2605   emit_int8(0x12);
2606   emit_operand(dst, src);
2607 }
2608 
2609 void Assembler::movq( MMXRegister dst, Address src ) {
2610   assert( VM_Version::supports_mmx(), "" );
2611   emit_int8(0x0F);
2612   emit_int8(0x6F);
2613   emit_operand(dst, src);
2614 }
2615 
2616 void Assembler::movq( Address dst, MMXRegister src ) {
2617   assert( VM_Version::supports_mmx(), "" );
2618   emit_int8(0x0F);
2619   emit_int8(0x7F);
2620   // workaround gcc (3.2.1-7a) bug
2621   // In that version of gcc with only an emit_operand(MMX, Address)
2622   // gcc will tail jump and try and reverse the parameters completely
2623   // obliterating dst in the process. By having a version available
2624   // that doesn't need to swap the args at the tail jump the bug is
2625   // avoided.
2626   emit_operand(dst, src);
2627 }
2628 
2629 void Assembler::movq(XMMRegister dst, Address src) {
2630   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2631   InstructionMark im(this);
2632   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2633   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
2634   simd_prefix(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2635   emit_int8(0x7E);
2636   emit_operand(dst, src);
2637 }
2638 
2639 void Assembler::movq(Address dst, XMMRegister src) {
2640   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2641   InstructionMark im(this);
2642   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2643   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
2644   simd_prefix(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2645   emit_int8((unsigned char)0xD6);
2646   emit_operand(src, dst);
2647 }
2648 
2649 void Assembler::movsbl(Register dst, Address src) { // movsxb
2650   InstructionMark im(this);
2651   prefix(src, dst);
2652   emit_int8(0x0F);
2653   emit_int8((unsigned char)0xBE);
2654   emit_operand(dst, src);
2655 }
2656 
2657 void Assembler::movsbl(Register dst, Register src) { // movsxb
2658   NOT_LP64(assert(src->has_byte_register(), "must have byte register"));
2659   int encode = prefix_and_encode(dst->encoding(), false, src->encoding(), true);
2660   emit_int8(0x0F);
2661   emit_int8((unsigned char)0xBE);
2662   emit_int8((unsigned char)(0xC0 | encode));
2663 }
2664 
2665 void Assembler::movsd(XMMRegister dst, XMMRegister src) {
2666   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2667   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
2668   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2669   emit_int8(0x10);
2670   emit_int8((unsigned char)(0xC0 | encode));
2671 }
2672 
2673 void Assembler::movsd(XMMRegister dst, Address src) {
2674   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2675   InstructionMark im(this);
2676   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
2677   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
2678   simd_prefix(dst, xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2679   emit_int8(0x10);
2680   emit_operand(dst, src);
2681 }
2682 
2683 void Assembler::movsd(Address dst, XMMRegister src) {
2684   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2685   InstructionMark im(this);
2686   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
2687   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
2688   simd_prefix(src, xnoreg, dst, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2689   emit_int8(0x11);
2690   emit_operand(src, dst);
2691 }
2692 
2693 void Assembler::movss(XMMRegister dst, XMMRegister src) {
2694   NOT_LP64(assert(VM_Version::supports_sse(), ""));
2695   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
2696   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2697   emit_int8(0x10);
2698   emit_int8((unsigned char)(0xC0 | encode));
2699 }
2700 
2701 void Assembler::movss(XMMRegister dst, Address src) {
2702   NOT_LP64(assert(VM_Version::supports_sse(), ""));
2703   InstructionMark im(this);
2704   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
2705   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
2706   simd_prefix(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2707   emit_int8(0x10);
2708   emit_operand(dst, src);
2709 }
2710 
2711 void Assembler::movss(Address dst, XMMRegister src) {
2712   NOT_LP64(assert(VM_Version::supports_sse(), ""));
2713   InstructionMark im(this);
2714   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
2715   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
2716   simd_prefix(src, xnoreg, dst, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2717   emit_int8(0x11);
2718   emit_operand(src, dst);
2719 }
2720 
2721 void Assembler::movswl(Register dst, Address src) { // movsxw
2722   InstructionMark im(this);
2723   prefix(src, dst);
2724   emit_int8(0x0F);
2725   emit_int8((unsigned char)0xBF);
2726   emit_operand(dst, src);
2727 }
2728 
2729 void Assembler::movswl(Register dst, Register src) { // movsxw
2730   int encode = prefix_and_encode(dst->encoding(), src->encoding());
2731   emit_int8(0x0F);
2732   emit_int8((unsigned char)0xBF);
2733   emit_int8((unsigned char)(0xC0 | encode));
2734 }
2735 
2736 void Assembler::movw(Address dst, int imm16) {
2737   InstructionMark im(this);
2738 
2739   emit_int8(0x66); // switch to 16-bit mode
2740   prefix(dst);
2741   emit_int8((unsigned char)0xC7);
2742   emit_operand(rax, dst, 2);
2743   emit_int16(imm16);
2744 }
2745 
2746 void Assembler::movw(Register dst, Address src) {
2747   InstructionMark im(this);
2748   emit_int8(0x66);
2749   prefix(src, dst);
2750   emit_int8((unsigned char)0x8B);
2751   emit_operand(dst, src);
2752 }
2753 
2754 void Assembler::movw(Address dst, Register src) {
2755   InstructionMark im(this);
2756   emit_int8(0x66);
2757   prefix(dst, src);
2758   emit_int8((unsigned char)0x89);
2759   emit_operand(src, dst);
2760 }
2761 
2762 void Assembler::movzbl(Register dst, Address src) { // movzxb
2763   InstructionMark im(this);
2764   prefix(src, dst);
2765   emit_int8(0x0F);
2766   emit_int8((unsigned char)0xB6);
2767   emit_operand(dst, src);
2768 }
2769 
2770 void Assembler::movzbl(Register dst, Register src) { // movzxb
2771   NOT_LP64(assert(src->has_byte_register(), "must have byte register"));
2772   int encode = prefix_and_encode(dst->encoding(), false, src->encoding(), true);
2773   emit_int8(0x0F);
2774   emit_int8((unsigned char)0xB6);
2775   emit_int8(0xC0 | encode);
2776 }
2777 
2778 void Assembler::movzwl(Register dst, Address src) { // movzxw
2779   InstructionMark im(this);
2780   prefix(src, dst);
2781   emit_int8(0x0F);
2782   emit_int8((unsigned char)0xB7);
2783   emit_operand(dst, src);
2784 }
2785 
2786 void Assembler::movzwl(Register dst, Register src) { // movzxw
2787   int encode = prefix_and_encode(dst->encoding(), src->encoding());
2788   emit_int8(0x0F);
2789   emit_int8((unsigned char)0xB7);
2790   emit_int8(0xC0 | encode);
2791 }
2792 
2793 void Assembler::mull(Address src) {
2794   InstructionMark im(this);
2795   prefix(src);
2796   emit_int8((unsigned char)0xF7);
2797   emit_operand(rsp, src);
2798 }
2799 
2800 void Assembler::mull(Register src) {
2801   int encode = prefix_and_encode(src->encoding());
2802   emit_int8((unsigned char)0xF7);
2803   emit_int8((unsigned char)(0xE0 | encode));
2804 }
2805 
2806 void Assembler::mulsd(XMMRegister dst, Address src) {
2807   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2808   InstructionMark im(this);
2809   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
2810   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
2811   simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2812   emit_int8(0x59);
2813   emit_operand(dst, src);
2814 }
2815 
2816 void Assembler::mulsd(XMMRegister dst, XMMRegister src) {
2817   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2818   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
2819   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2820   emit_int8(0x59);
2821   emit_int8((unsigned char)(0xC0 | encode));
2822 }
2823 
2824 void Assembler::mulss(XMMRegister dst, Address src) {
2825   NOT_LP64(assert(VM_Version::supports_sse(), ""));
2826   InstructionMark im(this);
2827   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
2828   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
2829   simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2830   emit_int8(0x59);
2831   emit_operand(dst, src);
2832 }
2833 
2834 void Assembler::mulss(XMMRegister dst, XMMRegister src) {
2835   NOT_LP64(assert(VM_Version::supports_sse(), ""));
2836   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
2837   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2838   emit_int8(0x59);
2839   emit_int8((unsigned char)(0xC0 | encode));
2840 }
2841 
2842 void Assembler::negl(Register dst) {
2843   int encode = prefix_and_encode(dst->encoding());
2844   emit_int8((unsigned char)0xF7);
2845   emit_int8((unsigned char)(0xD8 | encode));
2846 }
2847 
2848 void Assembler::nop(int i) {
2849 #ifdef ASSERT
2850   assert(i > 0, " ");
2851   // The fancy nops aren't currently recognized by debuggers making it a
2852   // pain to disassemble code while debugging. If asserts are on clearly
2853   // speed is not an issue so simply use the single byte traditional nop
2854   // to do alignment.
2855 
2856   for (; i > 0 ; i--) emit_int8((unsigned char)0x90);
2857   return;
2858 
2859 #endif // ASSERT
2860 
2861   if (UseAddressNop && VM_Version::is_intel()) {
2862     //
2863     // Using multi-bytes nops "0x0F 0x1F [address]" for Intel
2864     //  1: 0x90
2865     //  2: 0x66 0x90
2866     //  3: 0x66 0x66 0x90 (don't use "0x0F 0x1F 0x00" - need patching safe padding)
2867     //  4: 0x0F 0x1F 0x40 0x00
2868     //  5: 0x0F 0x1F 0x44 0x00 0x00
2869     //  6: 0x66 0x0F 0x1F 0x44 0x00 0x00
2870     //  7: 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00
2871     //  8: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
2872     //  9: 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
2873     // 10: 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
2874     // 11: 0x66 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
2875 
2876     // The rest coding is Intel specific - don't use consecutive address nops
2877 
2878     // 12: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
2879     // 13: 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
2880     // 14: 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
2881     // 15: 0x66 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
2882 
2883     while(i >= 15) {
2884       // For Intel don't generate consecutive addess nops (mix with regular nops)
2885       i -= 15;
2886       emit_int8(0x66);   // size prefix
2887       emit_int8(0x66);   // size prefix
2888       emit_int8(0x66);   // size prefix
2889       addr_nop_8();
2890       emit_int8(0x66);   // size prefix
2891       emit_int8(0x66);   // size prefix
2892       emit_int8(0x66);   // size prefix
2893       emit_int8((unsigned char)0x90);
2894                          // nop
2895     }
2896     switch (i) {
2897       case 14:
2898         emit_int8(0x66); // size prefix
2899       case 13:
2900         emit_int8(0x66); // size prefix
2901       case 12:
2902         addr_nop_8();
2903         emit_int8(0x66); // size prefix
2904         emit_int8(0x66); // size prefix
2905         emit_int8(0x66); // size prefix
2906         emit_int8((unsigned char)0x90);
2907                          // nop
2908         break;
2909       case 11:
2910         emit_int8(0x66); // size prefix
2911       case 10:
2912         emit_int8(0x66); // size prefix
2913       case 9:
2914         emit_int8(0x66); // size prefix
2915       case 8:
2916         addr_nop_8();
2917         break;
2918       case 7:
2919         addr_nop_7();
2920         break;
2921       case 6:
2922         emit_int8(0x66); // size prefix
2923       case 5:
2924         addr_nop_5();
2925         break;
2926       case 4:
2927         addr_nop_4();
2928         break;
2929       case 3:
2930         // Don't use "0x0F 0x1F 0x00" - need patching safe padding
2931         emit_int8(0x66); // size prefix
2932       case 2:
2933         emit_int8(0x66); // size prefix
2934       case 1:
2935         emit_int8((unsigned char)0x90);
2936                          // nop
2937         break;
2938       default:
2939         assert(i == 0, " ");
2940     }
2941     return;
2942   }
2943   if (UseAddressNop && VM_Version::is_amd()) {
2944     //
2945     // Using multi-bytes nops "0x0F 0x1F [address]" for AMD.
2946     //  1: 0x90
2947     //  2: 0x66 0x90
2948     //  3: 0x66 0x66 0x90 (don't use "0x0F 0x1F 0x00" - need patching safe padding)
2949     //  4: 0x0F 0x1F 0x40 0x00
2950     //  5: 0x0F 0x1F 0x44 0x00 0x00
2951     //  6: 0x66 0x0F 0x1F 0x44 0x00 0x00
2952     //  7: 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00
2953     //  8: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
2954     //  9: 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
2955     // 10: 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
2956     // 11: 0x66 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
2957 
2958     // The rest coding is AMD specific - use consecutive address nops
2959 
2960     // 12: 0x66 0x0F 0x1F 0x44 0x00 0x00 0x66 0x0F 0x1F 0x44 0x00 0x00
2961     // 13: 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00 0x66 0x0F 0x1F 0x44 0x00 0x00
2962     // 14: 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00
2963     // 15: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00
2964     // 16: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
2965     //     Size prefixes (0x66) are added for larger sizes
2966 
2967     while(i >= 22) {
2968       i -= 11;
2969       emit_int8(0x66); // size prefix
2970       emit_int8(0x66); // size prefix
2971       emit_int8(0x66); // size prefix
2972       addr_nop_8();
2973     }
2974     // Generate first nop for size between 21-12
2975     switch (i) {
2976       case 21:
2977         i -= 1;
2978         emit_int8(0x66); // size prefix
2979       case 20:
2980       case 19:
2981         i -= 1;
2982         emit_int8(0x66); // size prefix
2983       case 18:
2984       case 17:
2985         i -= 1;
2986         emit_int8(0x66); // size prefix
2987       case 16:
2988       case 15:
2989         i -= 8;
2990         addr_nop_8();
2991         break;
2992       case 14:
2993       case 13:
2994         i -= 7;
2995         addr_nop_7();
2996         break;
2997       case 12:
2998         i -= 6;
2999         emit_int8(0x66); // size prefix
3000         addr_nop_5();
3001         break;
3002       default:
3003         assert(i < 12, " ");
3004     }
3005 
3006     // Generate second nop for size between 11-1
3007     switch (i) {
3008       case 11:
3009         emit_int8(0x66); // size prefix
3010       case 10:
3011         emit_int8(0x66); // size prefix
3012       case 9:
3013         emit_int8(0x66); // size prefix
3014       case 8:
3015         addr_nop_8();
3016         break;
3017       case 7:
3018         addr_nop_7();
3019         break;
3020       case 6:
3021         emit_int8(0x66); // size prefix
3022       case 5:
3023         addr_nop_5();
3024         break;
3025       case 4:
3026         addr_nop_4();
3027         break;
3028       case 3:
3029         // Don't use "0x0F 0x1F 0x00" - need patching safe padding
3030         emit_int8(0x66); // size prefix
3031       case 2:
3032         emit_int8(0x66); // size prefix
3033       case 1:
3034         emit_int8((unsigned char)0x90);
3035                          // nop
3036         break;
3037       default:
3038         assert(i == 0, " ");
3039     }
3040     return;
3041   }
3042 
3043   // Using nops with size prefixes "0x66 0x90".
3044   // From AMD Optimization Guide:
3045   //  1: 0x90
3046   //  2: 0x66 0x90
3047   //  3: 0x66 0x66 0x90
3048   //  4: 0x66 0x66 0x66 0x90
3049   //  5: 0x66 0x66 0x90 0x66 0x90
3050   //  6: 0x66 0x66 0x90 0x66 0x66 0x90
3051   //  7: 0x66 0x66 0x66 0x90 0x66 0x66 0x90
3052   //  8: 0x66 0x66 0x66 0x90 0x66 0x66 0x66 0x90
3053   //  9: 0x66 0x66 0x90 0x66 0x66 0x90 0x66 0x66 0x90
3054   // 10: 0x66 0x66 0x66 0x90 0x66 0x66 0x90 0x66 0x66 0x90
3055   //
3056   while(i > 12) {
3057     i -= 4;
3058     emit_int8(0x66); // size prefix
3059     emit_int8(0x66);
3060     emit_int8(0x66);
3061     emit_int8((unsigned char)0x90);
3062                      // nop
3063   }
3064   // 1 - 12 nops
3065   if(i > 8) {
3066     if(i > 9) {
3067       i -= 1;
3068       emit_int8(0x66);
3069     }
3070     i -= 3;
3071     emit_int8(0x66);
3072     emit_int8(0x66);
3073     emit_int8((unsigned char)0x90);
3074   }
3075   // 1 - 8 nops
3076   if(i > 4) {
3077     if(i > 6) {
3078       i -= 1;
3079       emit_int8(0x66);
3080     }
3081     i -= 3;
3082     emit_int8(0x66);
3083     emit_int8(0x66);
3084     emit_int8((unsigned char)0x90);
3085   }
3086   switch (i) {
3087     case 4:
3088       emit_int8(0x66);
3089     case 3:
3090       emit_int8(0x66);
3091     case 2:
3092       emit_int8(0x66);
3093     case 1:
3094       emit_int8((unsigned char)0x90);
3095       break;
3096     default:
3097       assert(i == 0, " ");
3098   }
3099 }
3100 
3101 void Assembler::notl(Register dst) {
3102   int encode = prefix_and_encode(dst->encoding());
3103   emit_int8((unsigned char)0xF7);
3104   emit_int8((unsigned char)(0xD0 | encode));
3105 }
3106 
3107 void Assembler::orl(Address dst, int32_t imm32) {
3108   InstructionMark im(this);
3109   prefix(dst);
3110   emit_arith_operand(0x81, rcx, dst, imm32);
3111 }
3112 
3113 void Assembler::orl(Register dst, int32_t imm32) {
3114   prefix(dst);
3115   emit_arith(0x81, 0xC8, dst, imm32);
3116 }
3117 
3118 void Assembler::orl(Register dst, Address src) {
3119   InstructionMark im(this);
3120   prefix(src, dst);
3121   emit_int8(0x0B);
3122   emit_operand(dst, src);
3123 }
3124 
3125 void Assembler::orl(Register dst, Register src) {
3126   (void) prefix_and_encode(dst->encoding(), src->encoding());
3127   emit_arith(0x0B, 0xC0, dst, src);
3128 }
3129 
3130 void Assembler::orl(Address dst, Register src) {
3131   InstructionMark im(this);
3132   prefix(dst, src);
3133   emit_int8(0x09);
3134   emit_operand(src, dst);
3135 }
3136 
3137 void Assembler::packuswb(XMMRegister dst, Address src) {
3138   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3139   assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
3140   InstructionMark im(this);
3141   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
3142   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
3143   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3144   emit_int8(0x67);
3145   emit_operand(dst, src);
3146 }
3147 
3148 void Assembler::packuswb(XMMRegister dst, XMMRegister src) {
3149   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3150   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
3151   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3152   emit_int8(0x67);
3153   emit_int8((unsigned char)(0xC0 | encode));
3154 }
3155 
3156 void Assembler::vpackuswb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
3157   assert(UseAVX > 0, "some form of AVX must be enabled");
3158   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
3159   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3160   emit_int8(0x67);
3161   emit_int8((unsigned char)(0xC0 | encode));
3162 }
3163 
3164 void Assembler::vpermq(XMMRegister dst, XMMRegister src, int imm8, int vector_len) {
3165   assert(VM_Version::supports_avx2(), "");
3166   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
3167   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3168   emit_int8(0x00);
3169   emit_int8(0xC0 | encode);
3170   emit_int8(imm8);
3171 }
3172 
3173 void Assembler::pause() {
3174   emit_int8((unsigned char)0xF3);
3175   emit_int8((unsigned char)0x90);
3176 }
3177 
3178 void Assembler::pcmpestri(XMMRegister dst, Address src, int imm8) {
3179   assert(VM_Version::supports_sse4_2(), "");
3180   InstructionMark im(this);
3181   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3182   simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3183   emit_int8(0x61);
3184   emit_operand(dst, src);
3185   emit_int8(imm8);
3186 }
3187 
3188 void Assembler::pcmpestri(XMMRegister dst, XMMRegister src, int imm8) {
3189   assert(VM_Version::supports_sse4_2(), "");
3190   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3191   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3192   emit_int8(0x61);
3193   emit_int8((unsigned char)(0xC0 | encode));
3194   emit_int8(imm8);
3195 }
3196 
3197 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
3198 void Assembler::pcmpeqb(XMMRegister dst, XMMRegister src) {
3199   assert(VM_Version::supports_sse2(), "");
3200   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3201   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3202   emit_int8(0x74);
3203   emit_int8((unsigned char)(0xC0 | encode));
3204 }
3205 
3206 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
3207 void Assembler::vpcmpeqb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
3208   assert(VM_Version::supports_avx(), "");
3209   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3210   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3211   emit_int8(0x74);
3212   emit_int8((unsigned char)(0xC0 | encode));
3213 }
3214 
3215 // In this context, kdst is written the mask used to process the equal components
3216 void Assembler::evpcmpeqb(KRegister kdst, XMMRegister nds, XMMRegister src, int vector_len) {
3217   assert(VM_Version::supports_avx512bw(), "");
3218   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
3219   attributes.set_is_evex_instruction();
3220   int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3221   emit_int8(0x74);
3222   emit_int8((unsigned char)(0xC0 | encode));
3223 }
3224 
3225 void Assembler::evpcmpeqb(KRegister kdst, XMMRegister nds, Address src, int vector_len) {
3226   assert(VM_Version::supports_avx512bw(), "");
3227   InstructionMark im(this);
3228   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
3229   attributes.set_is_evex_instruction();
3230   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
3231   int dst_enc = kdst->encoding();
3232   vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3233   emit_int8(0x74);
3234   emit_operand(as_Register(dst_enc), src);
3235 }
3236 
3237 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
3238 void Assembler::pcmpeqw(XMMRegister dst, XMMRegister src) {
3239   assert(VM_Version::supports_sse2(), "");
3240   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3241   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3242   emit_int8(0x75);
3243   emit_int8((unsigned char)(0xC0 | encode));
3244 }
3245 
3246 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
3247 void Assembler::vpcmpeqw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
3248   assert(VM_Version::supports_avx(), "");
3249   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3250   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3251   emit_int8(0x75);
3252   emit_int8((unsigned char)(0xC0 | encode));
3253 }
3254 
3255 // In this context, kdst is written the mask used to process the equal components
3256 void Assembler::evpcmpeqw(KRegister kdst, XMMRegister nds, XMMRegister src, int vector_len) {
3257   assert(VM_Version::supports_avx512bw(), "");
3258   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
3259   attributes.set_is_evex_instruction();
3260   int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3261   emit_int8(0x75);
3262   emit_int8((unsigned char)(0xC0 | encode));
3263 }
3264 
3265 void Assembler::evpcmpeqw(KRegister kdst, XMMRegister nds, Address src, int vector_len) {
3266   assert(VM_Version::supports_avx512bw(), "");
3267   InstructionMark im(this);
3268   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
3269   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
3270   attributes.set_is_evex_instruction();
3271   int dst_enc = kdst->encoding();
3272   vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3273   emit_int8(0x75);
3274   emit_operand(as_Register(dst_enc), src);
3275 }
3276 
3277 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
3278 void Assembler::pcmpeqd(XMMRegister dst, XMMRegister src) {
3279   assert(VM_Version::supports_sse2(), "");
3280   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3281   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3282   emit_int8(0x76);
3283   emit_int8((unsigned char)(0xC0 | encode));
3284 }
3285 
3286 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
3287 void Assembler::vpcmpeqd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
3288   assert(VM_Version::supports_avx(), "");
3289   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3290   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3291   emit_int8(0x76);
3292   emit_int8((unsigned char)(0xC0 | encode));
3293 }
3294 
3295 // In this context, kdst is written the mask used to process the equal components
3296 void Assembler::evpcmpeqd(KRegister kdst, XMMRegister nds, XMMRegister src, int vector_len) {
3297   assert(VM_Version::supports_evex(), "");
3298   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
3299   attributes.set_is_evex_instruction();
3300   int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3301   emit_int8(0x76);
3302   emit_int8((unsigned char)(0xC0 | encode));
3303 }
3304 
3305 void Assembler::evpcmpeqd(KRegister kdst, XMMRegister nds, Address src, int vector_len) {
3306   assert(VM_Version::supports_evex(), "");
3307   InstructionMark im(this);
3308   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
3309   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
3310   attributes.set_is_evex_instruction();
3311   int dst_enc = kdst->encoding();
3312   vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3313   emit_int8(0x76);
3314   emit_operand(as_Register(dst_enc), src);
3315 }
3316 
3317 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
3318 void Assembler::pcmpeqq(XMMRegister dst, XMMRegister src) {
3319   assert(VM_Version::supports_sse4_1(), "");
3320   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3321   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3322   emit_int8(0x29);
3323   emit_int8((unsigned char)(0xC0 | encode));
3324 }
3325 
3326 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
3327 void Assembler::vpcmpeqq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
3328   assert(VM_Version::supports_avx(), "");
3329   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3330   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3331   emit_int8(0x29);
3332   emit_int8((unsigned char)(0xC0 | encode));
3333 }
3334 
3335 // In this context, kdst is written the mask used to process the equal components
3336 void Assembler::evpcmpeqq(KRegister kdst, XMMRegister nds, XMMRegister src, int vector_len) {
3337   assert(VM_Version::supports_evex(), "");
3338   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
3339   attributes.set_is_evex_instruction();
3340   int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3341   emit_int8(0x29);
3342   emit_int8((unsigned char)(0xC0 | encode));
3343 }
3344 
3345 // In this context, kdst is written the mask used to process the equal components
3346 void Assembler::evpcmpeqq(KRegister kdst, XMMRegister nds, Address src, int vector_len) {
3347   assert(VM_Version::supports_evex(), "");
3348   InstructionMark im(this);
3349   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
3350   attributes.set_is_evex_instruction();
3351   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
3352   int dst_enc = kdst->encoding();
3353   vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3354   emit_int8(0x29);
3355   emit_operand(as_Register(dst_enc), src);
3356 }
3357 
3358 void Assembler::pmovmskb(Register dst, XMMRegister src) {
3359   assert(VM_Version::supports_sse2(), "");
3360   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3361   int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3362   emit_int8((unsigned char)0xD7);
3363   emit_int8((unsigned char)(0xC0 | encode));
3364 }
3365 
3366 void Assembler::vpmovmskb(Register dst, XMMRegister src) {
3367   assert(VM_Version::supports_avx2(), "");
3368   InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3369   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3370   emit_int8((unsigned char)0xD7);
3371   emit_int8((unsigned char)(0xC0 | encode));
3372 }
3373 
3374 void Assembler::pextrd(Register dst, XMMRegister src, int imm8) {
3375   assert(VM_Version::supports_sse4_1(), "");
3376   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
3377   int encode = simd_prefix_and_encode(src, xnoreg, as_XMMRegister(dst->encoding()), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3378   emit_int8(0x16);
3379   emit_int8((unsigned char)(0xC0 | encode));
3380   emit_int8(imm8);
3381 }
3382 
3383 void Assembler::pextrd(Address dst, XMMRegister src, int imm8) {
3384   assert(VM_Version::supports_sse4_1(), "");
3385   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
3386   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
3387   simd_prefix(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3388   emit_int8(0x16);
3389   emit_operand(src, dst);
3390   emit_int8(imm8);
3391 }
3392 
3393 void Assembler::pextrq(Register dst, XMMRegister src, int imm8) {
3394   assert(VM_Version::supports_sse4_1(), "");
3395   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
3396   int encode = simd_prefix_and_encode(src, xnoreg, as_XMMRegister(dst->encoding()), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3397   emit_int8(0x16);
3398   emit_int8((unsigned char)(0xC0 | encode));
3399   emit_int8(imm8);
3400 }
3401 
3402 void Assembler::pextrq(Address dst, XMMRegister src, int imm8) {
3403   assert(VM_Version::supports_sse4_1(), "");
3404   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
3405   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
3406   simd_prefix(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3407   emit_int8(0x16);
3408   emit_operand(src, dst);
3409   emit_int8(imm8);
3410 }
3411 
3412 void Assembler::pextrw(Register dst, XMMRegister src, int imm8) {
3413   assert(VM_Version::supports_sse2(), "");
3414   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
3415   int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3416   emit_int8((unsigned char)0xC5);
3417   emit_int8((unsigned char)(0xC0 | encode));
3418   emit_int8(imm8);
3419 }
3420 
3421 void Assembler::pextrw(Address dst, XMMRegister src, int imm8) {
3422   assert(VM_Version::supports_sse4_1(), "");
3423   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
3424   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_16bit);
3425   simd_prefix(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3426   emit_int8((unsigned char)0x15);
3427   emit_operand(src, dst);
3428   emit_int8(imm8);
3429 }
3430 
3431 void Assembler::pextrb(Address dst, XMMRegister src, int imm8) {
3432   assert(VM_Version::supports_sse4_1(), "");
3433   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
3434   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_8bit);
3435   simd_prefix(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3436   emit_int8(0x14);
3437   emit_operand(src, dst);
3438   emit_int8(imm8);
3439 }
3440 
3441 void Assembler::pinsrd(XMMRegister dst, Register src, int imm8) {
3442   assert(VM_Version::supports_sse4_1(), "");
3443   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
3444   int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3445   emit_int8(0x22);
3446   emit_int8((unsigned char)(0xC0 | encode));
3447   emit_int8(imm8);
3448 }
3449 
3450 void Assembler::pinsrd(XMMRegister dst, Address src, int imm8) {
3451   assert(VM_Version::supports_sse4_1(), "");
3452   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
3453   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
3454   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3455   emit_int8(0x22);
3456   emit_operand(dst,src);
3457   emit_int8(imm8);
3458 }
3459 
3460 void Assembler::pinsrq(XMMRegister dst, Register src, int imm8) {
3461   assert(VM_Version::supports_sse4_1(), "");
3462   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
3463   int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3464   emit_int8(0x22);
3465   emit_int8((unsigned char)(0xC0 | encode));
3466   emit_int8(imm8);
3467 }
3468 
3469 void Assembler::pinsrq(XMMRegister dst, Address src, int imm8) {
3470   assert(VM_Version::supports_sse4_1(), "");
3471   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
3472   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
3473   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3474   emit_int8(0x22);
3475   emit_operand(dst, src);
3476   emit_int8(imm8);
3477 }
3478 
3479 void Assembler::pinsrw(XMMRegister dst, Register src, int imm8) {
3480   assert(VM_Version::supports_sse2(), "");
3481   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
3482   int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3483   emit_int8((unsigned char)0xC4);
3484   emit_int8((unsigned char)(0xC0 | encode));
3485   emit_int8(imm8);
3486 }
3487 
3488 void Assembler::pinsrw(XMMRegister dst, Address src, int imm8) {
3489   assert(VM_Version::supports_sse2(), "");
3490   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
3491   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_16bit);
3492   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3493   emit_int8((unsigned char)0xC4);
3494   emit_operand(dst, src);
3495   emit_int8(imm8);
3496 }
3497 
3498 void Assembler::pinsrb(XMMRegister dst, Address src, int imm8) {
3499   assert(VM_Version::supports_sse4_1(), "");
3500   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
3501   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_8bit);
3502   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3503   emit_int8(0x20);
3504   emit_operand(dst, src);
3505   emit_int8(imm8);
3506 }
3507 
3508 void Assembler::pmovzxbw(XMMRegister dst, Address src) {
3509   assert(VM_Version::supports_sse4_1(), "");
3510   InstructionMark im(this);
3511   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
3512   attributes.set_address_attributes(/* tuple_type */ EVEX_HVM, /* input_size_in_bits */ EVEX_NObit);
3513   simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3514   emit_int8(0x30);
3515   emit_operand(dst, src);
3516 }
3517 
3518 void Assembler::pmovzxbw(XMMRegister dst, XMMRegister src) {
3519   assert(VM_Version::supports_sse4_1(), "");
3520   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
3521   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3522   emit_int8(0x30);
3523   emit_int8((unsigned char)(0xC0 | encode));
3524 }
3525 
3526 void Assembler::vpmovzxbw(XMMRegister dst, Address src, int vector_len) {
3527   assert(VM_Version::supports_avx(), "");
3528   InstructionMark im(this);
3529   assert(dst != xnoreg, "sanity");
3530   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
3531   attributes.set_address_attributes(/* tuple_type */ EVEX_HVM, /* input_size_in_bits */ EVEX_NObit);
3532   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3533   emit_int8(0x30);
3534   emit_operand(dst, src);
3535 }
3536 
3537 // generic
3538 void Assembler::pop(Register dst) {
3539   int encode = prefix_and_encode(dst->encoding());
3540   emit_int8(0x58 | encode);
3541 }
3542 
3543 void Assembler::popcntl(Register dst, Address src) {
3544   assert(VM_Version::supports_popcnt(), "must support");
3545   InstructionMark im(this);
3546   emit_int8((unsigned char)0xF3);
3547   prefix(src, dst);
3548   emit_int8(0x0F);
3549   emit_int8((unsigned char)0xB8);
3550   emit_operand(dst, src);
3551 }
3552 
3553 void Assembler::popcntl(Register dst, Register src) {
3554   assert(VM_Version::supports_popcnt(), "must support");
3555   emit_int8((unsigned char)0xF3);
3556   int encode = prefix_and_encode(dst->encoding(), src->encoding());
3557   emit_int8(0x0F);
3558   emit_int8((unsigned char)0xB8);
3559   emit_int8((unsigned char)(0xC0 | encode));
3560 }
3561 
3562 void Assembler::popf() {
3563   emit_int8((unsigned char)0x9D);
3564 }
3565 
3566 #ifndef _LP64 // no 32bit push/pop on amd64
3567 void Assembler::popl(Address dst) {
3568   // NOTE: this will adjust stack by 8byte on 64bits
3569   InstructionMark im(this);
3570   prefix(dst);
3571   emit_int8((unsigned char)0x8F);
3572   emit_operand(rax, dst);
3573 }
3574 #endif
3575 
3576 void Assembler::prefetch_prefix(Address src) {
3577   prefix(src);
3578   emit_int8(0x0F);
3579 }
3580 
3581 void Assembler::prefetchnta(Address src) {
3582   NOT_LP64(assert(VM_Version::supports_sse(), "must support"));
3583   InstructionMark im(this);
3584   prefetch_prefix(src);
3585   emit_int8(0x18);
3586   emit_operand(rax, src); // 0, src
3587 }
3588 
3589 void Assembler::prefetchr(Address src) {
3590   assert(VM_Version::supports_3dnow_prefetch(), "must support");
3591   InstructionMark im(this);
3592   prefetch_prefix(src);
3593   emit_int8(0x0D);
3594   emit_operand(rax, src); // 0, src
3595 }
3596 
3597 void Assembler::prefetcht0(Address src) {
3598   NOT_LP64(assert(VM_Version::supports_sse(), "must support"));
3599   InstructionMark im(this);
3600   prefetch_prefix(src);
3601   emit_int8(0x18);
3602   emit_operand(rcx, src); // 1, src
3603 }
3604 
3605 void Assembler::prefetcht1(Address src) {
3606   NOT_LP64(assert(VM_Version::supports_sse(), "must support"));
3607   InstructionMark im(this);
3608   prefetch_prefix(src);
3609   emit_int8(0x18);
3610   emit_operand(rdx, src); // 2, src
3611 }
3612 
3613 void Assembler::prefetcht2(Address src) {
3614   NOT_LP64(assert(VM_Version::supports_sse(), "must support"));
3615   InstructionMark im(this);
3616   prefetch_prefix(src);
3617   emit_int8(0x18);
3618   emit_operand(rbx, src); // 3, src
3619 }
3620 
3621 void Assembler::prefetchw(Address src) {
3622   assert(VM_Version::supports_3dnow_prefetch(), "must support");
3623   InstructionMark im(this);
3624   prefetch_prefix(src);
3625   emit_int8(0x0D);
3626   emit_operand(rcx, src); // 1, src
3627 }
3628 
3629 void Assembler::prefix(Prefix p) {
3630   emit_int8(p);
3631 }
3632 
3633 void Assembler::pshufb(XMMRegister dst, XMMRegister src) {
3634   assert(VM_Version::supports_ssse3(), "");
3635   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
3636   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3637   emit_int8(0x00);
3638   emit_int8((unsigned char)(0xC0 | encode));
3639 }
3640 
3641 void Assembler::pshufb(XMMRegister dst, Address src) {
3642   assert(VM_Version::supports_ssse3(), "");
3643   InstructionMark im(this);
3644   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
3645   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
3646   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3647   emit_int8(0x00);
3648   emit_operand(dst, src);
3649 }
3650 
3651 void Assembler::pshufd(XMMRegister dst, XMMRegister src, int mode) {
3652   assert(isByte(mode), "invalid value");
3653   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3654   int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_128bit;
3655   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
3656   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3657   emit_int8(0x70);
3658   emit_int8((unsigned char)(0xC0 | encode));
3659   emit_int8(mode & 0xFF);
3660 }
3661 
3662 void Assembler::pshufd(XMMRegister dst, Address src, int mode) {
3663   assert(isByte(mode), "invalid value");
3664   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3665   assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
3666   InstructionMark im(this);
3667   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
3668   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
3669   simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3670   emit_int8(0x70);
3671   emit_operand(dst, src);
3672   emit_int8(mode & 0xFF);
3673 }
3674 
3675 void Assembler::pshuflw(XMMRegister dst, XMMRegister src, int mode) {
3676   assert(isByte(mode), "invalid value");
3677   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3678   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
3679   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
3680   emit_int8(0x70);
3681   emit_int8((unsigned char)(0xC0 | encode));
3682   emit_int8(mode & 0xFF);
3683 }
3684 
3685 void Assembler::pshuflw(XMMRegister dst, Address src, int mode) {
3686   assert(isByte(mode), "invalid value");
3687   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3688   assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
3689   InstructionMark im(this);
3690   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
3691   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
3692   simd_prefix(dst, xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
3693   emit_int8(0x70);
3694   emit_operand(dst, src);
3695   emit_int8(mode & 0xFF);
3696 }
3697 
3698 void Assembler::psrldq(XMMRegister dst, int shift) {
3699   // Shift left 128 bit value in dst XMMRegister by shift number of bytes.
3700   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3701   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
3702   // XMM3 is for /3 encoding: 66 0F 73 /3 ib
3703   int encode = simd_prefix_and_encode(xmm3, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3704   emit_int8(0x73);
3705   emit_int8((unsigned char)(0xC0 | encode));
3706   emit_int8(shift);
3707 }
3708 
3709 void Assembler::pslldq(XMMRegister dst, int shift) {
3710   // Shift left 128 bit value in dst XMMRegister by shift number of bytes.
3711   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3712   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
3713   // XMM7 is for /7 encoding: 66 0F 73 /7 ib
3714   int encode = simd_prefix_and_encode(xmm7, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3715   emit_int8(0x73);
3716   emit_int8((unsigned char)(0xC0 | encode));
3717   emit_int8(shift);
3718 }
3719 
3720 void Assembler::ptest(XMMRegister dst, Address src) {
3721   assert(VM_Version::supports_sse4_1(), "");
3722   assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
3723   InstructionMark im(this);
3724   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3725   simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3726   emit_int8(0x17);
3727   emit_operand(dst, src);
3728 }
3729 
3730 void Assembler::ptest(XMMRegister dst, XMMRegister src) {
3731   assert(VM_Version::supports_sse4_1(), "");
3732   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3733   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3734   emit_int8(0x17);
3735   emit_int8((unsigned char)(0xC0 | encode));
3736 }
3737 
3738 void Assembler::vptest(XMMRegister dst, Address src) {
3739   assert(VM_Version::supports_avx(), "");
3740   InstructionMark im(this);
3741   InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3742   assert(dst != xnoreg, "sanity");
3743   // swap src<->dst for encoding
3744   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3745   emit_int8(0x17);
3746   emit_operand(dst, src);
3747 }
3748 
3749 void Assembler::vptest(XMMRegister dst, XMMRegister src) {
3750   assert(VM_Version::supports_avx(), "");
3751   InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3752   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3753   emit_int8(0x17);
3754   emit_int8((unsigned char)(0xC0 | encode));
3755 }
3756 
3757 void Assembler::punpcklbw(XMMRegister dst, Address src) {
3758   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3759   assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
3760   InstructionMark im(this);
3761   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_vlbw, /* no_mask_reg */ false, /* uses_vl */ true);
3762   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
3763   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3764   emit_int8(0x60);
3765   emit_operand(dst, src);
3766 }
3767 
3768 void Assembler::punpcklbw(XMMRegister dst, XMMRegister src) {
3769   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3770   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_vlbw, /* no_mask_reg */ false, /* uses_vl */ true);
3771   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3772   emit_int8(0x60);
3773   emit_int8((unsigned char)(0xC0 | encode));
3774 }
3775 
3776 void Assembler::punpckldq(XMMRegister dst, Address src) {
3777   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3778   assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
3779   InstructionMark im(this);
3780   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
3781   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
3782   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3783   emit_int8(0x62);
3784   emit_operand(dst, src);
3785 }
3786 
3787 void Assembler::punpckldq(XMMRegister dst, XMMRegister src) {
3788   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3789   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
3790   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3791   emit_int8(0x62);
3792   emit_int8((unsigned char)(0xC0 | encode));
3793 }
3794 
3795 void Assembler::punpcklqdq(XMMRegister dst, XMMRegister src) {
3796   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3797   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
3798   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3799   emit_int8(0x6C);
3800   emit_int8((unsigned char)(0xC0 | encode));
3801 }
3802 
3803 void Assembler::push(int32_t imm32) {
3804   // in 64bits we push 64bits onto the stack but only
3805   // take a 32bit immediate
3806   emit_int8(0x68);
3807   emit_int32(imm32);
3808 }
3809 
3810 void Assembler::push(Register src) {
3811   int encode = prefix_and_encode(src->encoding());
3812 
3813   emit_int8(0x50 | encode);
3814 }
3815 
3816 void Assembler::pushf() {
3817   emit_int8((unsigned char)0x9C);
3818 }
3819 
3820 #ifndef _LP64 // no 32bit push/pop on amd64
3821 void Assembler::pushl(Address src) {
3822   // Note this will push 64bit on 64bit
3823   InstructionMark im(this);
3824   prefix(src);
3825   emit_int8((unsigned char)0xFF);
3826   emit_operand(rsi, src);
3827 }
3828 #endif
3829 
3830 void Assembler::rcll(Register dst, int imm8) {
3831   assert(isShiftCount(imm8), "illegal shift count");
3832   int encode = prefix_and_encode(dst->encoding());
3833   if (imm8 == 1) {
3834     emit_int8((unsigned char)0xD1);
3835     emit_int8((unsigned char)(0xD0 | encode));
3836   } else {
3837     emit_int8((unsigned char)0xC1);
3838     emit_int8((unsigned char)0xD0 | encode);
3839     emit_int8(imm8);
3840   }
3841 }
3842 
3843 void Assembler::rcpps(XMMRegister dst, XMMRegister src) {
3844   NOT_LP64(assert(VM_Version::supports_sse(), ""));
3845   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3846   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
3847   emit_int8(0x53);
3848   emit_int8((unsigned char)(0xC0 | encode));
3849 }
3850 
3851 void Assembler::rcpss(XMMRegister dst, XMMRegister src) {
3852   NOT_LP64(assert(VM_Version::supports_sse(), ""));
3853   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3854   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
3855   emit_int8(0x53);
3856   emit_int8((unsigned char)(0xC0 | encode));
3857 }
3858 
3859 void Assembler::rdtsc() {
3860   emit_int8((unsigned char)0x0F);
3861   emit_int8((unsigned char)0x31);
3862 }
3863 
3864 // copies data from [esi] to [edi] using rcx pointer sized words
3865 // generic
3866 void Assembler::rep_mov() {
3867   emit_int8((unsigned char)0xF3);
3868   // MOVSQ
3869   LP64_ONLY(prefix(REX_W));
3870   emit_int8((unsigned char)0xA5);
3871 }
3872 
3873 // sets rcx bytes with rax, value at [edi]
3874 void Assembler::rep_stosb() {
3875   emit_int8((unsigned char)0xF3); // REP
3876   LP64_ONLY(prefix(REX_W));
3877   emit_int8((unsigned char)0xAA); // STOSB
3878 }
3879 
3880 // sets rcx pointer sized words with rax, value at [edi]
3881 // generic
3882 void Assembler::rep_stos() {
3883   emit_int8((unsigned char)0xF3); // REP
3884   LP64_ONLY(prefix(REX_W));       // LP64:STOSQ, LP32:STOSD
3885   emit_int8((unsigned char)0xAB);
3886 }
3887 
3888 // scans rcx pointer sized words at [edi] for occurance of rax,
3889 // generic
3890 void Assembler::repne_scan() { // repne_scan
3891   emit_int8((unsigned char)0xF2);
3892   // SCASQ
3893   LP64_ONLY(prefix(REX_W));
3894   emit_int8((unsigned char)0xAF);
3895 }
3896 
3897 #ifdef _LP64
3898 // scans rcx 4 byte words at [edi] for occurance of rax,
3899 // generic
3900 void Assembler::repne_scanl() { // repne_scan
3901   emit_int8((unsigned char)0xF2);
3902   // SCASL
3903   emit_int8((unsigned char)0xAF);
3904 }
3905 #endif
3906 
3907 void Assembler::ret(int imm16) {
3908   if (imm16 == 0) {
3909     emit_int8((unsigned char)0xC3);
3910   } else {
3911     emit_int8((unsigned char)0xC2);
3912     emit_int16(imm16);
3913   }
3914 }
3915 
3916 void Assembler::sahf() {
3917 #ifdef _LP64
3918   // Not supported in 64bit mode
3919   ShouldNotReachHere();
3920 #endif
3921   emit_int8((unsigned char)0x9E);
3922 }
3923 
3924 void Assembler::sarl(Register dst, int imm8) {
3925   int encode = prefix_and_encode(dst->encoding());
3926   assert(isShiftCount(imm8), "illegal shift count");
3927   if (imm8 == 1) {
3928     emit_int8((unsigned char)0xD1);
3929     emit_int8((unsigned char)(0xF8 | encode));
3930   } else {
3931     emit_int8((unsigned char)0xC1);
3932     emit_int8((unsigned char)(0xF8 | encode));
3933     emit_int8(imm8);
3934   }
3935 }
3936 
3937 void Assembler::sarl(Register dst) {
3938   int encode = prefix_and_encode(dst->encoding());
3939   emit_int8((unsigned char)0xD3);
3940   emit_int8((unsigned char)(0xF8 | encode));
3941 }
3942 
3943 void Assembler::sbbl(Address dst, int32_t imm32) {
3944   InstructionMark im(this);
3945   prefix(dst);
3946   emit_arith_operand(0x81, rbx, dst, imm32);
3947 }
3948 
3949 void Assembler::sbbl(Register dst, int32_t imm32) {
3950   prefix(dst);
3951   emit_arith(0x81, 0xD8, dst, imm32);
3952 }
3953 
3954 
3955 void Assembler::sbbl(Register dst, Address src) {
3956   InstructionMark im(this);
3957   prefix(src, dst);
3958   emit_int8(0x1B);
3959   emit_operand(dst, src);
3960 }
3961 
3962 void Assembler::sbbl(Register dst, Register src) {
3963   (void) prefix_and_encode(dst->encoding(), src->encoding());
3964   emit_arith(0x1B, 0xC0, dst, src);
3965 }
3966 
3967 void Assembler::setb(Condition cc, Register dst) {
3968   assert(0 <= cc && cc < 16, "illegal cc");
3969   int encode = prefix_and_encode(dst->encoding(), true);
3970   emit_int8(0x0F);
3971   emit_int8((unsigned char)0x90 | cc);
3972   emit_int8((unsigned char)(0xC0 | encode));
3973 }
3974 
3975 void Assembler::palignr(XMMRegister dst, XMMRegister src, int imm8) {
3976   assert(VM_Version::supports_ssse3(), "");
3977   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ false);
3978   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3979   emit_int8((unsigned char)0x0F);
3980   emit_int8((unsigned char)(0xC0 | encode));
3981   emit_int8(imm8);
3982 }
3983 
3984 void Assembler::pblendw(XMMRegister dst, XMMRegister src, int imm8) {
3985   assert(VM_Version::supports_sse4_1(), "");
3986   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3987   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3988   emit_int8((unsigned char)0x0E);
3989   emit_int8((unsigned char)(0xC0 | encode));
3990   emit_int8(imm8);
3991 }
3992 
3993 void Assembler::sha1rnds4(XMMRegister dst, XMMRegister src, int imm8) {
3994   assert(VM_Version::supports_sha(), "");
3995   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3996   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F_3A, &attributes);
3997   emit_int8((unsigned char)0xCC);
3998   emit_int8((unsigned char)(0xC0 | encode));
3999   emit_int8((unsigned char)imm8);
4000 }
4001 
4002 void Assembler::sha1nexte(XMMRegister dst, XMMRegister src) {
4003   assert(VM_Version::supports_sha(), "");
4004   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
4005   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
4006   emit_int8((unsigned char)0xC8);
4007   emit_int8((unsigned char)(0xC0 | encode));
4008 }
4009 
4010 void Assembler::sha1msg1(XMMRegister dst, XMMRegister src) {
4011   assert(VM_Version::supports_sha(), "");
4012   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
4013   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
4014   emit_int8((unsigned char)0xC9);
4015   emit_int8((unsigned char)(0xC0 | encode));
4016 }
4017 
4018 void Assembler::sha1msg2(XMMRegister dst, XMMRegister src) {
4019   assert(VM_Version::supports_sha(), "");
4020   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
4021   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
4022   emit_int8((unsigned char)0xCA);
4023   emit_int8((unsigned char)(0xC0 | encode));
4024 }
4025 
4026 // xmm0 is implicit additional source to this instruction.
4027 void Assembler::sha256rnds2(XMMRegister dst, XMMRegister src) {
4028   assert(VM_Version::supports_sha(), "");
4029   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
4030   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
4031   emit_int8((unsigned char)0xCB);
4032   emit_int8((unsigned char)(0xC0 | encode));
4033 }
4034 
4035 void Assembler::sha256msg1(XMMRegister dst, XMMRegister src) {
4036   assert(VM_Version::supports_sha(), "");
4037   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
4038   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
4039   emit_int8((unsigned char)0xCC);
4040   emit_int8((unsigned char)(0xC0 | encode));
4041 }
4042 
4043 void Assembler::sha256msg2(XMMRegister dst, XMMRegister src) {
4044   assert(VM_Version::supports_sha(), "");
4045   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
4046   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
4047   emit_int8((unsigned char)0xCD);
4048   emit_int8((unsigned char)(0xC0 | encode));
4049 }
4050 
4051 
4052 void Assembler::shll(Register dst, int imm8) {
4053   assert(isShiftCount(imm8), "illegal shift count");
4054   int encode = prefix_and_encode(dst->encoding());
4055   if (imm8 == 1 ) {
4056     emit_int8((unsigned char)0xD1);
4057     emit_int8((unsigned char)(0xE0 | encode));
4058   } else {
4059     emit_int8((unsigned char)0xC1);
4060     emit_int8((unsigned char)(0xE0 | encode));
4061     emit_int8(imm8);
4062   }
4063 }
4064 
4065 void Assembler::shll(Register dst) {
4066   int encode = prefix_and_encode(dst->encoding());
4067   emit_int8((unsigned char)0xD3);
4068   emit_int8((unsigned char)(0xE0 | encode));
4069 }
4070 
4071 void Assembler::shrl(Register dst, int imm8) {
4072   assert(isShiftCount(imm8), "illegal shift count");
4073   int encode = prefix_and_encode(dst->encoding());
4074   emit_int8((unsigned char)0xC1);
4075   emit_int8((unsigned char)(0xE8 | encode));
4076   emit_int8(imm8);
4077 }
4078 
4079 void Assembler::shrl(Register dst) {
4080   int encode = prefix_and_encode(dst->encoding());
4081   emit_int8((unsigned char)0xD3);
4082   emit_int8((unsigned char)(0xE8 | encode));
4083 }
4084 
4085 // copies a single word from [esi] to [edi]
4086 void Assembler::smovl() {
4087   emit_int8((unsigned char)0xA5);
4088 }
4089 
4090 void Assembler::sqrtsd(XMMRegister dst, XMMRegister src) {
4091   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4092   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
4093   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
4094   emit_int8(0x51);
4095   emit_int8((unsigned char)(0xC0 | encode));
4096 }
4097 
4098 void Assembler::sqrtsd(XMMRegister dst, Address src) {
4099   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4100   InstructionMark im(this);
4101   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
4102   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
4103   simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
4104   emit_int8(0x51);
4105   emit_operand(dst, src);
4106 }
4107 
4108 void Assembler::sqrtss(XMMRegister dst, XMMRegister src) {
4109   NOT_LP64(assert(VM_Version::supports_sse(), ""));
4110   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
4111   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
4112   emit_int8(0x51);
4113   emit_int8((unsigned char)(0xC0 | encode));
4114 }
4115 
4116 void Assembler::std() {
4117   emit_int8((unsigned char)0xFD);
4118 }
4119 
4120 void Assembler::sqrtss(XMMRegister dst, Address src) {
4121   NOT_LP64(assert(VM_Version::supports_sse(), ""));
4122   InstructionMark im(this);
4123   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
4124   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
4125   simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
4126   emit_int8(0x51);
4127   emit_operand(dst, src);
4128 }
4129 
4130 void Assembler::stmxcsr( Address dst) {
4131   NOT_LP64(assert(VM_Version::supports_sse(), ""));
4132   InstructionMark im(this);
4133   prefix(dst);
4134   emit_int8(0x0F);
4135   emit_int8((unsigned char)0xAE);
4136   emit_operand(as_Register(3), dst);
4137 }
4138 
4139 void Assembler::subl(Address dst, int32_t imm32) {
4140   InstructionMark im(this);
4141   prefix(dst);
4142   emit_arith_operand(0x81, rbp, dst, imm32);
4143 }
4144 
4145 void Assembler::subl(Address dst, Register src) {
4146   InstructionMark im(this);
4147   prefix(dst, src);
4148   emit_int8(0x29);
4149   emit_operand(src, dst);
4150 }
4151 
4152 void Assembler::subl(Register dst, int32_t imm32) {
4153   prefix(dst);
4154   emit_arith(0x81, 0xE8, dst, imm32);
4155 }
4156 
4157 // Force generation of a 4 byte immediate value even if it fits into 8bit
4158 void Assembler::subl_imm32(Register dst, int32_t imm32) {
4159   prefix(dst);
4160   emit_arith_imm32(0x81, 0xE8, dst, imm32);
4161 }
4162 
4163 void Assembler::subl(Register dst, Address src) {
4164   InstructionMark im(this);
4165   prefix(src, dst);
4166   emit_int8(0x2B);
4167   emit_operand(dst, src);
4168 }
4169 
4170 void Assembler::subl(Register dst, Register src) {
4171   (void) prefix_and_encode(dst->encoding(), src->encoding());
4172   emit_arith(0x2B, 0xC0, dst, src);
4173 }
4174 
4175 void Assembler::subsd(XMMRegister dst, XMMRegister src) {
4176   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4177   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
4178   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
4179   emit_int8(0x5C);
4180   emit_int8((unsigned char)(0xC0 | encode));
4181 }
4182 
4183 void Assembler::subsd(XMMRegister dst, Address src) {
4184   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4185   InstructionMark im(this);
4186   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
4187   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
4188   simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
4189   emit_int8(0x5C);
4190   emit_operand(dst, src);
4191 }
4192 
4193 void Assembler::subss(XMMRegister dst, XMMRegister src) {
4194   NOT_LP64(assert(VM_Version::supports_sse(), ""));
4195   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false , /* uses_vl */ false);
4196   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
4197   emit_int8(0x5C);
4198   emit_int8((unsigned char)(0xC0 | encode));
4199 }
4200 
4201 void Assembler::subss(XMMRegister dst, Address src) {
4202   NOT_LP64(assert(VM_Version::supports_sse(), ""));
4203   InstructionMark im(this);
4204   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
4205   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
4206   simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
4207   emit_int8(0x5C);
4208   emit_operand(dst, src);
4209 }
4210 
4211 void Assembler::testb(Register dst, int imm8) {
4212   NOT_LP64(assert(dst->has_byte_register(), "must have byte register"));
4213   (void) prefix_and_encode(dst->encoding(), true);
4214   emit_arith_b(0xF6, 0xC0, dst, imm8);
4215 }
4216 
4217 void Assembler::testb(Address dst, int imm8) {
4218   InstructionMark im(this);
4219   prefix(dst);
4220   emit_int8((unsigned char)0xF6);
4221   emit_operand(rax, dst, 1);
4222   emit_int8(imm8);
4223 }
4224 
4225 void Assembler::testl(Register dst, int32_t imm32) {
4226   // not using emit_arith because test
4227   // doesn't support sign-extension of
4228   // 8bit operands
4229   int encode = dst->encoding();
4230   if (encode == 0) {
4231     emit_int8((unsigned char)0xA9);
4232   } else {
4233     encode = prefix_and_encode(encode);
4234     emit_int8((unsigned char)0xF7);
4235     emit_int8((unsigned char)(0xC0 | encode));
4236   }
4237   emit_int32(imm32);
4238 }
4239 
4240 void Assembler::testl(Register dst, Register src) {
4241   (void) prefix_and_encode(dst->encoding(), src->encoding());
4242   emit_arith(0x85, 0xC0, dst, src);
4243 }
4244 
4245 void Assembler::testl(Register dst, Address src) {
4246   InstructionMark im(this);
4247   prefix(src, dst);
4248   emit_int8((unsigned char)0x85);
4249   emit_operand(dst, src);
4250 }
4251 
4252 void Assembler::tzcntl(Register dst, Register src) {
4253   assert(VM_Version::supports_bmi1(), "tzcnt instruction not supported");
4254   emit_int8((unsigned char)0xF3);
4255   int encode = prefix_and_encode(dst->encoding(), src->encoding());
4256   emit_int8(0x0F);
4257   emit_int8((unsigned char)0xBC);
4258   emit_int8((unsigned char)0xC0 | encode);
4259 }
4260 
4261 void Assembler::tzcntq(Register dst, Register src) {
4262   assert(VM_Version::supports_bmi1(), "tzcnt instruction not supported");
4263   emit_int8((unsigned char)0xF3);
4264   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
4265   emit_int8(0x0F);
4266   emit_int8((unsigned char)0xBC);
4267   emit_int8((unsigned char)(0xC0 | encode));
4268 }
4269 
4270 void Assembler::ucomisd(XMMRegister dst, Address src) {
4271   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4272   InstructionMark im(this);
4273   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4274   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
4275   simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4276   emit_int8(0x2E);
4277   emit_operand(dst, src);
4278 }
4279 
4280 void Assembler::ucomisd(XMMRegister dst, XMMRegister src) {
4281   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4282   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4283   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4284   emit_int8(0x2E);
4285   emit_int8((unsigned char)(0xC0 | encode));
4286 }
4287 
4288 void Assembler::ucomiss(XMMRegister dst, Address src) {
4289   NOT_LP64(assert(VM_Version::supports_sse(), ""));
4290   InstructionMark im(this);
4291   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4292   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
4293   simd_prefix(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
4294   emit_int8(0x2E);
4295   emit_operand(dst, src);
4296 }
4297 
4298 void Assembler::ucomiss(XMMRegister dst, XMMRegister src) {
4299   NOT_LP64(assert(VM_Version::supports_sse(), ""));
4300   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4301   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
4302   emit_int8(0x2E);
4303   emit_int8((unsigned char)(0xC0 | encode));
4304 }
4305 
4306 void Assembler::xabort(int8_t imm8) {
4307   emit_int8((unsigned char)0xC6);
4308   emit_int8((unsigned char)0xF8);
4309   emit_int8((unsigned char)(imm8 & 0xFF));
4310 }
4311 
4312 void Assembler::xaddl(Address dst, Register src) {
4313   InstructionMark im(this);
4314   prefix(dst, src);
4315   emit_int8(0x0F);
4316   emit_int8((unsigned char)0xC1);
4317   emit_operand(src, dst);
4318 }
4319 
4320 void Assembler::xbegin(Label& abort, relocInfo::relocType rtype) {
4321   InstructionMark im(this);
4322   relocate(rtype);
4323   if (abort.is_bound()) {
4324     address entry = target(abort);
4325     assert(entry != NULL, "abort entry NULL");
4326     intptr_t offset = entry - pc();
4327     emit_int8((unsigned char)0xC7);
4328     emit_int8((unsigned char)0xF8);
4329     emit_int32(offset - 6); // 2 opcode + 4 address
4330   } else {
4331     abort.add_patch_at(code(), locator());
4332     emit_int8((unsigned char)0xC7);
4333     emit_int8((unsigned char)0xF8);
4334     emit_int32(0);
4335   }
4336 }
4337 
4338 void Assembler::xchgl(Register dst, Address src) { // xchg
4339   InstructionMark im(this);
4340   prefix(src, dst);
4341   emit_int8((unsigned char)0x87);
4342   emit_operand(dst, src);
4343 }
4344 
4345 void Assembler::xchgl(Register dst, Register src) {
4346   int encode = prefix_and_encode(dst->encoding(), src->encoding());
4347   emit_int8((unsigned char)0x87);
4348   emit_int8((unsigned char)(0xC0 | encode));
4349 }
4350 
4351 void Assembler::xend() {
4352   emit_int8((unsigned char)0x0F);
4353   emit_int8((unsigned char)0x01);
4354   emit_int8((unsigned char)0xD5);
4355 }
4356 
4357 void Assembler::xgetbv() {
4358   emit_int8(0x0F);
4359   emit_int8(0x01);
4360   emit_int8((unsigned char)0xD0);
4361 }
4362 
4363 void Assembler::xorl(Register dst, int32_t imm32) {
4364   prefix(dst);
4365   emit_arith(0x81, 0xF0, dst, imm32);
4366 }
4367 
4368 void Assembler::xorl(Register dst, Address src) {
4369   InstructionMark im(this);
4370   prefix(src, dst);
4371   emit_int8(0x33);
4372   emit_operand(dst, src);
4373 }
4374 
4375 void Assembler::xorl(Register dst, Register src) {
4376   (void) prefix_and_encode(dst->encoding(), src->encoding());
4377   emit_arith(0x33, 0xC0, dst, src);
4378 }
4379 
4380 void Assembler::xorb(Register dst, Address src) {
4381   InstructionMark im(this);
4382   prefix(src, dst);
4383   emit_int8(0x32);
4384   emit_operand(dst, src);
4385 }
4386 
4387 // AVX 3-operands scalar float-point arithmetic instructions
4388 
4389 void Assembler::vaddsd(XMMRegister dst, XMMRegister nds, Address src) {
4390   assert(VM_Version::supports_avx(), "");
4391   InstructionMark im(this);
4392   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
4393   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
4394   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
4395   emit_int8(0x58);
4396   emit_operand(dst, src);
4397 }
4398 
4399 void Assembler::vaddsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
4400   assert(VM_Version::supports_avx(), "");
4401   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
4402   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
4403   emit_int8(0x58);
4404   emit_int8((unsigned char)(0xC0 | encode));
4405 }
4406 
4407 void Assembler::vaddss(XMMRegister dst, XMMRegister nds, Address src) {
4408   assert(VM_Version::supports_avx(), "");
4409   InstructionMark im(this);
4410   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
4411   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
4412   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
4413   emit_int8(0x58);
4414   emit_operand(dst, src);
4415 }
4416 
4417 void Assembler::vaddss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
4418   assert(VM_Version::supports_avx(), "");
4419   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
4420   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
4421   emit_int8(0x58);
4422   emit_int8((unsigned char)(0xC0 | encode));
4423 }
4424 
4425 void Assembler::vdivsd(XMMRegister dst, XMMRegister nds, Address src) {
4426   assert(VM_Version::supports_avx(), "");
4427   InstructionMark im(this);
4428   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
4429   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
4430   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
4431   emit_int8(0x5E);
4432   emit_operand(dst, src);
4433 }
4434 
4435 void Assembler::vdivsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
4436   assert(VM_Version::supports_avx(), "");
4437   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
4438   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
4439   emit_int8(0x5E);
4440   emit_int8((unsigned char)(0xC0 | encode));
4441 }
4442 
4443 void Assembler::vdivss(XMMRegister dst, XMMRegister nds, Address src) {
4444   assert(VM_Version::supports_avx(), "");
4445   InstructionMark im(this);
4446   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
4447   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
4448   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
4449   emit_int8(0x5E);
4450   emit_operand(dst, src);
4451 }
4452 
4453 void Assembler::vdivss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
4454   assert(VM_Version::supports_avx(), "");
4455   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
4456   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
4457   emit_int8(0x5E);
4458   emit_int8((unsigned char)(0xC0 | encode));
4459 }
4460 
4461 void Assembler::vmulsd(XMMRegister dst, XMMRegister nds, Address src) {
4462   assert(VM_Version::supports_avx(), "");
4463   InstructionMark im(this);
4464   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
4465   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
4466   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
4467   emit_int8(0x59);
4468   emit_operand(dst, src);
4469 }
4470 
4471 void Assembler::vmulsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
4472   assert(VM_Version::supports_avx(), "");
4473   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
4474   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
4475   emit_int8(0x59);
4476   emit_int8((unsigned char)(0xC0 | encode));
4477 }
4478 
4479 void Assembler::vmulss(XMMRegister dst, XMMRegister nds, Address src) {
4480   assert(VM_Version::supports_avx(), "");
4481   InstructionMark im(this);
4482   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
4483   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
4484   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
4485   emit_int8(0x59);
4486   emit_operand(dst, src);
4487 }
4488 
4489 void Assembler::vmulss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
4490   assert(VM_Version::supports_avx(), "");
4491   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
4492   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
4493   emit_int8(0x59);
4494   emit_int8((unsigned char)(0xC0 | encode));
4495 }
4496 
4497 void Assembler::vsubsd(XMMRegister dst, XMMRegister nds, Address src) {
4498   assert(VM_Version::supports_avx(), "");
4499   InstructionMark im(this);
4500   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
4501   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
4502   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
4503   emit_int8(0x5C);
4504   emit_operand(dst, src);
4505 }
4506 
4507 void Assembler::vsubsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
4508   assert(VM_Version::supports_avx(), "");
4509   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
4510   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
4511   emit_int8(0x5C);
4512   emit_int8((unsigned char)(0xC0 | encode));
4513 }
4514 
4515 void Assembler::vsubss(XMMRegister dst, XMMRegister nds, Address src) {
4516   assert(VM_Version::supports_avx(), "");
4517   InstructionMark im(this);
4518   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
4519   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
4520   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
4521   emit_int8(0x5C);
4522   emit_operand(dst, src);
4523 }
4524 
4525 void Assembler::vsubss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
4526   assert(VM_Version::supports_avx(), "");
4527   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
4528   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
4529   emit_int8(0x5C);
4530   emit_int8((unsigned char)(0xC0 | encode));
4531 }
4532 
4533 //====================VECTOR ARITHMETIC=====================================
4534 
4535 // Float-point vector arithmetic
4536 
4537 void Assembler::addpd(XMMRegister dst, XMMRegister src) {
4538   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4539   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4540   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4541   emit_int8(0x58);
4542   emit_int8((unsigned char)(0xC0 | encode));
4543 }
4544 
4545 void Assembler::addpd(XMMRegister dst, Address src) {
4546   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4547   InstructionMark im(this);
4548   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4549   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
4550   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4551   emit_int8(0x58);
4552   emit_operand(dst, src);
4553 }
4554 
4555 
4556 void Assembler::addps(XMMRegister dst, XMMRegister src) {
4557   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4558   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4559   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
4560   emit_int8(0x58);
4561   emit_int8((unsigned char)(0xC0 | encode));
4562 }
4563 
4564 void Assembler::vaddpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4565   assert(VM_Version::supports_avx(), "");
4566   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4567   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4568   emit_int8(0x58);
4569   emit_int8((unsigned char)(0xC0 | encode));
4570 }
4571 
4572 void Assembler::vaddps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4573   assert(VM_Version::supports_avx(), "");
4574   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4575   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
4576   emit_int8(0x58);
4577   emit_int8((unsigned char)(0xC0 | encode));
4578 }
4579 
4580 void Assembler::vaddpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4581   assert(VM_Version::supports_avx(), "");
4582   InstructionMark im(this);
4583   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4584   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
4585   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4586   emit_int8(0x58);
4587   emit_operand(dst, src);
4588 }
4589 
4590 void Assembler::vaddps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4591   assert(VM_Version::supports_avx(), "");
4592   InstructionMark im(this);
4593   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4594   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
4595   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
4596   emit_int8(0x58);
4597   emit_operand(dst, src);
4598 }
4599 
4600 void Assembler::subpd(XMMRegister dst, XMMRegister src) {
4601   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4602   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4603   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4604   emit_int8(0x5C);
4605   emit_int8((unsigned char)(0xC0 | encode));
4606 }
4607 
4608 void Assembler::subps(XMMRegister dst, XMMRegister src) {
4609   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4610   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4611   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
4612   emit_int8(0x5C);
4613   emit_int8((unsigned char)(0xC0 | encode));
4614 }
4615 
4616 void Assembler::vsubpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4617   assert(VM_Version::supports_avx(), "");
4618   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4619   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4620   emit_int8(0x5C);
4621   emit_int8((unsigned char)(0xC0 | encode));
4622 }
4623 
4624 void Assembler::vsubps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4625   assert(VM_Version::supports_avx(), "");
4626   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4627   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
4628   emit_int8(0x5C);
4629   emit_int8((unsigned char)(0xC0 | encode));
4630 }
4631 
4632 void Assembler::vsubpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4633   assert(VM_Version::supports_avx(), "");
4634   InstructionMark im(this);
4635   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4636   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
4637   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4638   emit_int8(0x5C);
4639   emit_operand(dst, src);
4640 }
4641 
4642 void Assembler::vsubps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4643   assert(VM_Version::supports_avx(), "");
4644   InstructionMark im(this);
4645   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4646   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
4647   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
4648   emit_int8(0x5C);
4649   emit_operand(dst, src);
4650 }
4651 
4652 void Assembler::mulpd(XMMRegister dst, XMMRegister src) {
4653   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4654   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4655   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4656   emit_int8(0x59);
4657   emit_int8((unsigned char)(0xC0 | encode));
4658 }
4659 
4660 void Assembler::mulpd(XMMRegister dst, Address src) {
4661   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4662   InstructionMark im(this);
4663   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4664   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
4665   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4666   emit_int8(0x59);
4667   emit_operand(dst, src);
4668 }
4669 
4670 void Assembler::mulps(XMMRegister dst, XMMRegister src) {
4671   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4672   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4673   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
4674   emit_int8(0x59);
4675   emit_int8((unsigned char)(0xC0 | encode));
4676 }
4677 
4678 void Assembler::vmulpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4679   assert(VM_Version::supports_avx(), "");
4680   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4681   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4682   emit_int8(0x59);
4683   emit_int8((unsigned char)(0xC0 | encode));
4684 }
4685 
4686 void Assembler::vmulps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4687   assert(VM_Version::supports_avx(), "");
4688   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4689   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
4690   emit_int8(0x59);
4691   emit_int8((unsigned char)(0xC0 | encode));
4692 }
4693 
4694 void Assembler::vmulpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4695   assert(VM_Version::supports_avx(), "");
4696   InstructionMark im(this);
4697   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4698   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
4699   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4700   emit_int8(0x59);
4701   emit_operand(dst, src);
4702 }
4703 
4704 void Assembler::vmulps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4705   assert(VM_Version::supports_avx(), "");
4706   InstructionMark im(this);
4707   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4708   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
4709   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
4710   emit_int8(0x59);
4711   emit_operand(dst, src);
4712 }
4713 
4714 void Assembler::divpd(XMMRegister dst, XMMRegister src) {
4715   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4716   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4717   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4718   emit_int8(0x5E);
4719   emit_int8((unsigned char)(0xC0 | encode));
4720 }
4721 
4722 void Assembler::divps(XMMRegister dst, XMMRegister src) {
4723   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4724   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4725   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
4726   emit_int8(0x5E);
4727   emit_int8((unsigned char)(0xC0 | encode));
4728 }
4729 
4730 void Assembler::vdivpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4731   assert(VM_Version::supports_avx(), "");
4732   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4733   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4734   emit_int8(0x5E);
4735   emit_int8((unsigned char)(0xC0 | encode));
4736 }
4737 
4738 void Assembler::vdivps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4739   assert(VM_Version::supports_avx(), "");
4740   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4741   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
4742   emit_int8(0x5E);
4743   emit_int8((unsigned char)(0xC0 | encode));
4744 }
4745 
4746 void Assembler::vdivpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4747   assert(VM_Version::supports_avx(), "");
4748   InstructionMark im(this);
4749   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4750   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
4751   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4752   emit_int8(0x5E);
4753   emit_operand(dst, src);
4754 }
4755 
4756 void Assembler::vdivps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4757   assert(VM_Version::supports_avx(), "");
4758   InstructionMark im(this);
4759   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4760   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
4761   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
4762   emit_int8(0x5E);
4763   emit_operand(dst, src);
4764 }
4765 
4766 void Assembler::vsqrtpd(XMMRegister dst, XMMRegister src, int vector_len) {
4767   assert(VM_Version::supports_avx(), "");
4768   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4769   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4770   emit_int8(0x51);
4771   emit_int8((unsigned char)(0xC0 | encode));
4772 }
4773 
4774 void Assembler::vsqrtpd(XMMRegister dst, Address src, int vector_len) {
4775   assert(VM_Version::supports_avx(), "");
4776   InstructionMark im(this);
4777   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4778   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
4779   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4780   emit_int8(0x51);
4781   emit_operand(dst, src);
4782 }
4783 
4784 void Assembler::andpd(XMMRegister dst, XMMRegister src) {
4785   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4786   InstructionAttr attributes(AVX_128bit, /* rex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
4787   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4788   emit_int8(0x54);
4789   emit_int8((unsigned char)(0xC0 | encode));
4790 }
4791 
4792 void Assembler::andps(XMMRegister dst, XMMRegister src) {
4793   NOT_LP64(assert(VM_Version::supports_sse(), ""));
4794   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
4795   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
4796   emit_int8(0x54);
4797   emit_int8((unsigned char)(0xC0 | encode));
4798 }
4799 
4800 void Assembler::andps(XMMRegister dst, Address src) {
4801   NOT_LP64(assert(VM_Version::supports_sse(), ""));
4802   InstructionMark im(this);
4803   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
4804   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
4805   simd_prefix(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
4806   emit_int8(0x54);
4807   emit_operand(dst, src);
4808 }
4809 
4810 void Assembler::andpd(XMMRegister dst, Address src) {
4811   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4812   InstructionMark im(this);
4813   InstructionAttr attributes(AVX_128bit, /* rex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
4814   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
4815   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4816   emit_int8(0x54);
4817   emit_operand(dst, src);
4818 }
4819 
4820 void Assembler::vandpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4821   assert(VM_Version::supports_avx(), "");
4822   InstructionAttr attributes(vector_len, /* vex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
4823   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4824   emit_int8(0x54);
4825   emit_int8((unsigned char)(0xC0 | encode));
4826 }
4827 
4828 void Assembler::vandps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4829   assert(VM_Version::supports_avx(), "");
4830   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
4831   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
4832   emit_int8(0x54);
4833   emit_int8((unsigned char)(0xC0 | encode));
4834 }
4835 
4836 void Assembler::vandpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4837   assert(VM_Version::supports_avx(), "");
4838   InstructionMark im(this);
4839   InstructionAttr attributes(vector_len, /* vex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
4840   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
4841   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4842   emit_int8(0x54);
4843   emit_operand(dst, src);
4844 }
4845 
4846 void Assembler::vandps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4847   assert(VM_Version::supports_avx(), "");
4848   InstructionMark im(this);
4849   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
4850   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
4851   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
4852   emit_int8(0x54);
4853   emit_operand(dst, src);
4854 }
4855 
4856 void Assembler::unpckhpd(XMMRegister dst, XMMRegister src) {
4857   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4858   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4859   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4860   emit_int8(0x15);
4861   emit_int8((unsigned char)(0xC0 | encode));
4862 }
4863 
4864 void Assembler::unpcklpd(XMMRegister dst, XMMRegister src) {
4865   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4866   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4867   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4868   emit_int8(0x14);
4869   emit_int8((unsigned char)(0xC0 | encode));
4870 }
4871 
4872 void Assembler::xorpd(XMMRegister dst, XMMRegister src) {
4873   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4874   InstructionAttr attributes(AVX_128bit, /* rex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
4875   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4876   emit_int8(0x57);
4877   emit_int8((unsigned char)(0xC0 | encode));
4878 }
4879 
4880 void Assembler::xorps(XMMRegister dst, XMMRegister src) {
4881   NOT_LP64(assert(VM_Version::supports_sse(), ""));
4882   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
4883   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
4884   emit_int8(0x57);
4885   emit_int8((unsigned char)(0xC0 | encode));
4886 }
4887 
4888 void Assembler::xorpd(XMMRegister dst, Address src) {
4889   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4890   InstructionMark im(this);
4891   InstructionAttr attributes(AVX_128bit, /* rex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
4892   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
4893   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4894   emit_int8(0x57);
4895   emit_operand(dst, src);
4896 }
4897 
4898 void Assembler::xorps(XMMRegister dst, Address src) {
4899   NOT_LP64(assert(VM_Version::supports_sse(), ""));
4900   InstructionMark im(this);
4901   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
4902   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
4903   simd_prefix(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
4904   emit_int8(0x57);
4905   emit_operand(dst, src);
4906 }
4907 
4908 void Assembler::vxorpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4909   assert(VM_Version::supports_avx(), "");
4910   InstructionAttr attributes(vector_len, /* vex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
4911   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4912   emit_int8(0x57);
4913   emit_int8((unsigned char)(0xC0 | encode));
4914 }
4915 
4916 void Assembler::vxorps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4917   assert(VM_Version::supports_avx(), "");
4918   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
4919   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
4920   emit_int8(0x57);
4921   emit_int8((unsigned char)(0xC0 | encode));
4922 }
4923 
4924 void Assembler::vxorpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4925   assert(VM_Version::supports_avx(), "");
4926   InstructionMark im(this);
4927   InstructionAttr attributes(vector_len, /* vex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
4928   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
4929   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4930   emit_int8(0x57);
4931   emit_operand(dst, src);
4932 }
4933 
4934 void Assembler::vxorps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4935   assert(VM_Version::supports_avx(), "");
4936   InstructionMark im(this);
4937   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
4938   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
4939   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
4940   emit_int8(0x57);
4941   emit_operand(dst, src);
4942 }
4943 
4944 // Integer vector arithmetic
4945 void Assembler::vphaddw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4946   assert(VM_Version::supports_avx() && (vector_len == 0) ||
4947          VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
4948   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
4949   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4950   emit_int8(0x01);
4951   emit_int8((unsigned char)(0xC0 | encode));
4952 }
4953 
4954 void Assembler::vphaddd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4955   assert(VM_Version::supports_avx() && (vector_len == 0) ||
4956          VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
4957   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
4958   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4959   emit_int8(0x02);
4960   emit_int8((unsigned char)(0xC0 | encode));
4961 }
4962 
4963 void Assembler::paddb(XMMRegister dst, XMMRegister src) {
4964   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4965   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
4966   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4967   emit_int8((unsigned char)0xFC);
4968   emit_int8((unsigned char)(0xC0 | encode));
4969 }
4970 
4971 void Assembler::paddw(XMMRegister dst, XMMRegister src) {
4972   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4973   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
4974   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4975   emit_int8((unsigned char)0xFD);
4976   emit_int8((unsigned char)(0xC0 | encode));
4977 }
4978 
4979 void Assembler::paddd(XMMRegister dst, XMMRegister src) {
4980   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4981   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4982   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4983   emit_int8((unsigned char)0xFE);
4984   emit_int8((unsigned char)(0xC0 | encode));
4985 }
4986 
4987 void Assembler::paddd(XMMRegister dst, Address src) {
4988   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4989   InstructionMark im(this);
4990   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4991   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4992   emit_int8((unsigned char)0xFE);
4993   emit_operand(dst, src);
4994 }
4995 
4996 void Assembler::paddq(XMMRegister dst, XMMRegister src) {
4997   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4998   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4999   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5000   emit_int8((unsigned char)0xD4);
5001   emit_int8((unsigned char)(0xC0 | encode));
5002 }
5003 
5004 void Assembler::phaddw(XMMRegister dst, XMMRegister src) {
5005   assert(VM_Version::supports_sse3(), "");
5006   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
5007   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5008   emit_int8(0x01);
5009   emit_int8((unsigned char)(0xC0 | encode));
5010 }
5011 
5012 void Assembler::phaddd(XMMRegister dst, XMMRegister src) {
5013   assert(VM_Version::supports_sse3(), "");
5014   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
5015   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5016   emit_int8(0x02);
5017   emit_int8((unsigned char)(0xC0 | encode));
5018 }
5019 
5020 void Assembler::vpaddb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5021   assert(UseAVX > 0, "requires some form of AVX");
5022   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5023   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5024   emit_int8((unsigned char)0xFC);
5025   emit_int8((unsigned char)(0xC0 | encode));
5026 }
5027 
5028 void Assembler::vpaddw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5029   assert(UseAVX > 0, "requires some form of AVX");
5030   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5031   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5032   emit_int8((unsigned char)0xFD);
5033   emit_int8((unsigned char)(0xC0 | encode));
5034 }
5035 
5036 void Assembler::vpaddd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5037   assert(UseAVX > 0, "requires some form of AVX");
5038   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5039   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5040   emit_int8((unsigned char)0xFE);
5041   emit_int8((unsigned char)(0xC0 | encode));
5042 }
5043 
5044 void Assembler::vpaddq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5045   assert(UseAVX > 0, "requires some form of AVX");
5046   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5047   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5048   emit_int8((unsigned char)0xD4);
5049   emit_int8((unsigned char)(0xC0 | encode));
5050 }
5051 
5052 void Assembler::vpaddb(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5053   assert(UseAVX > 0, "requires some form of AVX");
5054   InstructionMark im(this);
5055   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5056   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
5057   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5058   emit_int8((unsigned char)0xFC);
5059   emit_operand(dst, src);
5060 }
5061 
5062 void Assembler::vpaddw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5063   assert(UseAVX > 0, "requires some form of AVX");
5064   InstructionMark im(this);
5065   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5066   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
5067   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5068   emit_int8((unsigned char)0xFD);
5069   emit_operand(dst, src);
5070 }
5071 
5072 void Assembler::vpaddd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5073   assert(UseAVX > 0, "requires some form of AVX");
5074   InstructionMark im(this);
5075   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5076   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
5077   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5078   emit_int8((unsigned char)0xFE);
5079   emit_operand(dst, src);
5080 }
5081 
5082 void Assembler::vpaddq(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5083   assert(UseAVX > 0, "requires some form of AVX");
5084   InstructionMark im(this);
5085   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5086   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
5087   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5088   emit_int8((unsigned char)0xD4);
5089   emit_operand(dst, src);
5090 }
5091 
5092 void Assembler::psubb(XMMRegister dst, XMMRegister src) {
5093   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5094   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5095   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5096   emit_int8((unsigned char)0xF8);
5097   emit_int8((unsigned char)(0xC0 | encode));
5098 }
5099 
5100 void Assembler::psubw(XMMRegister dst, XMMRegister src) {
5101   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5102   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5103   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5104   emit_int8((unsigned char)0xF9);
5105   emit_int8((unsigned char)(0xC0 | encode));
5106 }
5107 
5108 void Assembler::psubd(XMMRegister dst, XMMRegister src) {
5109   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5110   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5111   emit_int8((unsigned char)0xFA);
5112   emit_int8((unsigned char)(0xC0 | encode));
5113 }
5114 
5115 void Assembler::psubq(XMMRegister dst, XMMRegister src) {
5116   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5117   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5118   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5119   emit_int8((unsigned char)0xFB);
5120   emit_int8((unsigned char)(0xC0 | encode));
5121 }
5122 
5123 void Assembler::vpsubb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5124   assert(UseAVX > 0, "requires some form of AVX");
5125   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5126   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5127   emit_int8((unsigned char)0xF8);
5128   emit_int8((unsigned char)(0xC0 | encode));
5129 }
5130 
5131 void Assembler::vpsubw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5132   assert(UseAVX > 0, "requires some form of AVX");
5133   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5134   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5135   emit_int8((unsigned char)0xF9);
5136   emit_int8((unsigned char)(0xC0 | encode));
5137 }
5138 
5139 void Assembler::vpsubd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5140   assert(UseAVX > 0, "requires some form of AVX");
5141   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5142   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5143   emit_int8((unsigned char)0xFA);
5144   emit_int8((unsigned char)(0xC0 | encode));
5145 }
5146 
5147 void Assembler::vpsubq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5148   assert(UseAVX > 0, "requires some form of AVX");
5149   InstructionAttr attributes(vector_len, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5150   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5151   emit_int8((unsigned char)0xFB);
5152   emit_int8((unsigned char)(0xC0 | encode));
5153 }
5154 
5155 void Assembler::vpsubb(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5156   assert(UseAVX > 0, "requires some form of AVX");
5157   InstructionMark im(this);
5158   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5159   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
5160   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5161   emit_int8((unsigned char)0xF8);
5162   emit_operand(dst, src);
5163 }
5164 
5165 void Assembler::vpsubw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5166   assert(UseAVX > 0, "requires some form of AVX");
5167   InstructionMark im(this);
5168   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5169   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
5170   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5171   emit_int8((unsigned char)0xF9);
5172   emit_operand(dst, src);
5173 }
5174 
5175 void Assembler::vpsubd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5176   assert(UseAVX > 0, "requires some form of AVX");
5177   InstructionMark im(this);
5178   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5179   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
5180   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5181   emit_int8((unsigned char)0xFA);
5182   emit_operand(dst, src);
5183 }
5184 
5185 void Assembler::vpsubq(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5186   assert(UseAVX > 0, "requires some form of AVX");
5187   InstructionMark im(this);
5188   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5189   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
5190   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5191   emit_int8((unsigned char)0xFB);
5192   emit_operand(dst, src);
5193 }
5194 
5195 void Assembler::pmullw(XMMRegister dst, XMMRegister src) {
5196   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5197   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5198   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5199   emit_int8((unsigned char)0xD5);
5200   emit_int8((unsigned char)(0xC0 | encode));
5201 }
5202 
5203 void Assembler::pmulld(XMMRegister dst, XMMRegister src) {
5204   assert(VM_Version::supports_sse4_1(), "");
5205   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5206   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5207   emit_int8(0x40);
5208   emit_int8((unsigned char)(0xC0 | encode));
5209 }
5210 
5211 void Assembler::vpmullw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5212   assert(UseAVX > 0, "requires some form of AVX");
5213   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5214   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5215   emit_int8((unsigned char)0xD5);
5216   emit_int8((unsigned char)(0xC0 | encode));
5217 }
5218 
5219 void Assembler::vpmulld(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5220   assert(UseAVX > 0, "requires some form of AVX");
5221   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5222   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5223   emit_int8(0x40);
5224   emit_int8((unsigned char)(0xC0 | encode));
5225 }
5226 
5227 void Assembler::vpmullq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5228   assert(UseAVX > 2, "requires some form of AVX");
5229   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
5230   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5231   emit_int8(0x40);
5232   emit_int8((unsigned char)(0xC0 | encode));
5233 }
5234 
5235 void Assembler::vpmullw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5236   assert(UseAVX > 0, "requires some form of AVX");
5237   InstructionMark im(this);
5238   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5239   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
5240   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5241   emit_int8((unsigned char)0xD5);
5242   emit_operand(dst, src);
5243 }
5244 
5245 void Assembler::vpmulld(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5246   assert(UseAVX > 0, "requires some form of AVX");
5247   InstructionMark im(this);
5248   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5249   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
5250   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5251   emit_int8(0x40);
5252   emit_operand(dst, src);
5253 }
5254 
5255 void Assembler::vpmullq(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5256   assert(UseAVX > 0, "requires some form of AVX");
5257   InstructionMark im(this);
5258   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
5259   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
5260   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5261   emit_int8(0x40);
5262   emit_operand(dst, src);
5263 }
5264 
5265 // Shift packed integers left by specified number of bits.
5266 void Assembler::psllw(XMMRegister dst, int shift) {
5267   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5268   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5269   // XMM6 is for /6 encoding: 66 0F 71 /6 ib
5270   int encode = simd_prefix_and_encode(xmm6, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5271   emit_int8(0x71);
5272   emit_int8((unsigned char)(0xC0 | encode));
5273   emit_int8(shift & 0xFF);
5274 }
5275 
5276 void Assembler::pslld(XMMRegister dst, int shift) {
5277   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5278   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5279   // XMM6 is for /6 encoding: 66 0F 72 /6 ib
5280   int encode = simd_prefix_and_encode(xmm6, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5281   emit_int8(0x72);
5282   emit_int8((unsigned char)(0xC0 | encode));
5283   emit_int8(shift & 0xFF);
5284 }
5285 
5286 void Assembler::psllq(XMMRegister dst, int shift) {
5287   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5288   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5289   // XMM6 is for /6 encoding: 66 0F 73 /6 ib
5290   int encode = simd_prefix_and_encode(xmm6, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5291   emit_int8(0x73);
5292   emit_int8((unsigned char)(0xC0 | encode));
5293   emit_int8(shift & 0xFF);
5294 }
5295 
5296 void Assembler::psllw(XMMRegister dst, XMMRegister shift) {
5297   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5298   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5299   int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5300   emit_int8((unsigned char)0xF1);
5301   emit_int8((unsigned char)(0xC0 | encode));
5302 }
5303 
5304 void Assembler::pslld(XMMRegister dst, XMMRegister shift) {
5305   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5306   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5307   int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5308   emit_int8((unsigned char)0xF2);
5309   emit_int8((unsigned char)(0xC0 | encode));
5310 }
5311 
5312 void Assembler::psllq(XMMRegister dst, XMMRegister shift) {
5313   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5314   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5315   int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5316   emit_int8((unsigned char)0xF3);
5317   emit_int8((unsigned char)(0xC0 | encode));
5318 }
5319 
5320 void Assembler::vpsllw(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
5321   assert(UseAVX > 0, "requires some form of AVX");
5322   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5323   // XMM6 is for /6 encoding: 66 0F 71 /6 ib
5324   int encode = vex_prefix_and_encode(xmm6->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5325   emit_int8(0x71);
5326   emit_int8((unsigned char)(0xC0 | encode));
5327   emit_int8(shift & 0xFF);
5328 }
5329 
5330 void Assembler::vpslld(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
5331   assert(UseAVX > 0, "requires some form of AVX");
5332   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5333   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5334   // XMM6 is for /6 encoding: 66 0F 72 /6 ib
5335   int encode = vex_prefix_and_encode(xmm6->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5336   emit_int8(0x72);
5337   emit_int8((unsigned char)(0xC0 | encode));
5338   emit_int8(shift & 0xFF);
5339 }
5340 
5341 void Assembler::vpsllq(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
5342   assert(UseAVX > 0, "requires some form of AVX");
5343   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5344   // XMM6 is for /6 encoding: 66 0F 73 /6 ib
5345   int encode = vex_prefix_and_encode(xmm6->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5346   emit_int8(0x73);
5347   emit_int8((unsigned char)(0xC0 | encode));
5348   emit_int8(shift & 0xFF);
5349 }
5350 
5351 void Assembler::vpsllw(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
5352   assert(UseAVX > 0, "requires some form of AVX");
5353   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5354   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5355   emit_int8((unsigned char)0xF1);
5356   emit_int8((unsigned char)(0xC0 | encode));
5357 }
5358 
5359 void Assembler::vpslld(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
5360   assert(UseAVX > 0, "requires some form of AVX");
5361   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5362   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5363   emit_int8((unsigned char)0xF2);
5364   emit_int8((unsigned char)(0xC0 | encode));
5365 }
5366 
5367 void Assembler::vpsllq(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
5368   assert(UseAVX > 0, "requires some form of AVX");
5369   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5370   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5371   emit_int8((unsigned char)0xF3);
5372   emit_int8((unsigned char)(0xC0 | encode));
5373 }
5374 
5375 // Shift packed integers logically right by specified number of bits.
5376 void Assembler::psrlw(XMMRegister dst, int shift) {
5377   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5378   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5379   // XMM2 is for /2 encoding: 66 0F 71 /2 ib
5380   int encode = simd_prefix_and_encode(xmm2, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5381   emit_int8(0x71);
5382   emit_int8((unsigned char)(0xC0 | encode));
5383   emit_int8(shift & 0xFF);
5384 }
5385 
5386 void Assembler::psrld(XMMRegister dst, int shift) {
5387   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5388   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5389   // XMM2 is for /2 encoding: 66 0F 72 /2 ib
5390   int encode = simd_prefix_and_encode(xmm2, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5391   emit_int8(0x72);
5392   emit_int8((unsigned char)(0xC0 | encode));
5393   emit_int8(shift & 0xFF);
5394 }
5395 
5396 void Assembler::psrlq(XMMRegister dst, int shift) {
5397   // Do not confuse it with psrldq SSE2 instruction which
5398   // shifts 128 bit value in xmm register by number of bytes.
5399   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5400   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5401   // XMM2 is for /2 encoding: 66 0F 73 /2 ib
5402   int encode = simd_prefix_and_encode(xmm2, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5403   emit_int8(0x73);
5404   emit_int8((unsigned char)(0xC0 | encode));
5405   emit_int8(shift & 0xFF);
5406 }
5407 
5408 void Assembler::psrlw(XMMRegister dst, XMMRegister shift) {
5409   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5410   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5411   int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5412   emit_int8((unsigned char)0xD1);
5413   emit_int8((unsigned char)(0xC0 | encode));
5414 }
5415 
5416 void Assembler::psrld(XMMRegister dst, XMMRegister shift) {
5417   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5418   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5419   int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5420   emit_int8((unsigned char)0xD2);
5421   emit_int8((unsigned char)(0xC0 | encode));
5422 }
5423 
5424 void Assembler::psrlq(XMMRegister dst, XMMRegister shift) {
5425   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5426   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5427   int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5428   emit_int8((unsigned char)0xD3);
5429   emit_int8((unsigned char)(0xC0 | encode));
5430 }
5431 
5432 void Assembler::vpsrlw(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
5433   assert(UseAVX > 0, "requires some form of AVX");
5434   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5435   // XMM2 is for /2 encoding: 66 0F 71 /2 ib
5436   int encode = vex_prefix_and_encode(xmm2->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5437   emit_int8(0x71);
5438   emit_int8((unsigned char)(0xC0 | encode));
5439   emit_int8(shift & 0xFF);
5440 }
5441 
5442 void Assembler::vpsrld(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
5443   assert(UseAVX > 0, "requires some form of AVX");
5444   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5445   // XMM2 is for /2 encoding: 66 0F 72 /2 ib
5446   int encode = vex_prefix_and_encode(xmm2->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5447   emit_int8(0x72);
5448   emit_int8((unsigned char)(0xC0 | encode));
5449   emit_int8(shift & 0xFF);
5450 }
5451 
5452 void Assembler::vpsrlq(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
5453   assert(UseAVX > 0, "requires some form of AVX");
5454   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5455   // XMM2 is for /2 encoding: 66 0F 73 /2 ib
5456   int encode = vex_prefix_and_encode(xmm2->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5457   emit_int8(0x73);
5458   emit_int8((unsigned char)(0xC0 | encode));
5459   emit_int8(shift & 0xFF);
5460 }
5461 
5462 void Assembler::vpsrlw(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
5463   assert(UseAVX > 0, "requires some form of AVX");
5464   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5465   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5466   emit_int8((unsigned char)0xD1);
5467   emit_int8((unsigned char)(0xC0 | encode));
5468 }
5469 
5470 void Assembler::vpsrld(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
5471   assert(UseAVX > 0, "requires some form of AVX");
5472   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5473   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5474   emit_int8((unsigned char)0xD2);
5475   emit_int8((unsigned char)(0xC0 | encode));
5476 }
5477 
5478 void Assembler::vpsrlq(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
5479   assert(UseAVX > 0, "requires some form of AVX");
5480   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5481   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5482   emit_int8((unsigned char)0xD3);
5483   emit_int8((unsigned char)(0xC0 | encode));
5484 }
5485 
5486 // Shift packed integers arithmetically right by specified number of bits.
5487 void Assembler::psraw(XMMRegister dst, int shift) {
5488   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5489   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5490   // XMM4 is for /4 encoding: 66 0F 71 /4 ib
5491   int encode = simd_prefix_and_encode(xmm4, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5492   emit_int8(0x71);
5493   emit_int8((unsigned char)(0xC0 | encode));
5494   emit_int8(shift & 0xFF);
5495 }
5496 
5497 void Assembler::psrad(XMMRegister dst, int shift) {
5498   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5499   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5500   // XMM4 is for /4 encoding: 66 0F 72 /4 ib
5501   int encode = simd_prefix_and_encode(xmm4, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5502   emit_int8(0x72);
5503   emit_int8((unsigned char)(0xC0 | encode));
5504   emit_int8(shift & 0xFF);
5505 }
5506 
5507 void Assembler::psraw(XMMRegister dst, XMMRegister shift) {
5508   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5509   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5510   int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5511   emit_int8((unsigned char)0xE1);
5512   emit_int8((unsigned char)(0xC0 | encode));
5513 }
5514 
5515 void Assembler::psrad(XMMRegister dst, XMMRegister shift) {
5516   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5517   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5518   int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5519   emit_int8((unsigned char)0xE2);
5520   emit_int8((unsigned char)(0xC0 | encode));
5521 }
5522 
5523 void Assembler::vpsraw(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
5524   assert(UseAVX > 0, "requires some form of AVX");
5525   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5526   // XMM4 is for /4 encoding: 66 0F 71 /4 ib
5527   int encode = vex_prefix_and_encode(xmm4->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5528   emit_int8(0x71);
5529   emit_int8((unsigned char)(0xC0 | encode));
5530   emit_int8(shift & 0xFF);
5531 }
5532 
5533 void Assembler::vpsrad(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
5534   assert(UseAVX > 0, "requires some form of AVX");
5535   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5536   // XMM4 is for /4 encoding: 66 0F 71 /4 ib
5537   int encode = vex_prefix_and_encode(xmm4->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5538   emit_int8(0x72);
5539   emit_int8((unsigned char)(0xC0 | encode));
5540   emit_int8(shift & 0xFF);
5541 }
5542 
5543 void Assembler::vpsraw(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
5544   assert(UseAVX > 0, "requires some form of AVX");
5545   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5546   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5547   emit_int8((unsigned char)0xE1);
5548   emit_int8((unsigned char)(0xC0 | encode));
5549 }
5550 
5551 void Assembler::vpsrad(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
5552   assert(UseAVX > 0, "requires some form of AVX");
5553   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5554   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5555   emit_int8((unsigned char)0xE2);
5556   emit_int8((unsigned char)(0xC0 | encode));
5557 }
5558 
5559 
5560 // logical operations packed integers
5561 void Assembler::pand(XMMRegister dst, XMMRegister src) {
5562   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5563   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5564   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5565   emit_int8((unsigned char)0xDB);
5566   emit_int8((unsigned char)(0xC0 | encode));
5567 }
5568 
5569 void Assembler::vpand(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5570   assert(UseAVX > 0, "requires some form of AVX");
5571   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5572   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5573   emit_int8((unsigned char)0xDB);
5574   emit_int8((unsigned char)(0xC0 | encode));
5575 }
5576 
5577 void Assembler::vpand(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5578   assert(UseAVX > 0, "requires some form of AVX");
5579   InstructionMark im(this);
5580   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5581   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
5582   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5583   emit_int8((unsigned char)0xDB);
5584   emit_operand(dst, src);
5585 }
5586 
5587 void Assembler::pandn(XMMRegister dst, XMMRegister src) {
5588   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5589   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5590   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5591   emit_int8((unsigned char)0xDF);
5592   emit_int8((unsigned char)(0xC0 | encode));
5593 }
5594 
5595 void Assembler::por(XMMRegister dst, XMMRegister src) {
5596   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5597   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5598   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5599   emit_int8((unsigned char)0xEB);
5600   emit_int8((unsigned char)(0xC0 | encode));
5601 }
5602 
5603 void Assembler::vpor(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5604   assert(UseAVX > 0, "requires some form of AVX");
5605   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5606   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5607   emit_int8((unsigned char)0xEB);
5608   emit_int8((unsigned char)(0xC0 | encode));
5609 }
5610 
5611 void Assembler::vpor(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5612   assert(UseAVX > 0, "requires some form of AVX");
5613   InstructionMark im(this);
5614   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5615   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
5616   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5617   emit_int8((unsigned char)0xEB);
5618   emit_operand(dst, src);
5619 }
5620 
5621 void Assembler::pxor(XMMRegister dst, XMMRegister src) {
5622   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5623   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5624   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5625   emit_int8((unsigned char)0xEF);
5626   emit_int8((unsigned char)(0xC0 | encode));
5627 }
5628 
5629 void Assembler::vpxor(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5630   assert(UseAVX > 0, "requires some form of AVX");
5631   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5632   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5633   emit_int8((unsigned char)0xEF);
5634   emit_int8((unsigned char)(0xC0 | encode));
5635 }
5636 
5637 void Assembler::vpxor(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5638   assert(UseAVX > 0, "requires some form of AVX");
5639   InstructionMark im(this);
5640   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5641   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
5642   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5643   emit_int8((unsigned char)0xEF);
5644   emit_operand(dst, src);
5645 }
5646 
5647 
5648 // vinserti forms
5649 
5650 void Assembler::vinserti128(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) {
5651   assert(VM_Version::supports_avx2(), "");
5652   assert(imm8 <= 0x01, "imm8: %u", imm8);
5653   int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_256bit;
5654   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
5655   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
5656   emit_int8(0x38);
5657   emit_int8((unsigned char)(0xC0 | encode));
5658   // 0x00 - insert into lower 128 bits
5659   // 0x01 - insert into upper 128 bits
5660   emit_int8(imm8 & 0x01);
5661 }
5662 
5663 void Assembler::vinserti128(XMMRegister dst, XMMRegister nds, Address src, uint8_t imm8) {
5664   assert(VM_Version::supports_avx2(), "");
5665   assert(dst != xnoreg, "sanity");
5666   assert(imm8 <= 0x01, "imm8: %u", imm8);
5667   int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_256bit;
5668   InstructionMark im(this);
5669   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
5670   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
5671   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
5672   emit_int8(0x38);
5673   emit_operand(dst, src);
5674   // 0x00 - insert into lower 128 bits
5675   // 0x01 - insert into upper 128 bits
5676   emit_int8(imm8 & 0x01);
5677 }
5678 
5679 void Assembler::vinserti32x4(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) {
5680   assert(VM_Version::supports_evex(), "");
5681   assert(imm8 <= 0x03, "imm8: %u", imm8);
5682   InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
5683   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
5684   emit_int8(0x38);
5685   emit_int8((unsigned char)(0xC0 | encode));
5686   // 0x00 - insert into q0 128 bits (0..127)
5687   // 0x01 - insert into q1 128 bits (128..255)
5688   // 0x02 - insert into q2 128 bits (256..383)
5689   // 0x03 - insert into q3 128 bits (384..511)
5690   emit_int8(imm8 & 0x03);
5691 }
5692 
5693 void Assembler::vinserti32x4(XMMRegister dst, XMMRegister nds, Address src, uint8_t imm8) {
5694   assert(VM_Version::supports_avx(), "");
5695   assert(dst != xnoreg, "sanity");
5696   assert(imm8 <= 0x03, "imm8: %u", imm8);
5697   int vector_len = VM_Version::supports_evex() ? AVX_512bit : AVX_256bit;
5698   InstructionMark im(this);
5699   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
5700   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
5701   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
5702   emit_int8(0x18);
5703   emit_operand(dst, src);
5704   // 0x00 - insert into q0 128 bits (0..127)
5705   // 0x01 - insert into q1 128 bits (128..255)
5706   // 0x02 - insert into q2 128 bits (256..383)
5707   // 0x03 - insert into q3 128 bits (384..511)
5708   emit_int8(imm8 & 0x03);
5709 }
5710 
5711 void Assembler::vinserti64x4(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) {
5712   assert(VM_Version::supports_evex(), "");
5713   assert(imm8 <= 0x01, "imm8: %u", imm8);
5714   InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
5715   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
5716   emit_int8(0x38);
5717   emit_int8((unsigned char)(0xC0 | encode));
5718   // 0x00 - insert into lower 256 bits
5719   // 0x01 - insert into upper 256 bits
5720   emit_int8(imm8 & 0x01);
5721 }
5722 
5723 
5724 // vinsertf forms
5725 
5726 void Assembler::vinsertf128(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) {
5727   assert(VM_Version::supports_avx(), "");
5728   assert(imm8 <= 0x01, "imm8: %u", imm8);
5729   int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_256bit;
5730   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
5731   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
5732   emit_int8(0x18);
5733   emit_int8((unsigned char)(0xC0 | encode));
5734   // 0x00 - insert into lower 128 bits
5735   // 0x01 - insert into upper 128 bits
5736   emit_int8(imm8 & 0x01);
5737 }
5738 
5739 void Assembler::vinsertf128(XMMRegister dst, XMMRegister nds, Address src, uint8_t imm8) {
5740   assert(VM_Version::supports_avx(), "");
5741   assert(dst != xnoreg, "sanity");
5742   assert(imm8 <= 0x01, "imm8: %u", imm8);
5743   int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_256bit;
5744   InstructionMark im(this);
5745   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
5746   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
5747   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
5748   emit_int8(0x18);
5749   emit_operand(dst, src);
5750   // 0x00 - insert into lower 128 bits
5751   // 0x01 - insert into upper 128 bits
5752   emit_int8(imm8 & 0x01);
5753 }
5754 
5755 void Assembler::vinsertf32x4(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) {
5756   assert(VM_Version::supports_evex(), "");
5757   assert(imm8 <= 0x03, "imm8: %u", imm8);
5758   InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
5759   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
5760   emit_int8(0x18);
5761   emit_int8((unsigned char)(0xC0 | encode));
5762   // 0x00 - insert into q0 128 bits (0..127)
5763   // 0x01 - insert into q1 128 bits (128..255)
5764   // 0x02 - insert into q2 128 bits (256..383)
5765   // 0x03 - insert into q3 128 bits (384..511)
5766   emit_int8(imm8 & 0x03);
5767 }
5768 
5769 void Assembler::vinsertf32x4(XMMRegister dst, XMMRegister nds, Address src, uint8_t imm8) {
5770   assert(VM_Version::supports_avx(), "");
5771   assert(dst != xnoreg, "sanity");
5772   assert(imm8 <= 0x03, "imm8: %u", imm8);
5773   int vector_len = VM_Version::supports_evex() ? AVX_512bit : AVX_256bit;
5774   InstructionMark im(this);
5775   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
5776   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
5777   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
5778   emit_int8(0x18);
5779   emit_operand(dst, src);
5780   // 0x00 - insert into q0 128 bits (0..127)
5781   // 0x01 - insert into q1 128 bits (128..255)
5782   // 0x02 - insert into q2 128 bits (256..383)
5783   // 0x03 - insert into q3 128 bits (384..511)
5784   emit_int8(imm8 & 0x03);
5785 }
5786 
5787 void Assembler::vinsertf64x4(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) {
5788   assert(VM_Version::supports_evex(), "");
5789   assert(imm8 <= 0x01, "imm8: %u", imm8);
5790   InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
5791   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
5792   emit_int8(0x1A);
5793   emit_int8((unsigned char)(0xC0 | encode));
5794   // 0x00 - insert into lower 256 bits
5795   // 0x01 - insert into upper 256 bits
5796   emit_int8(imm8 & 0x01);
5797 }
5798 
5799 void Assembler::vinsertf64x4(XMMRegister dst, XMMRegister nds, Address src, uint8_t imm8) {
5800   assert(VM_Version::supports_evex(), "");
5801   assert(dst != xnoreg, "sanity");
5802   assert(imm8 <= 0x01, "imm8: %u", imm8);
5803   InstructionMark im(this);
5804   InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
5805   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_64bit);
5806   vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
5807   emit_int8(0x1A);
5808   emit_operand(dst, src);
5809   // 0x00 - insert into lower 256 bits
5810   // 0x01 - insert into upper 256 bits
5811   emit_int8(imm8 & 0x01);
5812 }
5813 
5814 
5815 // vextracti forms
5816 
5817 void Assembler::vextracti128(XMMRegister dst, XMMRegister src, uint8_t imm8) {
5818   assert(VM_Version::supports_avx(), "");
5819   assert(imm8 <= 0x01, "imm8: %u", imm8);
5820   int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_256bit;
5821   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
5822   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
5823   emit_int8(0x39);
5824   emit_int8((unsigned char)(0xC0 | encode));
5825   // 0x00 - extract from lower 128 bits
5826   // 0x01 - extract from upper 128 bits
5827   emit_int8(imm8 & 0x01);
5828 }
5829 
5830 void Assembler::vextracti128(Address dst, XMMRegister src, uint8_t imm8) {
5831   assert(VM_Version::supports_avx2(), "");
5832   assert(src != xnoreg, "sanity");
5833   assert(imm8 <= 0x01, "imm8: %u", imm8);
5834   int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_256bit;
5835   InstructionMark im(this);
5836   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
5837   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
5838   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
5839   emit_int8(0x39);
5840   emit_operand(src, dst);
5841   // 0x00 - extract from lower 128 bits
5842   // 0x01 - extract from upper 128 bits
5843   emit_int8(imm8 & 0x01);
5844 }
5845 
5846 void Assembler::vextracti32x4(XMMRegister dst, XMMRegister src, uint8_t imm8) {
5847   assert(VM_Version::supports_avx(), "");
5848   assert(imm8 <= 0x03, "imm8: %u", imm8);
5849   int vector_len = VM_Version::supports_evex() ? AVX_512bit : AVX_256bit;
5850   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
5851   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
5852   emit_int8(0x39);
5853   emit_int8((unsigned char)(0xC0 | encode));
5854   // 0x00 - extract from bits 127:0
5855   // 0x01 - extract from bits 255:128
5856   // 0x02 - extract from bits 383:256
5857   // 0x03 - extract from bits 511:384
5858   emit_int8(imm8 & 0x03);
5859 }
5860 
5861 void Assembler::vextracti32x4(Address dst, XMMRegister src, uint8_t imm8) {
5862   assert(VM_Version::supports_evex(), "");
5863   assert(src != xnoreg, "sanity");
5864   assert(imm8 <= 0x03, "imm8: %u", imm8);
5865   InstructionMark im(this);
5866   InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
5867   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
5868   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
5869   emit_int8(0x39);
5870   emit_operand(src, dst);
5871   // 0x00 - extract from bits 127:0
5872   // 0x01 - extract from bits 255:128
5873   // 0x02 - extract from bits 383:256
5874   // 0x03 - extract from bits 511:384
5875   emit_int8(imm8 & 0x03);
5876 }
5877 
5878 void Assembler::vextracti64x2(XMMRegister dst, XMMRegister src, uint8_t imm8) {
5879   assert(VM_Version::supports_evex(), "");
5880   assert(imm8 <= 0x03, "imm8: %u", imm8);
5881   InstructionAttr attributes(AVX_512bit, /* vex_w */ !_legacy_mode_dq, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
5882   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
5883   emit_int8(0x39);
5884   emit_int8((unsigned char)(0xC0 | encode));
5885   // 0x00 - extract from bits 127:0
5886   // 0x01 - extract from bits 255:128
5887   // 0x02 - extract from bits 383:256
5888   // 0x03 - extract from bits 511:384
5889   emit_int8(imm8 & 0x03);
5890 }
5891 
5892 void Assembler::vextracti64x4(XMMRegister dst, XMMRegister src, uint8_t imm8) {
5893   assert(VM_Version::supports_evex(), "");
5894   assert(imm8 <= 0x01, "imm8: %u", imm8);
5895   InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
5896   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
5897   emit_int8(0x3B);
5898   emit_int8((unsigned char)(0xC0 | encode));
5899   // 0x00 - extract from lower 256 bits
5900   // 0x01 - extract from upper 256 bits
5901   emit_int8(imm8 & 0x01);
5902 }
5903 
5904 
5905 // vextractf forms
5906 
5907 void Assembler::vextractf128(XMMRegister dst, XMMRegister src, uint8_t imm8) {
5908   assert(VM_Version::supports_avx(), "");
5909   assert(imm8 <= 0x01, "imm8: %u", imm8);
5910   int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_256bit;
5911   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
5912   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
5913   emit_int8(0x19);
5914   emit_int8((unsigned char)(0xC0 | encode));
5915   // 0x00 - extract from lower 128 bits
5916   // 0x01 - extract from upper 128 bits
5917   emit_int8(imm8 & 0x01);
5918 }
5919 
5920 void Assembler::vextractf128(Address dst, XMMRegister src, uint8_t imm8) {
5921   assert(VM_Version::supports_avx(), "");
5922   assert(src != xnoreg, "sanity");
5923   assert(imm8 <= 0x01, "imm8: %u", imm8);
5924   int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_256bit;
5925   InstructionMark im(this);
5926   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
5927   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
5928   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
5929   emit_int8(0x19);
5930   emit_operand(src, dst);
5931   // 0x00 - extract from lower 128 bits
5932   // 0x01 - extract from upper 128 bits
5933   emit_int8(imm8 & 0x01);
5934 }
5935 
5936 void Assembler::vextractf32x4(XMMRegister dst, XMMRegister src, uint8_t imm8) {
5937   assert(VM_Version::supports_avx(), "");
5938   assert(imm8 <= 0x03, "imm8: %u", imm8);
5939   int vector_len = VM_Version::supports_evex() ? AVX_512bit : AVX_256bit;
5940   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
5941   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
5942   emit_int8(0x19);
5943   emit_int8((unsigned char)(0xC0 | encode));
5944   // 0x00 - extract from bits 127:0
5945   // 0x01 - extract from bits 255:128
5946   // 0x02 - extract from bits 383:256
5947   // 0x03 - extract from bits 511:384
5948   emit_int8(imm8 & 0x03);
5949 }
5950 
5951 void Assembler::vextractf32x4(Address dst, XMMRegister src, uint8_t imm8) {
5952   assert(VM_Version::supports_evex(), "");
5953   assert(src != xnoreg, "sanity");
5954   assert(imm8 <= 0x03, "imm8: %u", imm8);
5955   InstructionMark im(this);
5956   InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
5957   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
5958   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
5959   emit_int8(0x19);
5960   emit_operand(src, dst);
5961   // 0x00 - extract from bits 127:0
5962   // 0x01 - extract from bits 255:128
5963   // 0x02 - extract from bits 383:256
5964   // 0x03 - extract from bits 511:384
5965   emit_int8(imm8 & 0x03);
5966 }
5967 
5968 void Assembler::vextractf64x2(XMMRegister dst, XMMRegister src, uint8_t imm8) {
5969   assert(VM_Version::supports_evex(), "");
5970   assert(imm8 <= 0x03, "imm8: %u", imm8);
5971   InstructionAttr attributes(AVX_512bit, /* vex_w */ !_legacy_mode_dq, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
5972   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
5973   emit_int8(0x19);
5974   emit_int8((unsigned char)(0xC0 | encode));
5975   // 0x00 - extract from bits 127:0
5976   // 0x01 - extract from bits 255:128
5977   // 0x02 - extract from bits 383:256
5978   // 0x03 - extract from bits 511:384
5979   emit_int8(imm8 & 0x03);
5980 }
5981 
5982 void Assembler::vextractf64x4(XMMRegister dst, XMMRegister src, uint8_t imm8) {
5983   assert(VM_Version::supports_evex(), "");
5984   assert(imm8 <= 0x01, "imm8: %u", imm8);
5985   InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
5986   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
5987   emit_int8(0x1B);
5988   emit_int8((unsigned char)(0xC0 | encode));
5989   // 0x00 - extract from lower 256 bits
5990   // 0x01 - extract from upper 256 bits
5991   emit_int8(imm8 & 0x01);
5992 }
5993 
5994 void Assembler::vextractf64x4(Address dst, XMMRegister src, uint8_t imm8) {
5995   assert(VM_Version::supports_evex(), "");
5996   assert(src != xnoreg, "sanity");
5997   assert(imm8 <= 0x01, "imm8: %u", imm8);
5998   InstructionMark im(this);
5999   InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6000   attributes.set_address_attributes(/* tuple_type */ EVEX_T4,/* input_size_in_bits */  EVEX_64bit);
6001   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6002   emit_int8(0x1B);
6003   emit_operand(src, dst);
6004   // 0x00 - extract from lower 256 bits
6005   // 0x01 - extract from upper 256 bits
6006   emit_int8(imm8 & 0x01);
6007 }
6008 
6009 
6010 // legacy word/dword replicate
6011 void Assembler::vpbroadcastw(XMMRegister dst, XMMRegister src) {
6012   assert(VM_Version::supports_avx2(), "");
6013   InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
6014   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6015   emit_int8(0x79);
6016   emit_int8((unsigned char)(0xC0 | encode));
6017 }
6018 
6019 void Assembler::vpbroadcastd(XMMRegister dst, XMMRegister src) {
6020   assert(VM_Version::supports_avx2(), "");
6021   InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6022   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6023   emit_int8(0x58);
6024   emit_int8((unsigned char)(0xC0 | encode));
6025 }
6026 
6027 
6028 // xmm/mem sourced byte/word/dword/qword replicate
6029 
6030 // duplicate 1-byte integer data from src into programmed locations in dest : requires AVX512BW and AVX512VL
6031 void Assembler::evpbroadcastb(XMMRegister dst, XMMRegister src, int vector_len) {
6032   assert(VM_Version::supports_evex(), "");
6033   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
6034   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6035   emit_int8(0x78);
6036   emit_int8((unsigned char)(0xC0 | encode));
6037 }
6038 
6039 void Assembler::evpbroadcastb(XMMRegister dst, Address src, int vector_len) {
6040   assert(VM_Version::supports_evex(), "");
6041   assert(dst != xnoreg, "sanity");
6042   InstructionMark im(this);
6043   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
6044   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_8bit);
6045   // swap src<->dst for encoding
6046   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6047   emit_int8(0x78);
6048   emit_operand(dst, src);
6049 }
6050 
6051 // duplicate 2-byte integer data from src into programmed locations in dest : requires AVX512BW and AVX512VL
6052 void Assembler::evpbroadcastw(XMMRegister dst, XMMRegister src, int vector_len) {
6053   assert(VM_Version::supports_evex(), "");
6054   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
6055   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6056   emit_int8(0x79);
6057   emit_int8((unsigned char)(0xC0 | encode));
6058 }
6059 
6060 void Assembler::evpbroadcastw(XMMRegister dst, Address src, int vector_len) {
6061   assert(VM_Version::supports_evex(), "");
6062   assert(dst != xnoreg, "sanity");
6063   InstructionMark im(this);
6064   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
6065   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_16bit);
6066   // swap src<->dst for encoding
6067   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6068   emit_int8(0x79);
6069   emit_operand(dst, src);
6070 }
6071 
6072 // duplicate 4-byte integer data from src into programmed locations in dest : requires AVX512VL
6073 void Assembler::evpbroadcastd(XMMRegister dst, XMMRegister src, int vector_len) {
6074   assert(VM_Version::supports_evex(), "");
6075   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6076   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6077   emit_int8(0x58);
6078   emit_int8((unsigned char)(0xC0 | encode));
6079 }
6080 
6081 void Assembler::evpbroadcastd(XMMRegister dst, Address src, int vector_len) {
6082   assert(VM_Version::supports_evex(), "");
6083   assert(dst != xnoreg, "sanity");
6084   InstructionMark im(this);
6085   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6086   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
6087   // swap src<->dst for encoding
6088   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6089   emit_int8(0x58);
6090   emit_operand(dst, src);
6091 }
6092 
6093 // duplicate 8-byte integer data from src into programmed locations in dest : requires AVX512VL
6094 void Assembler::evpbroadcastq(XMMRegister dst, XMMRegister src, int vector_len) {
6095   assert(VM_Version::supports_evex(), "");
6096   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6097   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6098   emit_int8(0x59);
6099   emit_int8((unsigned char)(0xC0 | encode));
6100 }
6101 
6102 void Assembler::evpbroadcastq(XMMRegister dst, Address src, int vector_len) {
6103   assert(VM_Version::supports_evex(), "");
6104   assert(dst != xnoreg, "sanity");
6105   InstructionMark im(this);
6106   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6107   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
6108   // swap src<->dst for encoding
6109   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6110   emit_int8(0x59);
6111   emit_operand(dst, src);
6112 }
6113 
6114 
6115 // scalar single/double precision replicate
6116 
6117 // duplicate single precision data from src into programmed locations in dest : requires AVX512VL
6118 void Assembler::evpbroadcastss(XMMRegister dst, XMMRegister src, int vector_len) {
6119   assert(VM_Version::supports_evex(), "");
6120   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6121   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6122   emit_int8(0x18);
6123   emit_int8((unsigned char)(0xC0 | encode));
6124 }
6125 
6126 void Assembler::evpbroadcastss(XMMRegister dst, Address src, int vector_len) {
6127   assert(VM_Version::supports_evex(), "");
6128   assert(dst != xnoreg, "sanity");
6129   InstructionMark im(this);
6130   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6131   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
6132   // swap src<->dst for encoding
6133   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6134   emit_int8(0x18);
6135   emit_operand(dst, src);
6136 }
6137 
6138 // duplicate double precision data from src into programmed locations in dest : requires AVX512VL
6139 void Assembler::evpbroadcastsd(XMMRegister dst, XMMRegister src, int vector_len) {
6140   assert(VM_Version::supports_evex(), "");
6141   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6142   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6143   emit_int8(0x19);
6144   emit_int8((unsigned char)(0xC0 | encode));
6145 }
6146 
6147 void Assembler::evpbroadcastsd(XMMRegister dst, Address src, int vector_len) {
6148   assert(VM_Version::supports_evex(), "");
6149   assert(dst != xnoreg, "sanity");
6150   InstructionMark im(this);
6151   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6152   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
6153   // swap src<->dst for encoding
6154   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6155   emit_int8(0x19);
6156   emit_operand(dst, src);
6157 }
6158 
6159 
6160 // gpr source broadcast forms
6161 
6162 // duplicate 1-byte integer data from src into programmed locations in dest : requires AVX512BW and AVX512VL
6163 void Assembler::evpbroadcastb(XMMRegister dst, Register src, int vector_len) {
6164   assert(VM_Version::supports_evex(), "");
6165   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
6166   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6167   if (attributes.is_evex_instruction()) {
6168     emit_int8(0x7A);
6169   } else {
6170     emit_int8(0x78);
6171   }
6172   emit_int8((unsigned char)(0xC0 | encode));
6173 }
6174 
6175 // duplicate 2-byte integer data from src into programmed locations in dest : requires AVX512BW and AVX512VL
6176 void Assembler::evpbroadcastw(XMMRegister dst, Register src, int vector_len) {
6177   assert(VM_Version::supports_evex(), "");
6178   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
6179   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6180   if (attributes.is_evex_instruction()) {
6181     emit_int8(0x7B);
6182   } else {
6183     emit_int8(0x79);
6184   }
6185   emit_int8((unsigned char)(0xC0 | encode));
6186 }
6187 
6188 // duplicate 4-byte integer data from src into programmed locations in dest : requires AVX512VL
6189 void Assembler::evpbroadcastd(XMMRegister dst, Register src, int vector_len) {
6190   assert(VM_Version::supports_evex(), "");
6191   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6192   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6193   if (attributes.is_evex_instruction()) {
6194     emit_int8(0x7C);
6195   } else {
6196     emit_int8(0x58);
6197   }
6198   emit_int8((unsigned char)(0xC0 | encode));
6199 }
6200 
6201 // duplicate 8-byte integer data from src into programmed locations in dest : requires AVX512VL
6202 void Assembler::evpbroadcastq(XMMRegister dst, Register src, int vector_len) {
6203   assert(VM_Version::supports_evex(), "");
6204   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6205   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6206   if (attributes.is_evex_instruction()) {
6207     emit_int8(0x7C);
6208   } else {
6209     emit_int8(0x59);
6210   }
6211   emit_int8((unsigned char)(0xC0 | encode));
6212 }
6213 
6214 
6215 // Carry-Less Multiplication Quadword
6216 void Assembler::pclmulqdq(XMMRegister dst, XMMRegister src, int mask) {
6217   assert(VM_Version::supports_clmul(), "");
6218   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
6219   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6220   emit_int8(0x44);
6221   emit_int8((unsigned char)(0xC0 | encode));
6222   emit_int8((unsigned char)mask);
6223 }
6224 
6225 // Carry-Less Multiplication Quadword
6226 void Assembler::vpclmulqdq(XMMRegister dst, XMMRegister nds, XMMRegister src, int mask) {
6227   assert(VM_Version::supports_avx() && VM_Version::supports_clmul(), "");
6228   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
6229   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6230   emit_int8(0x44);
6231   emit_int8((unsigned char)(0xC0 | encode));
6232   emit_int8((unsigned char)mask);
6233 }
6234 
6235 void Assembler::vzeroupper() {
6236   assert(VM_Version::supports_avx(), "");
6237   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
6238   (void)vex_prefix_and_encode(0, 0, 0, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
6239   emit_int8(0x77);
6240 }
6241 
6242 
6243 #ifndef _LP64
6244 // 32bit only pieces of the assembler
6245 
6246 void Assembler::cmp_literal32(Register src1, int32_t imm32, RelocationHolder const& rspec) {
6247   // NO PREFIX AS NEVER 64BIT
6248   InstructionMark im(this);
6249   emit_int8((unsigned char)0x81);
6250   emit_int8((unsigned char)(0xF8 | src1->encoding()));
6251   emit_data(imm32, rspec, 0);
6252 }
6253 
6254 void Assembler::cmp_literal32(Address src1, int32_t imm32, RelocationHolder const& rspec) {
6255   // NO PREFIX AS NEVER 64BIT (not even 32bit versions of 64bit regs
6256   InstructionMark im(this);
6257   emit_int8((unsigned char)0x81);
6258   emit_operand(rdi, src1);
6259   emit_data(imm32, rspec, 0);
6260 }
6261 
6262 // The 64-bit (32bit platform) cmpxchg compares the value at adr with the contents of rdx:rax,
6263 // and stores rcx:rbx into adr if so; otherwise, the value at adr is loaded
6264 // into rdx:rax.  The ZF is set if the compared values were equal, and cleared otherwise.
6265 void Assembler::cmpxchg8(Address adr) {
6266   InstructionMark im(this);
6267   emit_int8(0x0F);
6268   emit_int8((unsigned char)0xC7);
6269   emit_operand(rcx, adr);
6270 }
6271 
6272 void Assembler::decl(Register dst) {
6273   // Don't use it directly. Use MacroAssembler::decrementl() instead.
6274  emit_int8(0x48 | dst->encoding());
6275 }
6276 
6277 #endif // _LP64
6278 
6279 // 64bit typically doesn't use the x87 but needs to for the trig funcs
6280 
6281 void Assembler::fabs() {
6282   emit_int8((unsigned char)0xD9);
6283   emit_int8((unsigned char)0xE1);
6284 }
6285 
6286 void Assembler::fadd(int i) {
6287   emit_farith(0xD8, 0xC0, i);
6288 }
6289 
6290 void Assembler::fadd_d(Address src) {
6291   InstructionMark im(this);
6292   emit_int8((unsigned char)0xDC);
6293   emit_operand32(rax, src);
6294 }
6295 
6296 void Assembler::fadd_s(Address src) {
6297   InstructionMark im(this);
6298   emit_int8((unsigned char)0xD8);
6299   emit_operand32(rax, src);
6300 }
6301 
6302 void Assembler::fadda(int i) {
6303   emit_farith(0xDC, 0xC0, i);
6304 }
6305 
6306 void Assembler::faddp(int i) {
6307   emit_farith(0xDE, 0xC0, i);
6308 }
6309 
6310 void Assembler::fchs() {
6311   emit_int8((unsigned char)0xD9);
6312   emit_int8((unsigned char)0xE0);
6313 }
6314 
6315 void Assembler::fcom(int i) {
6316   emit_farith(0xD8, 0xD0, i);
6317 }
6318 
6319 void Assembler::fcomp(int i) {
6320   emit_farith(0xD8, 0xD8, i);
6321 }
6322 
6323 void Assembler::fcomp_d(Address src) {
6324   InstructionMark im(this);
6325   emit_int8((unsigned char)0xDC);
6326   emit_operand32(rbx, src);
6327 }
6328 
6329 void Assembler::fcomp_s(Address src) {
6330   InstructionMark im(this);
6331   emit_int8((unsigned char)0xD8);
6332   emit_operand32(rbx, src);
6333 }
6334 
6335 void Assembler::fcompp() {
6336   emit_int8((unsigned char)0xDE);
6337   emit_int8((unsigned char)0xD9);
6338 }
6339 
6340 void Assembler::fcos() {
6341   emit_int8((unsigned char)0xD9);
6342   emit_int8((unsigned char)0xFF);
6343 }
6344 
6345 void Assembler::fdecstp() {
6346   emit_int8((unsigned char)0xD9);
6347   emit_int8((unsigned char)0xF6);
6348 }
6349 
6350 void Assembler::fdiv(int i) {
6351   emit_farith(0xD8, 0xF0, i);
6352 }
6353 
6354 void Assembler::fdiv_d(Address src) {
6355   InstructionMark im(this);
6356   emit_int8((unsigned char)0xDC);
6357   emit_operand32(rsi, src);
6358 }
6359 
6360 void Assembler::fdiv_s(Address src) {
6361   InstructionMark im(this);
6362   emit_int8((unsigned char)0xD8);
6363   emit_operand32(rsi, src);
6364 }
6365 
6366 void Assembler::fdiva(int i) {
6367   emit_farith(0xDC, 0xF8, i);
6368 }
6369 
6370 // Note: The Intel manual (Pentium Processor User's Manual, Vol.3, 1994)
6371 //       is erroneous for some of the floating-point instructions below.
6372 
6373 void Assembler::fdivp(int i) {
6374   emit_farith(0xDE, 0xF8, i);                    // ST(0) <- ST(0) / ST(1) and pop (Intel manual wrong)
6375 }
6376 
6377 void Assembler::fdivr(int i) {
6378   emit_farith(0xD8, 0xF8, i);
6379 }
6380 
6381 void Assembler::fdivr_d(Address src) {
6382   InstructionMark im(this);
6383   emit_int8((unsigned char)0xDC);
6384   emit_operand32(rdi, src);
6385 }
6386 
6387 void Assembler::fdivr_s(Address src) {
6388   InstructionMark im(this);
6389   emit_int8((unsigned char)0xD8);
6390   emit_operand32(rdi, src);
6391 }
6392 
6393 void Assembler::fdivra(int i) {
6394   emit_farith(0xDC, 0xF0, i);
6395 }
6396 
6397 void Assembler::fdivrp(int i) {
6398   emit_farith(0xDE, 0xF0, i);                    // ST(0) <- ST(1) / ST(0) and pop (Intel manual wrong)
6399 }
6400 
6401 void Assembler::ffree(int i) {
6402   emit_farith(0xDD, 0xC0, i);
6403 }
6404 
6405 void Assembler::fild_d(Address adr) {
6406   InstructionMark im(this);
6407   emit_int8((unsigned char)0xDF);
6408   emit_operand32(rbp, adr);
6409 }
6410 
6411 void Assembler::fild_s(Address adr) {
6412   InstructionMark im(this);
6413   emit_int8((unsigned char)0xDB);
6414   emit_operand32(rax, adr);
6415 }
6416 
6417 void Assembler::fincstp() {
6418   emit_int8((unsigned char)0xD9);
6419   emit_int8((unsigned char)0xF7);
6420 }
6421 
6422 void Assembler::finit() {
6423   emit_int8((unsigned char)0x9B);
6424   emit_int8((unsigned char)0xDB);
6425   emit_int8((unsigned char)0xE3);
6426 }
6427 
6428 void Assembler::fist_s(Address adr) {
6429   InstructionMark im(this);
6430   emit_int8((unsigned char)0xDB);
6431   emit_operand32(rdx, adr);
6432 }
6433 
6434 void Assembler::fistp_d(Address adr) {
6435   InstructionMark im(this);
6436   emit_int8((unsigned char)0xDF);
6437   emit_operand32(rdi, adr);
6438 }
6439 
6440 void Assembler::fistp_s(Address adr) {
6441   InstructionMark im(this);
6442   emit_int8((unsigned char)0xDB);
6443   emit_operand32(rbx, adr);
6444 }
6445 
6446 void Assembler::fld1() {
6447   emit_int8((unsigned char)0xD9);
6448   emit_int8((unsigned char)0xE8);
6449 }
6450 
6451 void Assembler::fld_d(Address adr) {
6452   InstructionMark im(this);
6453   emit_int8((unsigned char)0xDD);
6454   emit_operand32(rax, adr);
6455 }
6456 
6457 void Assembler::fld_s(Address adr) {
6458   InstructionMark im(this);
6459   emit_int8((unsigned char)0xD9);
6460   emit_operand32(rax, adr);
6461 }
6462 
6463 
6464 void Assembler::fld_s(int index) {
6465   emit_farith(0xD9, 0xC0, index);
6466 }
6467 
6468 void Assembler::fld_x(Address adr) {
6469   InstructionMark im(this);
6470   emit_int8((unsigned char)0xDB);
6471   emit_operand32(rbp, adr);
6472 }
6473 
6474 void Assembler::fldcw(Address src) {
6475   InstructionMark im(this);
6476   emit_int8((unsigned char)0xD9);
6477   emit_operand32(rbp, src);
6478 }
6479 
6480 void Assembler::fldenv(Address src) {
6481   InstructionMark im(this);
6482   emit_int8((unsigned char)0xD9);
6483   emit_operand32(rsp, src);
6484 }
6485 
6486 void Assembler::fldlg2() {
6487   emit_int8((unsigned char)0xD9);
6488   emit_int8((unsigned char)0xEC);
6489 }
6490 
6491 void Assembler::fldln2() {
6492   emit_int8((unsigned char)0xD9);
6493   emit_int8((unsigned char)0xED);
6494 }
6495 
6496 void Assembler::fldz() {
6497   emit_int8((unsigned char)0xD9);
6498   emit_int8((unsigned char)0xEE);
6499 }
6500 
6501 void Assembler::flog() {
6502   fldln2();
6503   fxch();
6504   fyl2x();
6505 }
6506 
6507 void Assembler::flog10() {
6508   fldlg2();
6509   fxch();
6510   fyl2x();
6511 }
6512 
6513 void Assembler::fmul(int i) {
6514   emit_farith(0xD8, 0xC8, i);
6515 }
6516 
6517 void Assembler::fmul_d(Address src) {
6518   InstructionMark im(this);
6519   emit_int8((unsigned char)0xDC);
6520   emit_operand32(rcx, src);
6521 }
6522 
6523 void Assembler::fmul_s(Address src) {
6524   InstructionMark im(this);
6525   emit_int8((unsigned char)0xD8);
6526   emit_operand32(rcx, src);
6527 }
6528 
6529 void Assembler::fmula(int i) {
6530   emit_farith(0xDC, 0xC8, i);
6531 }
6532 
6533 void Assembler::fmulp(int i) {
6534   emit_farith(0xDE, 0xC8, i);
6535 }
6536 
6537 void Assembler::fnsave(Address dst) {
6538   InstructionMark im(this);
6539   emit_int8((unsigned char)0xDD);
6540   emit_operand32(rsi, dst);
6541 }
6542 
6543 void Assembler::fnstcw(Address src) {
6544   InstructionMark im(this);
6545   emit_int8((unsigned char)0x9B);
6546   emit_int8((unsigned char)0xD9);
6547   emit_operand32(rdi, src);
6548 }
6549 
6550 void Assembler::fnstsw_ax() {
6551   emit_int8((unsigned char)0xDF);
6552   emit_int8((unsigned char)0xE0);
6553 }
6554 
6555 void Assembler::fprem() {
6556   emit_int8((unsigned char)0xD9);
6557   emit_int8((unsigned char)0xF8);
6558 }
6559 
6560 void Assembler::fprem1() {
6561   emit_int8((unsigned char)0xD9);
6562   emit_int8((unsigned char)0xF5);
6563 }
6564 
6565 void Assembler::frstor(Address src) {
6566   InstructionMark im(this);
6567   emit_int8((unsigned char)0xDD);
6568   emit_operand32(rsp, src);
6569 }
6570 
6571 void Assembler::fsin() {
6572   emit_int8((unsigned char)0xD9);
6573   emit_int8((unsigned char)0xFE);
6574 }
6575 
6576 void Assembler::fsqrt() {
6577   emit_int8((unsigned char)0xD9);
6578   emit_int8((unsigned char)0xFA);
6579 }
6580 
6581 void Assembler::fst_d(Address adr) {
6582   InstructionMark im(this);
6583   emit_int8((unsigned char)0xDD);
6584   emit_operand32(rdx, adr);
6585 }
6586 
6587 void Assembler::fst_s(Address adr) {
6588   InstructionMark im(this);
6589   emit_int8((unsigned char)0xD9);
6590   emit_operand32(rdx, adr);
6591 }
6592 
6593 void Assembler::fstp_d(Address adr) {
6594   InstructionMark im(this);
6595   emit_int8((unsigned char)0xDD);
6596   emit_operand32(rbx, adr);
6597 }
6598 
6599 void Assembler::fstp_d(int index) {
6600   emit_farith(0xDD, 0xD8, index);
6601 }
6602 
6603 void Assembler::fstp_s(Address adr) {
6604   InstructionMark im(this);
6605   emit_int8((unsigned char)0xD9);
6606   emit_operand32(rbx, adr);
6607 }
6608 
6609 void Assembler::fstp_x(Address adr) {
6610   InstructionMark im(this);
6611   emit_int8((unsigned char)0xDB);
6612   emit_operand32(rdi, adr);
6613 }
6614 
6615 void Assembler::fsub(int i) {
6616   emit_farith(0xD8, 0xE0, i);
6617 }
6618 
6619 void Assembler::fsub_d(Address src) {
6620   InstructionMark im(this);
6621   emit_int8((unsigned char)0xDC);
6622   emit_operand32(rsp, src);
6623 }
6624 
6625 void Assembler::fsub_s(Address src) {
6626   InstructionMark im(this);
6627   emit_int8((unsigned char)0xD8);
6628   emit_operand32(rsp, src);
6629 }
6630 
6631 void Assembler::fsuba(int i) {
6632   emit_farith(0xDC, 0xE8, i);
6633 }
6634 
6635 void Assembler::fsubp(int i) {
6636   emit_farith(0xDE, 0xE8, i);                    // ST(0) <- ST(0) - ST(1) and pop (Intel manual wrong)
6637 }
6638 
6639 void Assembler::fsubr(int i) {
6640   emit_farith(0xD8, 0xE8, i);
6641 }
6642 
6643 void Assembler::fsubr_d(Address src) {
6644   InstructionMark im(this);
6645   emit_int8((unsigned char)0xDC);
6646   emit_operand32(rbp, src);
6647 }
6648 
6649 void Assembler::fsubr_s(Address src) {
6650   InstructionMark im(this);
6651   emit_int8((unsigned char)0xD8);
6652   emit_operand32(rbp, src);
6653 }
6654 
6655 void Assembler::fsubra(int i) {
6656   emit_farith(0xDC, 0xE0, i);
6657 }
6658 
6659 void Assembler::fsubrp(int i) {
6660   emit_farith(0xDE, 0xE0, i);                    // ST(0) <- ST(1) - ST(0) and pop (Intel manual wrong)
6661 }
6662 
6663 void Assembler::ftan() {
6664   emit_int8((unsigned char)0xD9);
6665   emit_int8((unsigned char)0xF2);
6666   emit_int8((unsigned char)0xDD);
6667   emit_int8((unsigned char)0xD8);
6668 }
6669 
6670 void Assembler::ftst() {
6671   emit_int8((unsigned char)0xD9);
6672   emit_int8((unsigned char)0xE4);
6673 }
6674 
6675 void Assembler::fucomi(int i) {
6676   // make sure the instruction is supported (introduced for P6, together with cmov)
6677   guarantee(VM_Version::supports_cmov(), "illegal instruction");
6678   emit_farith(0xDB, 0xE8, i);
6679 }
6680 
6681 void Assembler::fucomip(int i) {
6682   // make sure the instruction is supported (introduced for P6, together with cmov)
6683   guarantee(VM_Version::supports_cmov(), "illegal instruction");
6684   emit_farith(0xDF, 0xE8, i);
6685 }
6686 
6687 void Assembler::fwait() {
6688   emit_int8((unsigned char)0x9B);
6689 }
6690 
6691 void Assembler::fxch(int i) {
6692   emit_farith(0xD9, 0xC8, i);
6693 }
6694 
6695 void Assembler::fyl2x() {
6696   emit_int8((unsigned char)0xD9);
6697   emit_int8((unsigned char)0xF1);
6698 }
6699 
6700 void Assembler::frndint() {
6701   emit_int8((unsigned char)0xD9);
6702   emit_int8((unsigned char)0xFC);
6703 }
6704 
6705 void Assembler::f2xm1() {
6706   emit_int8((unsigned char)0xD9);
6707   emit_int8((unsigned char)0xF0);
6708 }
6709 
6710 void Assembler::fldl2e() {
6711   emit_int8((unsigned char)0xD9);
6712   emit_int8((unsigned char)0xEA);
6713 }
6714 
6715 // SSE SIMD prefix byte values corresponding to VexSimdPrefix encoding.
6716 static int simd_pre[4] = { 0, 0x66, 0xF3, 0xF2 };
6717 // SSE opcode second byte values (first is 0x0F) corresponding to VexOpcode encoding.
6718 static int simd_opc[4] = { 0,    0, 0x38, 0x3A };
6719 
6720 // Generate SSE legacy REX prefix and SIMD opcode based on VEX encoding.
6721 void Assembler::rex_prefix(Address adr, XMMRegister xreg, VexSimdPrefix pre, VexOpcode opc, bool rex_w) {
6722   if (pre > 0) {
6723     emit_int8(simd_pre[pre]);
6724   }
6725   if (rex_w) {
6726     prefixq(adr, xreg);
6727   } else {
6728     prefix(adr, xreg);
6729   }
6730   if (opc > 0) {
6731     emit_int8(0x0F);
6732     int opc2 = simd_opc[opc];
6733     if (opc2 > 0) {
6734       emit_int8(opc2);
6735     }
6736   }
6737 }
6738 
6739 int Assembler::rex_prefix_and_encode(int dst_enc, int src_enc, VexSimdPrefix pre, VexOpcode opc, bool rex_w) {
6740   if (pre > 0) {
6741     emit_int8(simd_pre[pre]);
6742   }
6743   int encode = (rex_w) ? prefixq_and_encode(dst_enc, src_enc) : prefix_and_encode(dst_enc, src_enc);
6744   if (opc > 0) {
6745     emit_int8(0x0F);
6746     int opc2 = simd_opc[opc];
6747     if (opc2 > 0) {
6748       emit_int8(opc2);
6749     }
6750   }
6751   return encode;
6752 }
6753 
6754 
6755 void Assembler::vex_prefix(bool vex_r, bool vex_b, bool vex_x, int nds_enc, VexSimdPrefix pre, VexOpcode opc) {
6756   int vector_len = _attributes->get_vector_len();
6757   bool vex_w = _attributes->is_rex_vex_w();
6758   if (vex_b || vex_x || vex_w || (opc == VEX_OPCODE_0F_38) || (opc == VEX_OPCODE_0F_3A)) {
6759     prefix(VEX_3bytes);
6760 
6761     int byte1 = (vex_r ? VEX_R : 0) | (vex_x ? VEX_X : 0) | (vex_b ? VEX_B : 0);
6762     byte1 = (~byte1) & 0xE0;
6763     byte1 |= opc;
6764     emit_int8(byte1);
6765 
6766     int byte2 = ((~nds_enc) & 0xf) << 3;
6767     byte2 |= (vex_w ? VEX_W : 0) | ((vector_len > 0) ? 4 : 0) | pre;
6768     emit_int8(byte2);
6769   } else {
6770     prefix(VEX_2bytes);
6771 
6772     int byte1 = vex_r ? VEX_R : 0;
6773     byte1 = (~byte1) & 0x80;
6774     byte1 |= ((~nds_enc) & 0xf) << 3;
6775     byte1 |= ((vector_len > 0 ) ? 4 : 0) | pre;
6776     emit_int8(byte1);
6777   }
6778 }
6779 
6780 // This is a 4 byte encoding
6781 void Assembler::evex_prefix(bool vex_r, bool vex_b, bool vex_x, bool evex_r, bool evex_v, int nds_enc, VexSimdPrefix pre, VexOpcode opc){
6782   // EVEX 0x62 prefix
6783   prefix(EVEX_4bytes);
6784   bool vex_w = _attributes->is_rex_vex_w();
6785   int evex_encoding = (vex_w ? VEX_W : 0);
6786   // EVEX.b is not currently used for broadcast of single element or data rounding modes
6787   _attributes->set_evex_encoding(evex_encoding);
6788 
6789   // P0: byte 2, initialized to RXBR`00mm
6790   // instead of not'd
6791   int byte2 = (vex_r ? VEX_R : 0) | (vex_x ? VEX_X : 0) | (vex_b ? VEX_B : 0) | (evex_r ? EVEX_Rb : 0);
6792   byte2 = (~byte2) & 0xF0;
6793   // confine opc opcode extensions in mm bits to lower two bits
6794   // of form {0F, 0F_38, 0F_3A}
6795   byte2 |= opc;
6796   emit_int8(byte2);
6797 
6798   // P1: byte 3 as Wvvvv1pp
6799   int byte3 = ((~nds_enc) & 0xf) << 3;
6800   // p[10] is always 1
6801   byte3 |= EVEX_F;
6802   byte3 |= (vex_w & 1) << 7;
6803   // confine pre opcode extensions in pp bits to lower two bits
6804   // of form {66, F3, F2}
6805   byte3 |= pre;
6806   emit_int8(byte3);
6807 
6808   // P2: byte 4 as zL'Lbv'aaa
6809   int byte4 = (_attributes->is_no_reg_mask()) ? 0 : 1; // kregs are implemented in the low 3 bits as aaa (hard code k1, it will be initialized for now)
6810   // EVEX.v` for extending EVEX.vvvv or VIDX
6811   byte4 |= (evex_v ? 0: EVEX_V);
6812   // third EXEC.b for broadcast actions
6813   byte4 |= (_attributes->is_extended_context() ? EVEX_Rb : 0);
6814   // fourth EVEX.L'L for vector length : 0 is 128, 1 is 256, 2 is 512, currently we do not support 1024
6815   byte4 |= ((_attributes->get_vector_len())& 0x3) << 5;
6816   // last is EVEX.z for zero/merge actions
6817   byte4 |= (_attributes->is_clear_context() ? EVEX_Z : 0);
6818   emit_int8(byte4);
6819 }
6820 
6821 void Assembler::vex_prefix(Address adr, int nds_enc, int xreg_enc, VexSimdPrefix pre, VexOpcode opc, InstructionAttr *attributes) {
6822   bool vex_r = ((xreg_enc & 8) == 8) ? 1 : 0;
6823   bool vex_b = adr.base_needs_rex();
6824   bool vex_x = adr.index_needs_rex();
6825   set_attributes(attributes);
6826   attributes->set_current_assembler(this);
6827 
6828   // if vector length is turned off, revert to AVX for vectors smaller than 512-bit
6829   if (UseAVX > 2 && _legacy_mode_vl && attributes->uses_vl()) {
6830     switch (attributes->get_vector_len()) {
6831     case AVX_128bit:
6832     case AVX_256bit:
6833       attributes->set_is_legacy_mode();
6834       break;
6835     }
6836   }
6837 
6838   // For pure EVEX check and see if this instruction
6839   // is allowed in legacy mode and has resources which will
6840   // fit in it.  Pure EVEX instructions will use set_is_evex_instruction in their definition,
6841   // else that field is set when we encode to EVEX
6842   if (UseAVX > 2 && !attributes->is_legacy_mode() &&
6843       !_is_managed && !attributes->is_evex_instruction()) {
6844     if (!_legacy_mode_vl && attributes->get_vector_len() != AVX_512bit) {
6845       bool check_register_bank = NOT_IA32(true) IA32_ONLY(false);
6846       if (check_register_bank) {
6847         // check nds_enc and xreg_enc for upper bank usage
6848         if (nds_enc < 16 && xreg_enc < 16) {
6849           attributes->set_is_legacy_mode();
6850         }
6851       } else {
6852         attributes->set_is_legacy_mode();
6853       }
6854     }
6855   }
6856 
6857   _is_managed = false;
6858   if (UseAVX > 2 && !attributes->is_legacy_mode())
6859   {
6860     bool evex_r = (xreg_enc >= 16);
6861     bool evex_v = (nds_enc >= 16);
6862     attributes->set_is_evex_instruction();
6863     evex_prefix(vex_r, vex_b, vex_x, evex_r, evex_v, nds_enc, pre, opc);
6864   } else {
6865     vex_prefix(vex_r, vex_b, vex_x, nds_enc, pre, opc);
6866   }
6867 }
6868 
6869 int Assembler::vex_prefix_and_encode(int dst_enc, int nds_enc, int src_enc, VexSimdPrefix pre, VexOpcode opc, InstructionAttr *attributes) {
6870   bool vex_r = ((dst_enc & 8) == 8) ? 1 : 0;
6871   bool vex_b = ((src_enc & 8) == 8) ? 1 : 0;
6872   bool vex_x = false;
6873   set_attributes(attributes);
6874   attributes->set_current_assembler(this);
6875   bool check_register_bank = NOT_IA32(true) IA32_ONLY(false);
6876 
6877   // if vector length is turned off, revert to AVX for vectors smaller than 512-bit
6878   if (UseAVX > 2 && _legacy_mode_vl && attributes->uses_vl()) {
6879     switch (attributes->get_vector_len()) {
6880     case AVX_128bit:
6881     case AVX_256bit:
6882       if (check_register_bank) {
6883         if (dst_enc >= 16 || nds_enc >= 16 || src_enc >= 16) {
6884           // up propagate arithmetic instructions to meet RA requirements
6885           attributes->set_vector_len(AVX_512bit);
6886         } else {
6887           attributes->set_is_legacy_mode();
6888         }
6889       } else {
6890         attributes->set_is_legacy_mode();
6891       }
6892       break;
6893     }
6894   }
6895 
6896   // For pure EVEX check and see if this instruction
6897   // is allowed in legacy mode and has resources which will
6898   // fit in it.  Pure EVEX instructions will use set_is_evex_instruction in their definition,
6899   // else that field is set when we encode to EVEX
6900   if (UseAVX > 2 && !attributes->is_legacy_mode() &&
6901       !_is_managed && !attributes->is_evex_instruction()) {
6902     if (!_legacy_mode_vl && attributes->get_vector_len() != AVX_512bit) {
6903       if (check_register_bank) {
6904         // check dst_enc, nds_enc and src_enc for upper bank usage
6905         if (dst_enc < 16 && nds_enc < 16 && src_enc < 16) {
6906           attributes->set_is_legacy_mode();
6907         }
6908       } else {
6909         attributes->set_is_legacy_mode();
6910       }
6911     }
6912   }
6913 
6914   _is_managed = false;
6915   if (UseAVX > 2 && !attributes->is_legacy_mode())
6916   {
6917     bool evex_r = (dst_enc >= 16);
6918     bool evex_v = (nds_enc >= 16);
6919     // can use vex_x as bank extender on rm encoding
6920     vex_x = (src_enc >= 16);
6921     attributes->set_is_evex_instruction();
6922     evex_prefix(vex_r, vex_b, vex_x, evex_r, evex_v, nds_enc, pre, opc);
6923   } else {
6924     vex_prefix(vex_r, vex_b, vex_x, nds_enc, pre, opc);
6925   }
6926 
6927   // return modrm byte components for operands
6928   return (((dst_enc & 7) << 3) | (src_enc & 7));
6929 }
6930 
6931 
6932 void Assembler::simd_prefix(XMMRegister xreg, XMMRegister nds, Address adr, VexSimdPrefix pre,
6933                             VexOpcode opc, InstructionAttr *attributes) {
6934   if (UseAVX > 0) {
6935     int xreg_enc = xreg->encoding();
6936     int nds_enc = nds->is_valid() ? nds->encoding() : 0;
6937     vex_prefix(adr, nds_enc, xreg_enc, pre, opc, attributes);
6938   } else {
6939     assert((nds == xreg) || (nds == xnoreg), "wrong sse encoding");
6940     rex_prefix(adr, xreg, pre, opc, attributes->is_rex_vex_w());
6941   }
6942 }
6943 
6944 int Assembler::simd_prefix_and_encode(XMMRegister dst, XMMRegister nds, XMMRegister src, VexSimdPrefix pre,
6945                                       VexOpcode opc, InstructionAttr *attributes) {
6946   int dst_enc = dst->encoding();
6947   int src_enc = src->encoding();
6948   if (UseAVX > 0) {
6949     int nds_enc = nds->is_valid() ? nds->encoding() : 0;
6950     return vex_prefix_and_encode(dst_enc, nds_enc, src_enc, pre, opc, attributes);
6951   } else {
6952     assert((nds == dst) || (nds == src) || (nds == xnoreg), "wrong sse encoding");
6953     return rex_prefix_and_encode(dst_enc, src_enc, pre, opc, attributes->is_rex_vex_w());
6954   }
6955 }
6956 
6957 void Assembler::cmppd(XMMRegister dst, XMMRegister nds, XMMRegister src, int cop, int vector_len) {
6958   assert(VM_Version::supports_avx(), "");
6959   assert(!VM_Version::supports_evex(), "");
6960   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
6961   int encode = simd_prefix_and_encode(dst, nds, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6962   emit_int8((unsigned char)0xC2);
6963   emit_int8((unsigned char)(0xC0 | encode));
6964   emit_int8((unsigned char)(0xF & cop));
6965 }
6966 
6967 void Assembler::vpblendd(XMMRegister dst, XMMRegister nds, XMMRegister src1, XMMRegister src2, int vector_len) {
6968   assert(VM_Version::supports_avx(), "");
6969   assert(!VM_Version::supports_evex(), "");
6970   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
6971   int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src1->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6972   emit_int8((unsigned char)0x4B);
6973   emit_int8((unsigned char)(0xC0 | encode));
6974   int src2_enc = src2->encoding();
6975   emit_int8((unsigned char)(0xF0 & src2_enc<<4));
6976 }
6977 
6978 
6979 #ifndef _LP64
6980 
6981 void Assembler::incl(Register dst) {
6982   // Don't use it directly. Use MacroAssembler::incrementl() instead.
6983   emit_int8(0x40 | dst->encoding());
6984 }
6985 
6986 void Assembler::lea(Register dst, Address src) {
6987   leal(dst, src);
6988 }
6989 
6990 void Assembler::mov_literal32(Address dst, int32_t imm32, RelocationHolder const& rspec) {
6991   InstructionMark im(this);
6992   emit_int8((unsigned char)0xC7);
6993   emit_operand(rax, dst);
6994   emit_data((int)imm32, rspec, 0);
6995 }
6996 
6997 void Assembler::mov_literal32(Register dst, int32_t imm32, RelocationHolder const& rspec) {
6998   InstructionMark im(this);
6999   int encode = prefix_and_encode(dst->encoding());
7000   emit_int8((unsigned char)(0xB8 | encode));
7001   emit_data((int)imm32, rspec, 0);
7002 }
7003 
7004 void Assembler::popa() { // 32bit
7005   emit_int8(0x61);
7006 }
7007 
7008 void Assembler::push_literal32(int32_t imm32, RelocationHolder const& rspec) {
7009   InstructionMark im(this);
7010   emit_int8(0x68);
7011   emit_data(imm32, rspec, 0);
7012 }
7013 
7014 void Assembler::pusha() { // 32bit
7015   emit_int8(0x60);
7016 }
7017 
7018 void Assembler::set_byte_if_not_zero(Register dst) {
7019   emit_int8(0x0F);
7020   emit_int8((unsigned char)0x95);
7021   emit_int8((unsigned char)(0xE0 | dst->encoding()));
7022 }
7023 
7024 void Assembler::shldl(Register dst, Register src) {
7025   emit_int8(0x0F);
7026   emit_int8((unsigned char)0xA5);
7027   emit_int8((unsigned char)(0xC0 | src->encoding() << 3 | dst->encoding()));
7028 }
7029 
7030 // 0F A4 / r ib
7031 void Assembler::shldl(Register dst, Register src, int8_t imm8) {
7032   emit_int8(0x0F);
7033   emit_int8((unsigned char)0xA4);
7034   emit_int8((unsigned char)(0xC0 | src->encoding() << 3 | dst->encoding()));
7035   emit_int8(imm8);
7036 }
7037 
7038 void Assembler::shrdl(Register dst, Register src) {
7039   emit_int8(0x0F);
7040   emit_int8((unsigned char)0xAD);
7041   emit_int8((unsigned char)(0xC0 | src->encoding() << 3 | dst->encoding()));
7042 }
7043 
7044 #else // LP64
7045 
7046 void Assembler::set_byte_if_not_zero(Register dst) {
7047   int enc = prefix_and_encode(dst->encoding(), true);
7048   emit_int8(0x0F);
7049   emit_int8((unsigned char)0x95);
7050   emit_int8((unsigned char)(0xE0 | enc));
7051 }
7052 
7053 // 64bit only pieces of the assembler
7054 // This should only be used by 64bit instructions that can use rip-relative
7055 // it cannot be used by instructions that want an immediate value.
7056 
7057 bool Assembler::reachable(AddressLiteral adr) {
7058   int64_t disp;
7059   // None will force a 64bit literal to the code stream. Likely a placeholder
7060   // for something that will be patched later and we need to certain it will
7061   // always be reachable.
7062   if (adr.reloc() == relocInfo::none) {
7063     return false;
7064   }
7065   if (adr.reloc() == relocInfo::internal_word_type) {
7066     // This should be rip relative and easily reachable.
7067     return true;
7068   }
7069   if (adr.reloc() == relocInfo::virtual_call_type ||
7070       adr.reloc() == relocInfo::opt_virtual_call_type ||
7071       adr.reloc() == relocInfo::static_call_type ||
7072       adr.reloc() == relocInfo::static_stub_type ) {
7073     // This should be rip relative within the code cache and easily
7074     // reachable until we get huge code caches. (At which point
7075     // ic code is going to have issues).
7076     return true;
7077   }
7078   if (adr.reloc() != relocInfo::external_word_type &&
7079       adr.reloc() != relocInfo::poll_return_type &&  // these are really external_word but need special
7080       adr.reloc() != relocInfo::poll_type &&         // relocs to identify them
7081       adr.reloc() != relocInfo::runtime_call_type ) {
7082     return false;
7083   }
7084 
7085   // Stress the correction code
7086   if (ForceUnreachable) {
7087     // Must be runtimecall reloc, see if it is in the codecache
7088     // Flipping stuff in the codecache to be unreachable causes issues
7089     // with things like inline caches where the additional instructions
7090     // are not handled.
7091     if (CodeCache::find_blob(adr._target) == NULL) {
7092       return false;
7093     }
7094   }
7095   // For external_word_type/runtime_call_type if it is reachable from where we
7096   // are now (possibly a temp buffer) and where we might end up
7097   // anywhere in the codeCache then we are always reachable.
7098   // This would have to change if we ever save/restore shared code
7099   // to be more pessimistic.
7100   disp = (int64_t)adr._target - ((int64_t)CodeCache::low_bound() + sizeof(int));
7101   if (!is_simm32(disp)) return false;
7102   disp = (int64_t)adr._target - ((int64_t)CodeCache::high_bound() + sizeof(int));
7103   if (!is_simm32(disp)) return false;
7104 
7105   disp = (int64_t)adr._target - ((int64_t)pc() + sizeof(int));
7106 
7107   // Because rip relative is a disp + address_of_next_instruction and we
7108   // don't know the value of address_of_next_instruction we apply a fudge factor
7109   // to make sure we will be ok no matter the size of the instruction we get placed into.
7110   // We don't have to fudge the checks above here because they are already worst case.
7111 
7112   // 12 == override/rex byte, opcode byte, rm byte, sib byte, a 4-byte disp , 4-byte literal
7113   // + 4 because better safe than sorry.
7114   const int fudge = 12 + 4;
7115   if (disp < 0) {
7116     disp -= fudge;
7117   } else {
7118     disp += fudge;
7119   }
7120   return is_simm32(disp);
7121 }
7122 
7123 // Check if the polling page is not reachable from the code cache using rip-relative
7124 // addressing.
7125 bool Assembler::is_polling_page_far() {
7126   intptr_t addr = (intptr_t)os::get_polling_page();
7127   return ForceUnreachable ||
7128          !is_simm32(addr - (intptr_t)CodeCache::low_bound()) ||
7129          !is_simm32(addr - (intptr_t)CodeCache::high_bound());
7130 }
7131 
7132 void Assembler::emit_data64(jlong data,
7133                             relocInfo::relocType rtype,
7134                             int format) {
7135   if (rtype == relocInfo::none) {
7136     emit_int64(data);
7137   } else {
7138     emit_data64(data, Relocation::spec_simple(rtype), format);
7139   }
7140 }
7141 
7142 void Assembler::emit_data64(jlong data,
7143                             RelocationHolder const& rspec,
7144                             int format) {
7145   assert(imm_operand == 0, "default format must be immediate in this file");
7146   assert(imm_operand == format, "must be immediate");
7147   assert(inst_mark() != NULL, "must be inside InstructionMark");
7148   // Do not use AbstractAssembler::relocate, which is not intended for
7149   // embedded words.  Instead, relocate to the enclosing instruction.
7150   code_section()->relocate(inst_mark(), rspec, format);
7151 #ifdef ASSERT
7152   check_relocation(rspec, format);
7153 #endif
7154   emit_int64(data);
7155 }
7156 
7157 int Assembler::prefix_and_encode(int reg_enc, bool byteinst) {
7158   if (reg_enc >= 8) {
7159     prefix(REX_B);
7160     reg_enc -= 8;
7161   } else if (byteinst && reg_enc >= 4) {
7162     prefix(REX);
7163   }
7164   return reg_enc;
7165 }
7166 
7167 int Assembler::prefixq_and_encode(int reg_enc) {
7168   if (reg_enc < 8) {
7169     prefix(REX_W);
7170   } else {
7171     prefix(REX_WB);
7172     reg_enc -= 8;
7173   }
7174   return reg_enc;
7175 }
7176 
7177 int Assembler::prefix_and_encode(int dst_enc, bool dst_is_byte, int src_enc, bool src_is_byte) {
7178   if (dst_enc < 8) {
7179     if (src_enc >= 8) {
7180       prefix(REX_B);
7181       src_enc -= 8;
7182     } else if ((src_is_byte && src_enc >= 4) || (dst_is_byte && dst_enc >= 4)) {
7183       prefix(REX);
7184     }
7185   } else {
7186     if (src_enc < 8) {
7187       prefix(REX_R);
7188     } else {
7189       prefix(REX_RB);
7190       src_enc -= 8;
7191     }
7192     dst_enc -= 8;
7193   }
7194   return dst_enc << 3 | src_enc;
7195 }
7196 
7197 int Assembler::prefixq_and_encode(int dst_enc, int src_enc) {
7198   if (dst_enc < 8) {
7199     if (src_enc < 8) {
7200       prefix(REX_W);
7201     } else {
7202       prefix(REX_WB);
7203       src_enc -= 8;
7204     }
7205   } else {
7206     if (src_enc < 8) {
7207       prefix(REX_WR);
7208     } else {
7209       prefix(REX_WRB);
7210       src_enc -= 8;
7211     }
7212     dst_enc -= 8;
7213   }
7214   return dst_enc << 3 | src_enc;
7215 }
7216 
7217 void Assembler::prefix(Register reg) {
7218   if (reg->encoding() >= 8) {
7219     prefix(REX_B);
7220   }
7221 }
7222 
7223 void Assembler::prefix(Register dst, Register src, Prefix p) {
7224   if (src->encoding() >= 8) {
7225     p = (Prefix)(p | REX_B);
7226   }
7227   if (dst->encoding() >= 8) {
7228     p = (Prefix)( p | REX_R);
7229   }
7230   if (p != Prefix_EMPTY) {
7231     // do not generate an empty prefix
7232     prefix(p);
7233   }
7234 }
7235 
7236 void Assembler::prefix(Register dst, Address adr, Prefix p) {
7237   if (adr.base_needs_rex()) {
7238     if (adr.index_needs_rex()) {
7239       assert(false, "prefix(Register dst, Address adr, Prefix p) does not support handling of an X");
7240     } else {
7241       prefix(REX_B);
7242     }
7243   } else {
7244     if (adr.index_needs_rex()) {
7245       assert(false, "prefix(Register dst, Address adr, Prefix p) does not support handling of an X");
7246     }
7247   }
7248   if (dst->encoding() >= 8) {
7249     p = (Prefix)(p | REX_R);
7250   }
7251   if (p != Prefix_EMPTY) {
7252     // do not generate an empty prefix
7253     prefix(p);
7254   }
7255 }
7256 
7257 void Assembler::prefix(Address adr) {
7258   if (adr.base_needs_rex()) {
7259     if (adr.index_needs_rex()) {
7260       prefix(REX_XB);
7261     } else {
7262       prefix(REX_B);
7263     }
7264   } else {
7265     if (adr.index_needs_rex()) {
7266       prefix(REX_X);
7267     }
7268   }
7269 }
7270 
7271 void Assembler::prefixq(Address adr) {
7272   if (adr.base_needs_rex()) {
7273     if (adr.index_needs_rex()) {
7274       prefix(REX_WXB);
7275     } else {
7276       prefix(REX_WB);
7277     }
7278   } else {
7279     if (adr.index_needs_rex()) {
7280       prefix(REX_WX);
7281     } else {
7282       prefix(REX_W);
7283     }
7284   }
7285 }
7286 
7287 
7288 void Assembler::prefix(Address adr, Register reg, bool byteinst) {
7289   if (reg->encoding() < 8) {
7290     if (adr.base_needs_rex()) {
7291       if (adr.index_needs_rex()) {
7292         prefix(REX_XB);
7293       } else {
7294         prefix(REX_B);
7295       }
7296     } else {
7297       if (adr.index_needs_rex()) {
7298         prefix(REX_X);
7299       } else if (byteinst && reg->encoding() >= 4 ) {
7300         prefix(REX);
7301       }
7302     }
7303   } else {
7304     if (adr.base_needs_rex()) {
7305       if (adr.index_needs_rex()) {
7306         prefix(REX_RXB);
7307       } else {
7308         prefix(REX_RB);
7309       }
7310     } else {
7311       if (adr.index_needs_rex()) {
7312         prefix(REX_RX);
7313       } else {
7314         prefix(REX_R);
7315       }
7316     }
7317   }
7318 }
7319 
7320 void Assembler::prefixq(Address adr, Register src) {
7321   if (src->encoding() < 8) {
7322     if (adr.base_needs_rex()) {
7323       if (adr.index_needs_rex()) {
7324         prefix(REX_WXB);
7325       } else {
7326         prefix(REX_WB);
7327       }
7328     } else {
7329       if (adr.index_needs_rex()) {
7330         prefix(REX_WX);
7331       } else {
7332         prefix(REX_W);
7333       }
7334     }
7335   } else {
7336     if (adr.base_needs_rex()) {
7337       if (adr.index_needs_rex()) {
7338         prefix(REX_WRXB);
7339       } else {
7340         prefix(REX_WRB);
7341       }
7342     } else {
7343       if (adr.index_needs_rex()) {
7344         prefix(REX_WRX);
7345       } else {
7346         prefix(REX_WR);
7347       }
7348     }
7349   }
7350 }
7351 
7352 void Assembler::prefix(Address adr, XMMRegister reg) {
7353   if (reg->encoding() < 8) {
7354     if (adr.base_needs_rex()) {
7355       if (adr.index_needs_rex()) {
7356         prefix(REX_XB);
7357       } else {
7358         prefix(REX_B);
7359       }
7360     } else {
7361       if (adr.index_needs_rex()) {
7362         prefix(REX_X);
7363       }
7364     }
7365   } else {
7366     if (adr.base_needs_rex()) {
7367       if (adr.index_needs_rex()) {
7368         prefix(REX_RXB);
7369       } else {
7370         prefix(REX_RB);
7371       }
7372     } else {
7373       if (adr.index_needs_rex()) {
7374         prefix(REX_RX);
7375       } else {
7376         prefix(REX_R);
7377       }
7378     }
7379   }
7380 }
7381 
7382 void Assembler::prefixq(Address adr, XMMRegister src) {
7383   if (src->encoding() < 8) {
7384     if (adr.base_needs_rex()) {
7385       if (adr.index_needs_rex()) {
7386         prefix(REX_WXB);
7387       } else {
7388         prefix(REX_WB);
7389       }
7390     } else {
7391       if (adr.index_needs_rex()) {
7392         prefix(REX_WX);
7393       } else {
7394         prefix(REX_W);
7395       }
7396     }
7397   } else {
7398     if (adr.base_needs_rex()) {
7399       if (adr.index_needs_rex()) {
7400         prefix(REX_WRXB);
7401       } else {
7402         prefix(REX_WRB);
7403       }
7404     } else {
7405       if (adr.index_needs_rex()) {
7406         prefix(REX_WRX);
7407       } else {
7408         prefix(REX_WR);
7409       }
7410     }
7411   }
7412 }
7413 
7414 void Assembler::adcq(Register dst, int32_t imm32) {
7415   (void) prefixq_and_encode(dst->encoding());
7416   emit_arith(0x81, 0xD0, dst, imm32);
7417 }
7418 
7419 void Assembler::adcq(Register dst, Address src) {
7420   InstructionMark im(this);
7421   prefixq(src, dst);
7422   emit_int8(0x13);
7423   emit_operand(dst, src);
7424 }
7425 
7426 void Assembler::adcq(Register dst, Register src) {
7427   (void) prefixq_and_encode(dst->encoding(), src->encoding());
7428   emit_arith(0x13, 0xC0, dst, src);
7429 }
7430 
7431 void Assembler::addq(Address dst, int32_t imm32) {
7432   InstructionMark im(this);
7433   prefixq(dst);
7434   emit_arith_operand(0x81, rax, dst,imm32);
7435 }
7436 
7437 void Assembler::addq(Address dst, Register src) {
7438   InstructionMark im(this);
7439   prefixq(dst, src);
7440   emit_int8(0x01);
7441   emit_operand(src, dst);
7442 }
7443 
7444 void Assembler::addq(Register dst, int32_t imm32) {
7445   (void) prefixq_and_encode(dst->encoding());
7446   emit_arith(0x81, 0xC0, dst, imm32);
7447 }
7448 
7449 void Assembler::addq(Register dst, Address src) {
7450   InstructionMark im(this);
7451   prefixq(src, dst);
7452   emit_int8(0x03);
7453   emit_operand(dst, src);
7454 }
7455 
7456 void Assembler::addq(Register dst, Register src) {
7457   (void) prefixq_and_encode(dst->encoding(), src->encoding());
7458   emit_arith(0x03, 0xC0, dst, src);
7459 }
7460 
7461 void Assembler::adcxq(Register dst, Register src) {
7462   //assert(VM_Version::supports_adx(), "adx instructions not supported");
7463   emit_int8((unsigned char)0x66);
7464   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
7465   emit_int8(0x0F);
7466   emit_int8(0x38);
7467   emit_int8((unsigned char)0xF6);
7468   emit_int8((unsigned char)(0xC0 | encode));
7469 }
7470 
7471 void Assembler::adoxq(Register dst, Register src) {
7472   //assert(VM_Version::supports_adx(), "adx instructions not supported");
7473   emit_int8((unsigned char)0xF3);
7474   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
7475   emit_int8(0x0F);
7476   emit_int8(0x38);
7477   emit_int8((unsigned char)0xF6);
7478   emit_int8((unsigned char)(0xC0 | encode));
7479 }
7480 
7481 void Assembler::andq(Address dst, int32_t imm32) {
7482   InstructionMark im(this);
7483   prefixq(dst);
7484   emit_int8((unsigned char)0x81);
7485   emit_operand(rsp, dst, 4);
7486   emit_int32(imm32);
7487 }
7488 
7489 void Assembler::andq(Register dst, int32_t imm32) {
7490   (void) prefixq_and_encode(dst->encoding());
7491   emit_arith(0x81, 0xE0, dst, imm32);
7492 }
7493 
7494 void Assembler::andq(Register dst, Address src) {
7495   InstructionMark im(this);
7496   prefixq(src, dst);
7497   emit_int8(0x23);
7498   emit_operand(dst, src);
7499 }
7500 
7501 void Assembler::andq(Register dst, Register src) {
7502   (void) prefixq_and_encode(dst->encoding(), src->encoding());
7503   emit_arith(0x23, 0xC0, dst, src);
7504 }
7505 
7506 void Assembler::andnq(Register dst, Register src1, Register src2) {
7507   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
7508   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
7509   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
7510   emit_int8((unsigned char)0xF2);
7511   emit_int8((unsigned char)(0xC0 | encode));
7512 }
7513 
7514 void Assembler::andnq(Register dst, Register src1, Address src2) {
7515   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
7516   InstructionMark im(this);
7517   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
7518   vex_prefix(src2, src1->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
7519   emit_int8((unsigned char)0xF2);
7520   emit_operand(dst, src2);
7521 }
7522 
7523 void Assembler::bsfq(Register dst, Register src) {
7524   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
7525   emit_int8(0x0F);
7526   emit_int8((unsigned char)0xBC);
7527   emit_int8((unsigned char)(0xC0 | encode));
7528 }
7529 
7530 void Assembler::bsrq(Register dst, Register src) {
7531   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
7532   emit_int8(0x0F);
7533   emit_int8((unsigned char)0xBD);
7534   emit_int8((unsigned char)(0xC0 | encode));
7535 }
7536 
7537 void Assembler::bswapq(Register reg) {
7538   int encode = prefixq_and_encode(reg->encoding());
7539   emit_int8(0x0F);
7540   emit_int8((unsigned char)(0xC8 | encode));
7541 }
7542 
7543 void Assembler::blsiq(Register dst, Register src) {
7544   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
7545   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
7546   int encode = vex_prefix_and_encode(rbx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
7547   emit_int8((unsigned char)0xF3);
7548   emit_int8((unsigned char)(0xC0 | encode));
7549 }
7550 
7551 void Assembler::blsiq(Register dst, Address src) {
7552   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
7553   InstructionMark im(this);
7554   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
7555   vex_prefix(src, dst->encoding(), rbx->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
7556   emit_int8((unsigned char)0xF3);
7557   emit_operand(rbx, src);
7558 }
7559 
7560 void Assembler::blsmskq(Register dst, Register src) {
7561   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
7562   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
7563   int encode = vex_prefix_and_encode(rdx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
7564   emit_int8((unsigned char)0xF3);
7565   emit_int8((unsigned char)(0xC0 | encode));
7566 }
7567 
7568 void Assembler::blsmskq(Register dst, Address src) {
7569   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
7570   InstructionMark im(this);
7571   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
7572   vex_prefix(src, dst->encoding(), rdx->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
7573   emit_int8((unsigned char)0xF3);
7574   emit_operand(rdx, src);
7575 }
7576 
7577 void Assembler::blsrq(Register dst, Register src) {
7578   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
7579   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
7580   int encode = vex_prefix_and_encode(rcx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
7581   emit_int8((unsigned char)0xF3);
7582   emit_int8((unsigned char)(0xC0 | encode));
7583 }
7584 
7585 void Assembler::blsrq(Register dst, Address src) {
7586   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
7587   InstructionMark im(this);
7588   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
7589   vex_prefix(src, dst->encoding(), rcx->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
7590   emit_int8((unsigned char)0xF3);
7591   emit_operand(rcx, src);
7592 }
7593 
7594 void Assembler::cdqq() {
7595   prefix(REX_W);
7596   emit_int8((unsigned char)0x99);
7597 }
7598 
7599 void Assembler::clflush(Address adr) {
7600   prefix(adr);
7601   emit_int8(0x0F);
7602   emit_int8((unsigned char)0xAE);
7603   emit_operand(rdi, adr);
7604 }
7605 
7606 void Assembler::cmovq(Condition cc, Register dst, Register src) {
7607   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
7608   emit_int8(0x0F);
7609   emit_int8(0x40 | cc);
7610   emit_int8((unsigned char)(0xC0 | encode));
7611 }
7612 
7613 void Assembler::cmovq(Condition cc, Register dst, Address src) {
7614   InstructionMark im(this);
7615   prefixq(src, dst);
7616   emit_int8(0x0F);
7617   emit_int8(0x40 | cc);
7618   emit_operand(dst, src);
7619 }
7620 
7621 void Assembler::cmpq(Address dst, int32_t imm32) {
7622   InstructionMark im(this);
7623   prefixq(dst);
7624   emit_int8((unsigned char)0x81);
7625   emit_operand(rdi, dst, 4);
7626   emit_int32(imm32);
7627 }
7628 
7629 void Assembler::cmpq(Register dst, int32_t imm32) {
7630   (void) prefixq_and_encode(dst->encoding());
7631   emit_arith(0x81, 0xF8, dst, imm32);
7632 }
7633 
7634 void Assembler::cmpq(Address dst, Register src) {
7635   InstructionMark im(this);
7636   prefixq(dst, src);
7637   emit_int8(0x3B);
7638   emit_operand(src, dst);
7639 }
7640 
7641 void Assembler::cmpq(Register dst, Register src) {
7642   (void) prefixq_and_encode(dst->encoding(), src->encoding());
7643   emit_arith(0x3B, 0xC0, dst, src);
7644 }
7645 
7646 void Assembler::cmpq(Register dst, Address  src) {
7647   InstructionMark im(this);
7648   prefixq(src, dst);
7649   emit_int8(0x3B);
7650   emit_operand(dst, src);
7651 }
7652 
7653 void Assembler::cmpxchgq(Register reg, Address adr) {
7654   InstructionMark im(this);
7655   prefixq(adr, reg);
7656   emit_int8(0x0F);
7657   emit_int8((unsigned char)0xB1);
7658   emit_operand(reg, adr);
7659 }
7660 
7661 void Assembler::cvtsi2sdq(XMMRegister dst, Register src) {
7662   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
7663   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
7664   int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
7665   emit_int8(0x2A);
7666   emit_int8((unsigned char)(0xC0 | encode));
7667 }
7668 
7669 void Assembler::cvtsi2sdq(XMMRegister dst, Address src) {
7670   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
7671   InstructionMark im(this);
7672   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
7673   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
7674   simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
7675   emit_int8(0x2A);
7676   emit_operand(dst, src);
7677 }
7678 
7679 void Assembler::cvtsi2ssq(XMMRegister dst, Address src) {
7680   NOT_LP64(assert(VM_Version::supports_sse(), ""));
7681   InstructionMark im(this);
7682   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
7683   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
7684   simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
7685   emit_int8(0x2A);
7686   emit_operand(dst, src);
7687 }
7688 
7689 void Assembler::cvttsd2siq(Register dst, XMMRegister src) {
7690   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
7691   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
7692   int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
7693   emit_int8(0x2C);
7694   emit_int8((unsigned char)(0xC0 | encode));
7695 }
7696 
7697 void Assembler::cvttss2siq(Register dst, XMMRegister src) {
7698   NOT_LP64(assert(VM_Version::supports_sse(), ""));
7699   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
7700   int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
7701   emit_int8(0x2C);
7702   emit_int8((unsigned char)(0xC0 | encode));
7703 }
7704 
7705 void Assembler::decl(Register dst) {
7706   // Don't use it directly. Use MacroAssembler::decrementl() instead.
7707   // Use two-byte form (one-byte form is a REX prefix in 64-bit mode)
7708   int encode = prefix_and_encode(dst->encoding());
7709   emit_int8((unsigned char)0xFF);
7710   emit_int8((unsigned char)(0xC8 | encode));
7711 }
7712 
7713 void Assembler::decq(Register dst) {
7714   // Don't use it directly. Use MacroAssembler::decrementq() instead.
7715   // Use two-byte form (one-byte from is a REX prefix in 64-bit mode)
7716   int encode = prefixq_and_encode(dst->encoding());
7717   emit_int8((unsigned char)0xFF);
7718   emit_int8(0xC8 | encode);
7719 }
7720 
7721 void Assembler::decq(Address dst) {
7722   // Don't use it directly. Use MacroAssembler::decrementq() instead.
7723   InstructionMark im(this);
7724   prefixq(dst);
7725   emit_int8((unsigned char)0xFF);
7726   emit_operand(rcx, dst);
7727 }
7728 
7729 void Assembler::fxrstor(Address src) {
7730   prefixq(src);
7731   emit_int8(0x0F);
7732   emit_int8((unsigned char)0xAE);
7733   emit_operand(as_Register(1), src);
7734 }
7735 
7736 void Assembler::xrstor(Address src) {
7737   prefixq(src);
7738   emit_int8(0x0F);
7739   emit_int8((unsigned char)0xAE);
7740   emit_operand(as_Register(5), src);
7741 }
7742 
7743 void Assembler::fxsave(Address dst) {
7744   prefixq(dst);
7745   emit_int8(0x0F);
7746   emit_int8((unsigned char)0xAE);
7747   emit_operand(as_Register(0), dst);
7748 }
7749 
7750 void Assembler::xsave(Address dst) {
7751   prefixq(dst);
7752   emit_int8(0x0F);
7753   emit_int8((unsigned char)0xAE);
7754   emit_operand(as_Register(4), dst);
7755 }
7756 
7757 void Assembler::idivq(Register src) {
7758   int encode = prefixq_and_encode(src->encoding());
7759   emit_int8((unsigned char)0xF7);
7760   emit_int8((unsigned char)(0xF8 | encode));
7761 }
7762 
7763 void Assembler::imulq(Register dst, Register src) {
7764   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
7765   emit_int8(0x0F);
7766   emit_int8((unsigned char)0xAF);
7767   emit_int8((unsigned char)(0xC0 | encode));
7768 }
7769 
7770 void Assembler::imulq(Register dst, Register src, int value) {
7771   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
7772   if (is8bit(value)) {
7773     emit_int8(0x6B);
7774     emit_int8((unsigned char)(0xC0 | encode));
7775     emit_int8(value & 0xFF);
7776   } else {
7777     emit_int8(0x69);
7778     emit_int8((unsigned char)(0xC0 | encode));
7779     emit_int32(value);
7780   }
7781 }
7782 
7783 void Assembler::imulq(Register dst, Address src) {
7784   InstructionMark im(this);
7785   prefixq(src, dst);
7786   emit_int8(0x0F);
7787   emit_int8((unsigned char) 0xAF);
7788   emit_operand(dst, src);
7789 }
7790 
7791 void Assembler::incl(Register dst) {
7792   // Don't use it directly. Use MacroAssembler::incrementl() instead.
7793   // Use two-byte form (one-byte from is a REX prefix in 64-bit mode)
7794   int encode = prefix_and_encode(dst->encoding());
7795   emit_int8((unsigned char)0xFF);
7796   emit_int8((unsigned char)(0xC0 | encode));
7797 }
7798 
7799 void Assembler::incq(Register dst) {
7800   // Don't use it directly. Use MacroAssembler::incrementq() instead.
7801   // Use two-byte form (one-byte from is a REX prefix in 64-bit mode)
7802   int encode = prefixq_and_encode(dst->encoding());
7803   emit_int8((unsigned char)0xFF);
7804   emit_int8((unsigned char)(0xC0 | encode));
7805 }
7806 
7807 void Assembler::incq(Address dst) {
7808   // Don't use it directly. Use MacroAssembler::incrementq() instead.
7809   InstructionMark im(this);
7810   prefixq(dst);
7811   emit_int8((unsigned char)0xFF);
7812   emit_operand(rax, dst);
7813 }
7814 
7815 void Assembler::lea(Register dst, Address src) {
7816   leaq(dst, src);
7817 }
7818 
7819 void Assembler::leaq(Register dst, Address src) {
7820   InstructionMark im(this);
7821   prefixq(src, dst);
7822   emit_int8((unsigned char)0x8D);
7823   emit_operand(dst, src);
7824 }
7825 
7826 void Assembler::mov64(Register dst, int64_t imm64) {
7827   InstructionMark im(this);
7828   int encode = prefixq_and_encode(dst->encoding());
7829   emit_int8((unsigned char)(0xB8 | encode));
7830   emit_int64(imm64);
7831 }
7832 
7833 void Assembler::mov_literal64(Register dst, intptr_t imm64, RelocationHolder const& rspec) {
7834   InstructionMark im(this);
7835   int encode = prefixq_and_encode(dst->encoding());
7836   emit_int8(0xB8 | encode);
7837   emit_data64(imm64, rspec);
7838 }
7839 
7840 void Assembler::mov_narrow_oop(Register dst, int32_t imm32, RelocationHolder const& rspec) {
7841   InstructionMark im(this);
7842   int encode = prefix_and_encode(dst->encoding());
7843   emit_int8((unsigned char)(0xB8 | encode));
7844   emit_data((int)imm32, rspec, narrow_oop_operand);
7845 }
7846 
7847 void Assembler::mov_narrow_oop(Address dst, int32_t imm32,  RelocationHolder const& rspec) {
7848   InstructionMark im(this);
7849   prefix(dst);
7850   emit_int8((unsigned char)0xC7);
7851   emit_operand(rax, dst, 4);
7852   emit_data((int)imm32, rspec, narrow_oop_operand);
7853 }
7854 
7855 void Assembler::cmp_narrow_oop(Register src1, int32_t imm32, RelocationHolder const& rspec) {
7856   InstructionMark im(this);
7857   int encode = prefix_and_encode(src1->encoding());
7858   emit_int8((unsigned char)0x81);
7859   emit_int8((unsigned char)(0xF8 | encode));
7860   emit_data((int)imm32, rspec, narrow_oop_operand);
7861 }
7862 
7863 void Assembler::cmp_narrow_oop(Address src1, int32_t imm32, RelocationHolder const& rspec) {
7864   InstructionMark im(this);
7865   prefix(src1);
7866   emit_int8((unsigned char)0x81);
7867   emit_operand(rax, src1, 4);
7868   emit_data((int)imm32, rspec, narrow_oop_operand);
7869 }
7870 
7871 void Assembler::lzcntq(Register dst, Register src) {
7872   assert(VM_Version::supports_lzcnt(), "encoding is treated as BSR");
7873   emit_int8((unsigned char)0xF3);
7874   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
7875   emit_int8(0x0F);
7876   emit_int8((unsigned char)0xBD);
7877   emit_int8((unsigned char)(0xC0 | encode));
7878 }
7879 
7880 void Assembler::movdq(XMMRegister dst, Register src) {
7881   // table D-1 says MMX/SSE2
7882   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
7883   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
7884   int encode = simd_prefix_and_encode(dst, xnoreg, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7885   emit_int8(0x6E);
7886   emit_int8((unsigned char)(0xC0 | encode));
7887 }
7888 
7889 void Assembler::movdq(Register dst, XMMRegister src) {
7890   // table D-1 says MMX/SSE2
7891   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
7892   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
7893   // swap src/dst to get correct prefix
7894   int encode = simd_prefix_and_encode(src, xnoreg, as_XMMRegister(dst->encoding()), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7895   emit_int8(0x7E);
7896   emit_int8((unsigned char)(0xC0 | encode));
7897 }
7898 
7899 void Assembler::movq(Register dst, Register src) {
7900   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
7901   emit_int8((unsigned char)0x8B);
7902   emit_int8((unsigned char)(0xC0 | encode));
7903 }
7904 
7905 void Assembler::movq(Register dst, Address src) {
7906   InstructionMark im(this);
7907   prefixq(src, dst);
7908   emit_int8((unsigned char)0x8B);
7909   emit_operand(dst, src);
7910 }
7911 
7912 void Assembler::movq(Address dst, Register src) {
7913   InstructionMark im(this);
7914   prefixq(dst, src);
7915   emit_int8((unsigned char)0x89);
7916   emit_operand(src, dst);
7917 }
7918 
7919 void Assembler::movsbq(Register dst, Address src) {
7920   InstructionMark im(this);
7921   prefixq(src, dst);
7922   emit_int8(0x0F);
7923   emit_int8((unsigned char)0xBE);
7924   emit_operand(dst, src);
7925 }
7926 
7927 void Assembler::movsbq(Register dst, Register src) {
7928   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
7929   emit_int8(0x0F);
7930   emit_int8((unsigned char)0xBE);
7931   emit_int8((unsigned char)(0xC0 | encode));
7932 }
7933 
7934 void Assembler::movslq(Register dst, int32_t imm32) {
7935   // dbx shows movslq(rcx, 3) as movq     $0x0000000049000000,(%rbx)
7936   // and movslq(r8, 3); as movl     $0x0000000048000000,(%rbx)
7937   // as a result we shouldn't use until tested at runtime...
7938   ShouldNotReachHere();
7939   InstructionMark im(this);
7940   int encode = prefixq_and_encode(dst->encoding());
7941   emit_int8((unsigned char)(0xC7 | encode));
7942   emit_int32(imm32);
7943 }
7944 
7945 void Assembler::movslq(Address dst, int32_t imm32) {
7946   assert(is_simm32(imm32), "lost bits");
7947   InstructionMark im(this);
7948   prefixq(dst);
7949   emit_int8((unsigned char)0xC7);
7950   emit_operand(rax, dst, 4);
7951   emit_int32(imm32);
7952 }
7953 
7954 void Assembler::movslq(Register dst, Address src) {
7955   InstructionMark im(this);
7956   prefixq(src, dst);
7957   emit_int8(0x63);
7958   emit_operand(dst, src);
7959 }
7960 
7961 void Assembler::movslq(Register dst, Register src) {
7962   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
7963   emit_int8(0x63);
7964   emit_int8((unsigned char)(0xC0 | encode));
7965 }
7966 
7967 void Assembler::movswq(Register dst, Address src) {
7968   InstructionMark im(this);
7969   prefixq(src, dst);
7970   emit_int8(0x0F);
7971   emit_int8((unsigned char)0xBF);
7972   emit_operand(dst, src);
7973 }
7974 
7975 void Assembler::movswq(Register dst, Register src) {
7976   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
7977   emit_int8((unsigned char)0x0F);
7978   emit_int8((unsigned char)0xBF);
7979   emit_int8((unsigned char)(0xC0 | encode));
7980 }
7981 
7982 void Assembler::movzbq(Register dst, Address src) {
7983   InstructionMark im(this);
7984   prefixq(src, dst);
7985   emit_int8((unsigned char)0x0F);
7986   emit_int8((unsigned char)0xB6);
7987   emit_operand(dst, src);
7988 }
7989 
7990 void Assembler::movzbq(Register dst, Register src) {
7991   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
7992   emit_int8(0x0F);
7993   emit_int8((unsigned char)0xB6);
7994   emit_int8(0xC0 | encode);
7995 }
7996 
7997 void Assembler::movzwq(Register dst, Address src) {
7998   InstructionMark im(this);
7999   prefixq(src, dst);
8000   emit_int8((unsigned char)0x0F);
8001   emit_int8((unsigned char)0xB7);
8002   emit_operand(dst, src);
8003 }
8004 
8005 void Assembler::movzwq(Register dst, Register src) {
8006   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
8007   emit_int8((unsigned char)0x0F);
8008   emit_int8((unsigned char)0xB7);
8009   emit_int8((unsigned char)(0xC0 | encode));
8010 }
8011 
8012 void Assembler::mulq(Address src) {
8013   InstructionMark im(this);
8014   prefixq(src);
8015   emit_int8((unsigned char)0xF7);
8016   emit_operand(rsp, src);
8017 }
8018 
8019 void Assembler::mulq(Register src) {
8020   int encode = prefixq_and_encode(src->encoding());
8021   emit_int8((unsigned char)0xF7);
8022   emit_int8((unsigned char)(0xE0 | encode));
8023 }
8024 
8025 void Assembler::mulxq(Register dst1, Register dst2, Register src) {
8026   assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
8027   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
8028   int encode = vex_prefix_and_encode(dst1->encoding(), dst2->encoding(), src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_38, &attributes);
8029   emit_int8((unsigned char)0xF6);
8030   emit_int8((unsigned char)(0xC0 | encode));
8031 }
8032 
8033 void Assembler::negq(Register dst) {
8034   int encode = prefixq_and_encode(dst->encoding());
8035   emit_int8((unsigned char)0xF7);
8036   emit_int8((unsigned char)(0xD8 | encode));
8037 }
8038 
8039 void Assembler::notq(Register dst) {
8040   int encode = prefixq_and_encode(dst->encoding());
8041   emit_int8((unsigned char)0xF7);
8042   emit_int8((unsigned char)(0xD0 | encode));
8043 }
8044 
8045 void Assembler::orq(Address dst, int32_t imm32) {
8046   InstructionMark im(this);
8047   prefixq(dst);
8048   emit_int8((unsigned char)0x81);
8049   emit_operand(rcx, dst, 4);
8050   emit_int32(imm32);
8051 }
8052 
8053 void Assembler::orq(Register dst, int32_t imm32) {
8054   (void) prefixq_and_encode(dst->encoding());
8055   emit_arith(0x81, 0xC8, dst, imm32);
8056 }
8057 
8058 void Assembler::orq(Register dst, Address src) {
8059   InstructionMark im(this);
8060   prefixq(src, dst);
8061   emit_int8(0x0B);
8062   emit_operand(dst, src);
8063 }
8064 
8065 void Assembler::orq(Register dst, Register src) {
8066   (void) prefixq_and_encode(dst->encoding(), src->encoding());
8067   emit_arith(0x0B, 0xC0, dst, src);
8068 }
8069 
8070 void Assembler::popa() { // 64bit
8071   movq(r15, Address(rsp, 0));
8072   movq(r14, Address(rsp, wordSize));
8073   movq(r13, Address(rsp, 2 * wordSize));
8074   movq(r12, Address(rsp, 3 * wordSize));
8075   movq(r11, Address(rsp, 4 * wordSize));
8076   movq(r10, Address(rsp, 5 * wordSize));
8077   movq(r9,  Address(rsp, 6 * wordSize));
8078   movq(r8,  Address(rsp, 7 * wordSize));
8079   movq(rdi, Address(rsp, 8 * wordSize));
8080   movq(rsi, Address(rsp, 9 * wordSize));
8081   movq(rbp, Address(rsp, 10 * wordSize));
8082   // skip rsp
8083   movq(rbx, Address(rsp, 12 * wordSize));
8084   movq(rdx, Address(rsp, 13 * wordSize));
8085   movq(rcx, Address(rsp, 14 * wordSize));
8086   movq(rax, Address(rsp, 15 * wordSize));
8087 
8088   addq(rsp, 16 * wordSize);
8089 }
8090 
8091 void Assembler::popcntq(Register dst, Address src) {
8092   assert(VM_Version::supports_popcnt(), "must support");
8093   InstructionMark im(this);
8094   emit_int8((unsigned char)0xF3);
8095   prefixq(src, dst);
8096   emit_int8((unsigned char)0x0F);
8097   emit_int8((unsigned char)0xB8);
8098   emit_operand(dst, src);
8099 }
8100 
8101 void Assembler::popcntq(Register dst, Register src) {
8102   assert(VM_Version::supports_popcnt(), "must support");
8103   emit_int8((unsigned char)0xF3);
8104   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
8105   emit_int8((unsigned char)0x0F);
8106   emit_int8((unsigned char)0xB8);
8107   emit_int8((unsigned char)(0xC0 | encode));
8108 }
8109 
8110 void Assembler::popq(Address dst) {
8111   InstructionMark im(this);
8112   prefixq(dst);
8113   emit_int8((unsigned char)0x8F);
8114   emit_operand(rax, dst);
8115 }
8116 
8117 void Assembler::pusha() { // 64bit
8118   // we have to store original rsp.  ABI says that 128 bytes
8119   // below rsp are local scratch.
8120   movq(Address(rsp, -5 * wordSize), rsp);
8121 
8122   subq(rsp, 16 * wordSize);
8123 
8124   movq(Address(rsp, 15 * wordSize), rax);
8125   movq(Address(rsp, 14 * wordSize), rcx);
8126   movq(Address(rsp, 13 * wordSize), rdx);
8127   movq(Address(rsp, 12 * wordSize), rbx);
8128   // skip rsp
8129   movq(Address(rsp, 10 * wordSize), rbp);
8130   movq(Address(rsp, 9 * wordSize), rsi);
8131   movq(Address(rsp, 8 * wordSize), rdi);
8132   movq(Address(rsp, 7 * wordSize), r8);
8133   movq(Address(rsp, 6 * wordSize), r9);
8134   movq(Address(rsp, 5 * wordSize), r10);
8135   movq(Address(rsp, 4 * wordSize), r11);
8136   movq(Address(rsp, 3 * wordSize), r12);
8137   movq(Address(rsp, 2 * wordSize), r13);
8138   movq(Address(rsp, wordSize), r14);
8139   movq(Address(rsp, 0), r15);
8140 }
8141 
8142 void Assembler::pushq(Address src) {
8143   InstructionMark im(this);
8144   prefixq(src);
8145   emit_int8((unsigned char)0xFF);
8146   emit_operand(rsi, src);
8147 }
8148 
8149 void Assembler::rclq(Register dst, int imm8) {
8150   assert(isShiftCount(imm8 >> 1), "illegal shift count");
8151   int encode = prefixq_and_encode(dst->encoding());
8152   if (imm8 == 1) {
8153     emit_int8((unsigned char)0xD1);
8154     emit_int8((unsigned char)(0xD0 | encode));
8155   } else {
8156     emit_int8((unsigned char)0xC1);
8157     emit_int8((unsigned char)(0xD0 | encode));
8158     emit_int8(imm8);
8159   }
8160 }
8161 
8162 void Assembler::rcrq(Register dst, int imm8) {
8163   assert(isShiftCount(imm8 >> 1), "illegal shift count");
8164   int encode = prefixq_and_encode(dst->encoding());
8165   if (imm8 == 1) {
8166     emit_int8((unsigned char)0xD1);
8167     emit_int8((unsigned char)(0xD8 | encode));
8168   } else {
8169     emit_int8((unsigned char)0xC1);
8170     emit_int8((unsigned char)(0xD8 | encode));
8171     emit_int8(imm8);
8172   }
8173 }
8174 
8175 void Assembler::rorq(Register dst, int imm8) {
8176   assert(isShiftCount(imm8 >> 1), "illegal shift count");
8177   int encode = prefixq_and_encode(dst->encoding());
8178   if (imm8 == 1) {
8179     emit_int8((unsigned char)0xD1);
8180     emit_int8((unsigned char)(0xC8 | encode));
8181   } else {
8182     emit_int8((unsigned char)0xC1);
8183     emit_int8((unsigned char)(0xc8 | encode));
8184     emit_int8(imm8);
8185   }
8186 }
8187 
8188 void Assembler::rorxq(Register dst, Register src, int imm8) {
8189   assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
8190   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
8191   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_3A, &attributes);
8192   emit_int8((unsigned char)0xF0);
8193   emit_int8((unsigned char)(0xC0 | encode));
8194   emit_int8(imm8);
8195 }
8196 
8197 void Assembler::sarq(Register dst, int imm8) {
8198   assert(isShiftCount(imm8 >> 1), "illegal shift count");
8199   int encode = prefixq_and_encode(dst->encoding());
8200   if (imm8 == 1) {
8201     emit_int8((unsigned char)0xD1);
8202     emit_int8((unsigned char)(0xF8 | encode));
8203   } else {
8204     emit_int8((unsigned char)0xC1);
8205     emit_int8((unsigned char)(0xF8 | encode));
8206     emit_int8(imm8);
8207   }
8208 }
8209 
8210 void Assembler::sarq(Register dst) {
8211   int encode = prefixq_and_encode(dst->encoding());
8212   emit_int8((unsigned char)0xD3);
8213   emit_int8((unsigned char)(0xF8 | encode));
8214 }
8215 
8216 void Assembler::sbbq(Address dst, int32_t imm32) {
8217   InstructionMark im(this);
8218   prefixq(dst);
8219   emit_arith_operand(0x81, rbx, dst, imm32);
8220 }
8221 
8222 void Assembler::sbbq(Register dst, int32_t imm32) {
8223   (void) prefixq_and_encode(dst->encoding());
8224   emit_arith(0x81, 0xD8, dst, imm32);
8225 }
8226 
8227 void Assembler::sbbq(Register dst, Address src) {
8228   InstructionMark im(this);
8229   prefixq(src, dst);
8230   emit_int8(0x1B);
8231   emit_operand(dst, src);
8232 }
8233 
8234 void Assembler::sbbq(Register dst, Register src) {
8235   (void) prefixq_and_encode(dst->encoding(), src->encoding());
8236   emit_arith(0x1B, 0xC0, dst, src);
8237 }
8238 
8239 void Assembler::shlq(Register dst, int imm8) {
8240   assert(isShiftCount(imm8 >> 1), "illegal shift count");
8241   int encode = prefixq_and_encode(dst->encoding());
8242   if (imm8 == 1) {
8243     emit_int8((unsigned char)0xD1);
8244     emit_int8((unsigned char)(0xE0 | encode));
8245   } else {
8246     emit_int8((unsigned char)0xC1);
8247     emit_int8((unsigned char)(0xE0 | encode));
8248     emit_int8(imm8);
8249   }
8250 }
8251 
8252 void Assembler::shlq(Register dst) {
8253   int encode = prefixq_and_encode(dst->encoding());
8254   emit_int8((unsigned char)0xD3);
8255   emit_int8((unsigned char)(0xE0 | encode));
8256 }
8257 
8258 void Assembler::shrq(Register dst, int imm8) {
8259   assert(isShiftCount(imm8 >> 1), "illegal shift count");
8260   int encode = prefixq_and_encode(dst->encoding());
8261   emit_int8((unsigned char)0xC1);
8262   emit_int8((unsigned char)(0xE8 | encode));
8263   emit_int8(imm8);
8264 }
8265 
8266 void Assembler::shrq(Register dst) {
8267   int encode = prefixq_and_encode(dst->encoding());
8268   emit_int8((unsigned char)0xD3);
8269   emit_int8(0xE8 | encode);
8270 }
8271 
8272 void Assembler::subq(Address dst, int32_t imm32) {
8273   InstructionMark im(this);
8274   prefixq(dst);
8275   emit_arith_operand(0x81, rbp, dst, imm32);
8276 }
8277 
8278 void Assembler::subq(Address dst, Register src) {
8279   InstructionMark im(this);
8280   prefixq(dst, src);
8281   emit_int8(0x29);
8282   emit_operand(src, dst);
8283 }
8284 
8285 void Assembler::subq(Register dst, int32_t imm32) {
8286   (void) prefixq_and_encode(dst->encoding());
8287   emit_arith(0x81, 0xE8, dst, imm32);
8288 }
8289 
8290 // Force generation of a 4 byte immediate value even if it fits into 8bit
8291 void Assembler::subq_imm32(Register dst, int32_t imm32) {
8292   (void) prefixq_and_encode(dst->encoding());
8293   emit_arith_imm32(0x81, 0xE8, dst, imm32);
8294 }
8295 
8296 void Assembler::subq(Register dst, Address src) {
8297   InstructionMark im(this);
8298   prefixq(src, dst);
8299   emit_int8(0x2B);
8300   emit_operand(dst, src);
8301 }
8302 
8303 void Assembler::subq(Register dst, Register src) {
8304   (void) prefixq_and_encode(dst->encoding(), src->encoding());
8305   emit_arith(0x2B, 0xC0, dst, src);
8306 }
8307 
8308 void Assembler::testq(Register dst, int32_t imm32) {
8309   // not using emit_arith because test
8310   // doesn't support sign-extension of
8311   // 8bit operands
8312   int encode = dst->encoding();
8313   if (encode == 0) {
8314     prefix(REX_W);
8315     emit_int8((unsigned char)0xA9);
8316   } else {
8317     encode = prefixq_and_encode(encode);
8318     emit_int8((unsigned char)0xF7);
8319     emit_int8((unsigned char)(0xC0 | encode));
8320   }
8321   emit_int32(imm32);
8322 }
8323 
8324 void Assembler::testq(Register dst, Register src) {
8325   (void) prefixq_and_encode(dst->encoding(), src->encoding());
8326   emit_arith(0x85, 0xC0, dst, src);
8327 }
8328 
8329 void Assembler::xaddq(Address dst, Register src) {
8330   InstructionMark im(this);
8331   prefixq(dst, src);
8332   emit_int8(0x0F);
8333   emit_int8((unsigned char)0xC1);
8334   emit_operand(src, dst);
8335 }
8336 
8337 void Assembler::xchgq(Register dst, Address src) {
8338   InstructionMark im(this);
8339   prefixq(src, dst);
8340   emit_int8((unsigned char)0x87);
8341   emit_operand(dst, src);
8342 }
8343 
8344 void Assembler::xchgq(Register dst, Register src) {
8345   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
8346   emit_int8((unsigned char)0x87);
8347   emit_int8((unsigned char)(0xc0 | encode));
8348 }
8349 
8350 void Assembler::xorq(Register dst, Register src) {
8351   (void) prefixq_and_encode(dst->encoding(), src->encoding());
8352   emit_arith(0x33, 0xC0, dst, src);
8353 }
8354 
8355 void Assembler::xorq(Register dst, Address src) {
8356   InstructionMark im(this);
8357   prefixq(src, dst);
8358   emit_int8(0x33);
8359   emit_operand(dst, src);
8360 }
8361 
8362 #endif // !LP64