1 /*
   2  * Copyright (c) 1997, 2015, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #include "precompiled.hpp"
  26 #include "asm/assembler.hpp"
  27 #include "asm/assembler.inline.hpp"
  28 #include "gc/shared/cardTableModRefBS.hpp"
  29 #include "gc/shared/collectedHeap.inline.hpp"
  30 #include "interpreter/interpreter.hpp"
  31 #include "memory/resourceArea.hpp"
  32 #include "prims/methodHandles.hpp"
  33 #include "runtime/biasedLocking.hpp"
  34 #include "runtime/interfaceSupport.hpp"
  35 #include "runtime/objectMonitor.hpp"
  36 #include "runtime/os.hpp"
  37 #include "runtime/sharedRuntime.hpp"
  38 #include "runtime/stubRoutines.hpp"
  39 #include "utilities/macros.hpp"
  40 #if INCLUDE_ALL_GCS
  41 #include "gc/g1/g1CollectedHeap.inline.hpp"
  42 #include "gc/g1/g1SATBCardTableModRefBS.hpp"
  43 #include "gc/g1/heapRegion.hpp"
  44 #endif // INCLUDE_ALL_GCS
  45 
  46 #ifdef PRODUCT
  47 #define BLOCK_COMMENT(str) /* nothing */
  48 #define STOP(error) stop(error)
  49 #else
  50 #define BLOCK_COMMENT(str) block_comment(str)
  51 #define STOP(error) block_comment(error); stop(error)
  52 #endif
  53 
  54 #define BIND(label) bind(label); BLOCK_COMMENT(#label ":")
  55 // Implementation of AddressLiteral
  56 
  57 // A 2-D table for managing compressed displacement(disp8) on EVEX enabled platforms.
  58 unsigned char tuple_table[Assembler::EVEX_ETUP + 1][Assembler::AVX_512bit + 1] = {
  59   // -----------------Table 4.5 -------------------- //
  60   16, 32, 64,  // EVEX_FV(0)
  61   4,  4,  4,   // EVEX_FV(1) - with Evex.b
  62   16, 32, 64,  // EVEX_FV(2) - with Evex.w
  63   8,  8,  8,   // EVEX_FV(3) - with Evex.w and Evex.b
  64   8,  16, 32,  // EVEX_HV(0)
  65   4,  4,  4,   // EVEX_HV(1) - with Evex.b
  66   // -----------------Table 4.6 -------------------- //
  67   16, 32, 64,  // EVEX_FVM(0)
  68   1,  1,  1,   // EVEX_T1S(0)
  69   2,  2,  2,   // EVEX_T1S(1)
  70   4,  4,  4,   // EVEX_T1S(2)
  71   8,  8,  8,   // EVEX_T1S(3)
  72   4,  4,  4,   // EVEX_T1F(0)
  73   8,  8,  8,   // EVEX_T1F(1)
  74   8,  8,  8,   // EVEX_T2(0)
  75   0,  16, 16,  // EVEX_T2(1)
  76   0,  16, 16,  // EVEX_T4(0)
  77   0,  0,  32,  // EVEX_T4(1)
  78   0,  0,  32,  // EVEX_T8(0)
  79   8,  16, 32,  // EVEX_HVM(0)
  80   4,  8,  16,  // EVEX_QVM(0)
  81   2,  4,  8,   // EVEX_OVM(0)
  82   16, 16, 16,  // EVEX_M128(0)
  83   8,  32, 64,  // EVEX_DUP(0)
  84   0,  0,  0    // EVEX_NTUP
  85 };
  86 
  87 AddressLiteral::AddressLiteral(address target, relocInfo::relocType rtype) {
  88   _is_lval = false;
  89   _target = target;
  90   switch (rtype) {
  91   case relocInfo::oop_type:
  92   case relocInfo::metadata_type:
  93     // Oops are a special case. Normally they would be their own section
  94     // but in cases like icBuffer they are literals in the code stream that
  95     // we don't have a section for. We use none so that we get a literal address
  96     // which is always patchable.
  97     break;
  98   case relocInfo::external_word_type:
  99     _rspec = external_word_Relocation::spec(target);
 100     break;
 101   case relocInfo::internal_word_type:
 102     _rspec = internal_word_Relocation::spec(target);
 103     break;
 104   case relocInfo::opt_virtual_call_type:
 105     _rspec = opt_virtual_call_Relocation::spec();
 106     break;
 107   case relocInfo::static_call_type:
 108     _rspec = static_call_Relocation::spec();
 109     break;
 110   case relocInfo::runtime_call_type:
 111     _rspec = runtime_call_Relocation::spec();
 112     break;
 113   case relocInfo::poll_type:
 114   case relocInfo::poll_return_type:
 115     _rspec = Relocation::spec_simple(rtype);
 116     break;
 117   case relocInfo::none:
 118     break;
 119   default:
 120     ShouldNotReachHere();
 121     break;
 122   }
 123 }
 124 
 125 // Implementation of Address
 126 
 127 #ifdef _LP64
 128 
 129 Address Address::make_array(ArrayAddress adr) {
 130   // Not implementable on 64bit machines
 131   // Should have been handled higher up the call chain.
 132   ShouldNotReachHere();
 133   return Address();
 134 }
 135 
 136 // exceedingly dangerous constructor
 137 Address::Address(int disp, address loc, relocInfo::relocType rtype) {
 138   _base  = noreg;
 139   _index = noreg;
 140   _scale = no_scale;
 141   _disp  = disp;
 142   switch (rtype) {
 143     case relocInfo::external_word_type:
 144       _rspec = external_word_Relocation::spec(loc);
 145       break;
 146     case relocInfo::internal_word_type:
 147       _rspec = internal_word_Relocation::spec(loc);
 148       break;
 149     case relocInfo::runtime_call_type:
 150       // HMM
 151       _rspec = runtime_call_Relocation::spec();
 152       break;
 153     case relocInfo::poll_type:
 154     case relocInfo::poll_return_type:
 155       _rspec = Relocation::spec_simple(rtype);
 156       break;
 157     case relocInfo::none:
 158       break;
 159     default:
 160       ShouldNotReachHere();
 161   }
 162 }
 163 #else // LP64
 164 
 165 Address Address::make_array(ArrayAddress adr) {
 166   AddressLiteral base = adr.base();
 167   Address index = adr.index();
 168   assert(index._disp == 0, "must not have disp"); // maybe it can?
 169   Address array(index._base, index._index, index._scale, (intptr_t) base.target());
 170   array._rspec = base._rspec;
 171   return array;
 172 }
 173 
 174 // exceedingly dangerous constructor
 175 Address::Address(address loc, RelocationHolder spec) {
 176   _base  = noreg;
 177   _index = noreg;
 178   _scale = no_scale;
 179   _disp  = (intptr_t) loc;
 180   _rspec = spec;
 181 }
 182 
 183 #endif // _LP64
 184 
 185 
 186 
 187 // Convert the raw encoding form into the form expected by the constructor for
 188 // Address.  An index of 4 (rsp) corresponds to having no index, so convert
 189 // that to noreg for the Address constructor.
 190 Address Address::make_raw(int base, int index, int scale, int disp, relocInfo::relocType disp_reloc) {
 191   RelocationHolder rspec;
 192   if (disp_reloc != relocInfo::none) {
 193     rspec = Relocation::spec_simple(disp_reloc);
 194   }
 195   bool valid_index = index != rsp->encoding();
 196   if (valid_index) {
 197     Address madr(as_Register(base), as_Register(index), (Address::ScaleFactor)scale, in_ByteSize(disp));
 198     madr._rspec = rspec;
 199     return madr;
 200   } else {
 201     Address madr(as_Register(base), noreg, Address::no_scale, in_ByteSize(disp));
 202     madr._rspec = rspec;
 203     return madr;
 204   }
 205 }
 206 
 207 // Implementation of Assembler
 208 
 209 int AbstractAssembler::code_fill_byte() {
 210   return (u_char)'\xF4'; // hlt
 211 }
 212 
 213 // make this go away someday
 214 void Assembler::emit_data(jint data, relocInfo::relocType rtype, int format) {
 215   if (rtype == relocInfo::none)
 216     emit_int32(data);
 217   else
 218     emit_data(data, Relocation::spec_simple(rtype), format);
 219 }
 220 
 221 void Assembler::emit_data(jint data, RelocationHolder const& rspec, int format) {
 222   assert(imm_operand == 0, "default format must be immediate in this file");
 223   assert(inst_mark() != NULL, "must be inside InstructionMark");
 224   if (rspec.type() !=  relocInfo::none) {
 225     #ifdef ASSERT
 226       check_relocation(rspec, format);
 227     #endif
 228     // Do not use AbstractAssembler::relocate, which is not intended for
 229     // embedded words.  Instead, relocate to the enclosing instruction.
 230 
 231     // hack. call32 is too wide for mask so use disp32
 232     if (format == call32_operand)
 233       code_section()->relocate(inst_mark(), rspec, disp32_operand);
 234     else
 235       code_section()->relocate(inst_mark(), rspec, format);
 236   }
 237   emit_int32(data);
 238 }
 239 
 240 static int encode(Register r) {
 241   int enc = r->encoding();
 242   if (enc >= 8) {
 243     enc -= 8;
 244   }
 245   return enc;
 246 }
 247 
 248 void Assembler::emit_arith_b(int op1, int op2, Register dst, int imm8) {
 249   assert(dst->has_byte_register(), "must have byte register");
 250   assert(isByte(op1) && isByte(op2), "wrong opcode");
 251   assert(isByte(imm8), "not a byte");
 252   assert((op1 & 0x01) == 0, "should be 8bit operation");
 253   emit_int8(op1);
 254   emit_int8(op2 | encode(dst));
 255   emit_int8(imm8);
 256 }
 257 
 258 
 259 void Assembler::emit_arith(int op1, int op2, Register dst, int32_t imm32) {
 260   assert(isByte(op1) && isByte(op2), "wrong opcode");
 261   assert((op1 & 0x01) == 1, "should be 32bit operation");
 262   assert((op1 & 0x02) == 0, "sign-extension bit should not be set");
 263   if (is8bit(imm32)) {
 264     emit_int8(op1 | 0x02); // set sign bit
 265     emit_int8(op2 | encode(dst));
 266     emit_int8(imm32 & 0xFF);
 267   } else {
 268     emit_int8(op1);
 269     emit_int8(op2 | encode(dst));
 270     emit_int32(imm32);
 271   }
 272 }
 273 
 274 // Force generation of a 4 byte immediate value even if it fits into 8bit
 275 void Assembler::emit_arith_imm32(int op1, int op2, Register dst, int32_t imm32) {
 276   assert(isByte(op1) && isByte(op2), "wrong opcode");
 277   assert((op1 & 0x01) == 1, "should be 32bit operation");
 278   assert((op1 & 0x02) == 0, "sign-extension bit should not be set");
 279   emit_int8(op1);
 280   emit_int8(op2 | encode(dst));
 281   emit_int32(imm32);
 282 }
 283 
 284 // immediate-to-memory forms
 285 void Assembler::emit_arith_operand(int op1, Register rm, Address adr, int32_t imm32) {
 286   assert((op1 & 0x01) == 1, "should be 32bit operation");
 287   assert((op1 & 0x02) == 0, "sign-extension bit should not be set");
 288   if (is8bit(imm32)) {
 289     emit_int8(op1 | 0x02); // set sign bit
 290     emit_operand(rm, adr, 1);
 291     emit_int8(imm32 & 0xFF);
 292   } else {
 293     emit_int8(op1);
 294     emit_operand(rm, adr, 4);
 295     emit_int32(imm32);
 296   }
 297 }
 298 
 299 
 300 void Assembler::emit_arith(int op1, int op2, Register dst, Register src) {
 301   assert(isByte(op1) && isByte(op2), "wrong opcode");
 302   emit_int8(op1);
 303   emit_int8(op2 | encode(dst) << 3 | encode(src));
 304 }
 305 
 306 
 307 bool Assembler::query_compressed_disp_byte(int disp, bool is_evex_inst, int vector_len,
 308                                            int cur_tuple_type, int in_size_in_bits, int cur_encoding) {
 309   int mod_idx = 0;
 310   // We will test if the displacement fits the compressed format and if so
 311   // apply the compression to the displacment iff the result is8bit.
 312   if (VM_Version::supports_evex() && is_evex_inst) {
 313     switch (cur_tuple_type) {
 314     case EVEX_FV:
 315       if ((cur_encoding & VEX_W) == VEX_W) {
 316         mod_idx = ((cur_encoding & EVEX_Rb) == EVEX_Rb) ? 3 : 2;
 317       } else {
 318         mod_idx = ((cur_encoding & EVEX_Rb) == EVEX_Rb) ? 1 : 0;
 319       }
 320       break;
 321 
 322     case EVEX_HV:
 323       mod_idx = ((cur_encoding & EVEX_Rb) == EVEX_Rb) ? 1 : 0;
 324       break;
 325 
 326     case EVEX_FVM:
 327       break;
 328 
 329     case EVEX_T1S:
 330       switch (in_size_in_bits) {
 331       case EVEX_8bit:
 332         break;
 333 
 334       case EVEX_16bit:
 335         mod_idx = 1;
 336         break;
 337 
 338       case EVEX_32bit:
 339         mod_idx = 2;
 340         break;
 341 
 342       case EVEX_64bit:
 343         mod_idx = 3;
 344         break;
 345       }
 346       break;
 347 
 348     case EVEX_T1F:
 349     case EVEX_T2:
 350     case EVEX_T4:
 351       mod_idx = (in_size_in_bits == EVEX_64bit) ? 1 : 0;
 352       break;
 353 
 354     case EVEX_T8:
 355       break;
 356 
 357     case EVEX_HVM:
 358       break;
 359 
 360     case EVEX_QVM:
 361       break;
 362 
 363     case EVEX_OVM:
 364       break;
 365 
 366     case EVEX_M128:
 367       break;
 368 
 369     case EVEX_DUP:
 370       break;
 371 
 372     default:
 373       assert(0, "no valid evex tuple_table entry");
 374       break;
 375     }
 376 
 377     if (vector_len >= AVX_128bit && vector_len <= AVX_512bit) {
 378       int disp_factor = tuple_table[cur_tuple_type + mod_idx][vector_len];
 379       if ((disp % disp_factor) == 0) {
 380         int new_disp = disp / disp_factor;
 381         if ((-0x80 <= new_disp && new_disp < 0x80)) {
 382           disp = new_disp;
 383         }
 384       } else {
 385         return false;
 386       }
 387     }
 388   }
 389   return (-0x80 <= disp && disp < 0x80);
 390 }
 391 
 392 
 393 bool Assembler::emit_compressed_disp_byte(int &disp) {
 394   int mod_idx = 0;
 395   // We will test if the displacement fits the compressed format and if so
 396   // apply the compression to the displacment iff the result is8bit.
 397   if (VM_Version::supports_evex() && (_attributes != NULL) && _attributes->is_evex_instruction()) {
 398     int evex_encoding = _attributes->get_evex_encoding();
 399     int tuple_type = _attributes->get_tuple_type();
 400     switch (tuple_type) {
 401     case EVEX_FV:
 402       if ((evex_encoding & VEX_W) == VEX_W) {
 403         mod_idx = ((evex_encoding & EVEX_Rb) == EVEX_Rb) ? 3 : 2;
 404       } else {
 405         mod_idx = ((evex_encoding & EVEX_Rb) == EVEX_Rb) ? 1 : 0;
 406       }
 407       break;
 408 
 409     case EVEX_HV:
 410       mod_idx = ((evex_encoding & EVEX_Rb) == EVEX_Rb) ? 1 : 0;
 411       break;
 412 
 413     case EVEX_FVM:
 414       break;
 415 
 416     case EVEX_T1S:
 417       switch (_attributes->get_input_size()) {
 418       case EVEX_8bit:
 419         break;
 420 
 421       case EVEX_16bit:
 422         mod_idx = 1;
 423         break;
 424 
 425       case EVEX_32bit:
 426         mod_idx = 2;
 427         break;
 428 
 429       case EVEX_64bit:
 430         mod_idx = 3;
 431         break;
 432       }
 433       break;
 434 
 435     case EVEX_T1F:
 436     case EVEX_T2:
 437     case EVEX_T4:
 438       mod_idx = (_attributes->get_input_size() == EVEX_64bit) ? 1 : 0;
 439       break;
 440 
 441     case EVEX_T8:
 442       break;
 443 
 444     case EVEX_HVM:
 445       break;
 446 
 447     case EVEX_QVM:
 448       break;
 449 
 450     case EVEX_OVM:
 451       break;
 452 
 453     case EVEX_M128:
 454       break;
 455 
 456     case EVEX_DUP:
 457       break;
 458 
 459     default:
 460       assert(0, "no valid evex tuple_table entry");
 461       break;
 462     }
 463 
 464     int vector_len = _attributes->get_vector_len();
 465     if (vector_len >= AVX_128bit && vector_len <= AVX_512bit) {
 466       int disp_factor = tuple_table[tuple_type + mod_idx][vector_len];
 467       if ((disp % disp_factor) == 0) {
 468         int new_disp = disp / disp_factor;
 469         if (is8bit(new_disp)) {
 470           disp = new_disp;
 471         }
 472       } else {
 473         return false;
 474       }
 475     }
 476   }
 477   return is8bit(disp);
 478 }
 479 
 480 
 481 void Assembler::emit_operand(Register reg, Register base, Register index,
 482                              Address::ScaleFactor scale, int disp,
 483                              RelocationHolder const& rspec,
 484                              int rip_relative_correction) {
 485   relocInfo::relocType rtype = (relocInfo::relocType) rspec.type();
 486 
 487   // Encode the registers as needed in the fields they are used in
 488 
 489   int regenc = encode(reg) << 3;
 490   int indexenc = index->is_valid() ? encode(index) << 3 : 0;
 491   int baseenc = base->is_valid() ? encode(base) : 0;
 492 
 493   if (base->is_valid()) {
 494     if (index->is_valid()) {
 495       assert(scale != Address::no_scale, "inconsistent address");
 496       // [base + index*scale + disp]
 497       if (disp == 0 && rtype == relocInfo::none  &&
 498           base != rbp LP64_ONLY(&& base != r13)) {
 499         // [base + index*scale]
 500         // [00 reg 100][ss index base]
 501         assert(index != rsp, "illegal addressing mode");
 502         emit_int8(0x04 | regenc);
 503         emit_int8(scale << 6 | indexenc | baseenc);
 504       } else if (emit_compressed_disp_byte(disp) && rtype == relocInfo::none) {
 505         // [base + index*scale + imm8]
 506         // [01 reg 100][ss index base] imm8
 507         assert(index != rsp, "illegal addressing mode");
 508         emit_int8(0x44 | regenc);
 509         emit_int8(scale << 6 | indexenc | baseenc);
 510         emit_int8(disp & 0xFF);
 511       } else {
 512         // [base + index*scale + disp32]
 513         // [10 reg 100][ss index base] disp32
 514         assert(index != rsp, "illegal addressing mode");
 515         emit_int8(0x84 | regenc);
 516         emit_int8(scale << 6 | indexenc | baseenc);
 517         emit_data(disp, rspec, disp32_operand);
 518       }
 519     } else if (base == rsp LP64_ONLY(|| base == r12)) {
 520       // [rsp + disp]
 521       if (disp == 0 && rtype == relocInfo::none) {
 522         // [rsp]
 523         // [00 reg 100][00 100 100]
 524         emit_int8(0x04 | regenc);
 525         emit_int8(0x24);
 526       } else if (emit_compressed_disp_byte(disp) && rtype == relocInfo::none) {
 527         // [rsp + imm8]
 528         // [01 reg 100][00 100 100] disp8
 529         emit_int8(0x44 | regenc);
 530         emit_int8(0x24);
 531         emit_int8(disp & 0xFF);
 532       } else {
 533         // [rsp + imm32]
 534         // [10 reg 100][00 100 100] disp32
 535         emit_int8(0x84 | regenc);
 536         emit_int8(0x24);
 537         emit_data(disp, rspec, disp32_operand);
 538       }
 539     } else {
 540       // [base + disp]
 541       assert(base != rsp LP64_ONLY(&& base != r12), "illegal addressing mode");
 542       if (disp == 0 && rtype == relocInfo::none &&
 543           base != rbp LP64_ONLY(&& base != r13)) {
 544         // [base]
 545         // [00 reg base]
 546         emit_int8(0x00 | regenc | baseenc);
 547       } else if (emit_compressed_disp_byte(disp) && rtype == relocInfo::none) {
 548         // [base + disp8]
 549         // [01 reg base] disp8
 550         emit_int8(0x40 | regenc | baseenc);
 551         emit_int8(disp & 0xFF);
 552       } else {
 553         // [base + disp32]
 554         // [10 reg base] disp32
 555         emit_int8(0x80 | regenc | baseenc);
 556         emit_data(disp, rspec, disp32_operand);
 557       }
 558     }
 559   } else {
 560     if (index->is_valid()) {
 561       assert(scale != Address::no_scale, "inconsistent address");
 562       // [index*scale + disp]
 563       // [00 reg 100][ss index 101] disp32
 564       assert(index != rsp, "illegal addressing mode");
 565       emit_int8(0x04 | regenc);
 566       emit_int8(scale << 6 | indexenc | 0x05);
 567       emit_data(disp, rspec, disp32_operand);
 568     } else if (rtype != relocInfo::none ) {
 569       // [disp] (64bit) RIP-RELATIVE (32bit) abs
 570       // [00 000 101] disp32
 571 
 572       emit_int8(0x05 | regenc);
 573       // Note that the RIP-rel. correction applies to the generated
 574       // disp field, but _not_ to the target address in the rspec.
 575 
 576       // disp was created by converting the target address minus the pc
 577       // at the start of the instruction. That needs more correction here.
 578       // intptr_t disp = target - next_ip;
 579       assert(inst_mark() != NULL, "must be inside InstructionMark");
 580       address next_ip = pc() + sizeof(int32_t) + rip_relative_correction;
 581       int64_t adjusted = disp;
 582       // Do rip-rel adjustment for 64bit
 583       LP64_ONLY(adjusted -=  (next_ip - inst_mark()));
 584       assert(is_simm32(adjusted),
 585              "must be 32bit offset (RIP relative address)");
 586       emit_data((int32_t) adjusted, rspec, disp32_operand);
 587 
 588     } else {
 589       // 32bit never did this, did everything as the rip-rel/disp code above
 590       // [disp] ABSOLUTE
 591       // [00 reg 100][00 100 101] disp32
 592       emit_int8(0x04 | regenc);
 593       emit_int8(0x25);
 594       emit_data(disp, rspec, disp32_operand);
 595     }
 596   }
 597 }
 598 
 599 void Assembler::emit_operand(XMMRegister reg, Register base, Register index,
 600                              Address::ScaleFactor scale, int disp,
 601                              RelocationHolder const& rspec) {
 602   if (UseAVX > 2) {
 603     int xreg_enc = reg->encoding();
 604     if (xreg_enc > 15) {
 605       XMMRegister new_reg = as_XMMRegister(xreg_enc & 0xf);
 606       emit_operand((Register)new_reg, base, index, scale, disp, rspec);
 607       return;
 608     }
 609   }
 610   emit_operand((Register)reg, base, index, scale, disp, rspec);
 611 }
 612 
 613 // Secret local extension to Assembler::WhichOperand:
 614 #define end_pc_operand (_WhichOperand_limit)
 615 
 616 address Assembler::locate_operand(address inst, WhichOperand which) {
 617   // Decode the given instruction, and return the address of
 618   // an embedded 32-bit operand word.
 619 
 620   // If "which" is disp32_operand, selects the displacement portion
 621   // of an effective address specifier.
 622   // If "which" is imm64_operand, selects the trailing immediate constant.
 623   // If "which" is call32_operand, selects the displacement of a call or jump.
 624   // Caller is responsible for ensuring that there is such an operand,
 625   // and that it is 32/64 bits wide.
 626 
 627   // If "which" is end_pc_operand, find the end of the instruction.
 628 
 629   address ip = inst;
 630   bool is_64bit = false;
 631 
 632   debug_only(bool has_disp32 = false);
 633   int tail_size = 0; // other random bytes (#32, #16, etc.) at end of insn
 634 
 635   again_after_prefix:
 636   switch (0xFF & *ip++) {
 637 
 638   // These convenience macros generate groups of "case" labels for the switch.
 639 #define REP4(x) (x)+0: case (x)+1: case (x)+2: case (x)+3
 640 #define REP8(x) (x)+0: case (x)+1: case (x)+2: case (x)+3: \
 641              case (x)+4: case (x)+5: case (x)+6: case (x)+7
 642 #define REP16(x) REP8((x)+0): \
 643               case REP8((x)+8)
 644 
 645   case CS_segment:
 646   case SS_segment:
 647   case DS_segment:
 648   case ES_segment:
 649   case FS_segment:
 650   case GS_segment:
 651     // Seems dubious
 652     LP64_ONLY(assert(false, "shouldn't have that prefix"));
 653     assert(ip == inst+1, "only one prefix allowed");
 654     goto again_after_prefix;
 655 
 656   case 0x67:
 657   case REX:
 658   case REX_B:
 659   case REX_X:
 660   case REX_XB:
 661   case REX_R:
 662   case REX_RB:
 663   case REX_RX:
 664   case REX_RXB:
 665     NOT_LP64(assert(false, "64bit prefixes"));
 666     goto again_after_prefix;
 667 
 668   case REX_W:
 669   case REX_WB:
 670   case REX_WX:
 671   case REX_WXB:
 672   case REX_WR:
 673   case REX_WRB:
 674   case REX_WRX:
 675   case REX_WRXB:
 676     NOT_LP64(assert(false, "64bit prefixes"));
 677     is_64bit = true;
 678     goto again_after_prefix;
 679 
 680   case 0xFF: // pushq a; decl a; incl a; call a; jmp a
 681   case 0x88: // movb a, r
 682   case 0x89: // movl a, r
 683   case 0x8A: // movb r, a
 684   case 0x8B: // movl r, a
 685   case 0x8F: // popl a
 686     debug_only(has_disp32 = true);
 687     break;
 688 
 689   case 0x68: // pushq #32
 690     if (which == end_pc_operand) {
 691       return ip + 4;
 692     }
 693     assert(which == imm_operand && !is_64bit, "pushl has no disp32 or 64bit immediate");
 694     return ip;                  // not produced by emit_operand
 695 
 696   case 0x66: // movw ... (size prefix)
 697     again_after_size_prefix2:
 698     switch (0xFF & *ip++) {
 699     case REX:
 700     case REX_B:
 701     case REX_X:
 702     case REX_XB:
 703     case REX_R:
 704     case REX_RB:
 705     case REX_RX:
 706     case REX_RXB:
 707     case REX_W:
 708     case REX_WB:
 709     case REX_WX:
 710     case REX_WXB:
 711     case REX_WR:
 712     case REX_WRB:
 713     case REX_WRX:
 714     case REX_WRXB:
 715       NOT_LP64(assert(false, "64bit prefix found"));
 716       goto again_after_size_prefix2;
 717     case 0x8B: // movw r, a
 718     case 0x89: // movw a, r
 719       debug_only(has_disp32 = true);
 720       break;
 721     case 0xC7: // movw a, #16
 722       debug_only(has_disp32 = true);
 723       tail_size = 2;  // the imm16
 724       break;
 725     case 0x0F: // several SSE/SSE2 variants
 726       ip--;    // reparse the 0x0F
 727       goto again_after_prefix;
 728     default:
 729       ShouldNotReachHere();
 730     }
 731     break;
 732 
 733   case REP8(0xB8): // movl/q r, #32/#64(oop?)
 734     if (which == end_pc_operand)  return ip + (is_64bit ? 8 : 4);
 735     // these asserts are somewhat nonsensical
 736 #ifndef _LP64
 737     assert(which == imm_operand || which == disp32_operand,
 738            "which %d is_64_bit %d ip " INTPTR_FORMAT, which, is_64bit, p2i(ip));
 739 #else
 740     assert((which == call32_operand || which == imm_operand) && is_64bit ||
 741            which == narrow_oop_operand && !is_64bit,
 742            "which %d is_64_bit %d ip " INTPTR_FORMAT, which, is_64bit, p2i(ip));
 743 #endif // _LP64
 744     return ip;
 745 
 746   case 0x69: // imul r, a, #32
 747   case 0xC7: // movl a, #32(oop?)
 748     tail_size = 4;
 749     debug_only(has_disp32 = true); // has both kinds of operands!
 750     break;
 751 
 752   case 0x0F: // movx..., etc.
 753     switch (0xFF & *ip++) {
 754     case 0x3A: // pcmpestri
 755       tail_size = 1;
 756     case 0x38: // ptest, pmovzxbw
 757       ip++; // skip opcode
 758       debug_only(has_disp32 = true); // has both kinds of operands!
 759       break;
 760 
 761     case 0x70: // pshufd r, r/a, #8
 762       debug_only(has_disp32 = true); // has both kinds of operands!
 763     case 0x73: // psrldq r, #8
 764       tail_size = 1;
 765       break;
 766 
 767     case 0x12: // movlps
 768     case 0x28: // movaps
 769     case 0x2E: // ucomiss
 770     case 0x2F: // comiss
 771     case 0x54: // andps
 772     case 0x55: // andnps
 773     case 0x56: // orps
 774     case 0x57: // xorps
 775     case 0x59: // mulpd
 776     case 0x6E: // movd
 777     case 0x7E: // movd
 778     case 0xAE: // ldmxcsr, stmxcsr, fxrstor, fxsave, clflush
 779       debug_only(has_disp32 = true);
 780       break;
 781 
 782     case 0xAD: // shrd r, a, %cl
 783     case 0xAF: // imul r, a
 784     case 0xBE: // movsbl r, a (movsxb)
 785     case 0xBF: // movswl r, a (movsxw)
 786     case 0xB6: // movzbl r, a (movzxb)
 787     case 0xB7: // movzwl r, a (movzxw)
 788     case REP16(0x40): // cmovl cc, r, a
 789     case 0xB0: // cmpxchgb
 790     case 0xB1: // cmpxchg
 791     case 0xC1: // xaddl
 792     case 0xC7: // cmpxchg8
 793     case REP16(0x90): // setcc a
 794       debug_only(has_disp32 = true);
 795       // fall out of the switch to decode the address
 796       break;
 797 
 798     case 0xC4: // pinsrw r, a, #8
 799       debug_only(has_disp32 = true);
 800     case 0xC5: // pextrw r, r, #8
 801       tail_size = 1;  // the imm8
 802       break;
 803 
 804     case 0xAC: // shrd r, a, #8
 805       debug_only(has_disp32 = true);
 806       tail_size = 1;  // the imm8
 807       break;
 808 
 809     case REP16(0x80): // jcc rdisp32
 810       if (which == end_pc_operand)  return ip + 4;
 811       assert(which == call32_operand, "jcc has no disp32 or imm");
 812       return ip;
 813     default:
 814       ShouldNotReachHere();
 815     }
 816     break;
 817 
 818   case 0x81: // addl a, #32; addl r, #32
 819     // also: orl, adcl, sbbl, andl, subl, xorl, cmpl
 820     // on 32bit in the case of cmpl, the imm might be an oop
 821     tail_size = 4;
 822     debug_only(has_disp32 = true); // has both kinds of operands!
 823     break;
 824 
 825   case 0x83: // addl a, #8; addl r, #8
 826     // also: orl, adcl, sbbl, andl, subl, xorl, cmpl
 827     debug_only(has_disp32 = true); // has both kinds of operands!
 828     tail_size = 1;
 829     break;
 830 
 831   case 0x9B:
 832     switch (0xFF & *ip++) {
 833     case 0xD9: // fnstcw a
 834       debug_only(has_disp32 = true);
 835       break;
 836     default:
 837       ShouldNotReachHere();
 838     }
 839     break;
 840 
 841   case REP4(0x00): // addb a, r; addl a, r; addb r, a; addl r, a
 842   case REP4(0x10): // adc...
 843   case REP4(0x20): // and...
 844   case REP4(0x30): // xor...
 845   case REP4(0x08): // or...
 846   case REP4(0x18): // sbb...
 847   case REP4(0x28): // sub...
 848   case 0xF7: // mull a
 849   case 0x8D: // lea r, a
 850   case 0x87: // xchg r, a
 851   case REP4(0x38): // cmp...
 852   case 0x85: // test r, a
 853     debug_only(has_disp32 = true); // has both kinds of operands!
 854     break;
 855 
 856   case 0xC1: // sal a, #8; sar a, #8; shl a, #8; shr a, #8
 857   case 0xC6: // movb a, #8
 858   case 0x80: // cmpb a, #8
 859   case 0x6B: // imul r, a, #8
 860     debug_only(has_disp32 = true); // has both kinds of operands!
 861     tail_size = 1; // the imm8
 862     break;
 863 
 864   case 0xC4: // VEX_3bytes
 865   case 0xC5: // VEX_2bytes
 866     assert((UseAVX > 0), "shouldn't have VEX prefix");
 867     assert(ip == inst+1, "no prefixes allowed");
 868     // C4 and C5 are also used as opcodes for PINSRW and PEXTRW instructions
 869     // but they have prefix 0x0F and processed when 0x0F processed above.
 870     //
 871     // In 32-bit mode the VEX first byte C4 and C5 alias onto LDS and LES
 872     // instructions (these instructions are not supported in 64-bit mode).
 873     // To distinguish them bits [7:6] are set in the VEX second byte since
 874     // ModRM byte can not be of the form 11xxxxxx in 32-bit mode. To set
 875     // those VEX bits REX and vvvv bits are inverted.
 876     //
 877     // Fortunately C2 doesn't generate these instructions so we don't need
 878     // to check for them in product version.
 879 
 880     // Check second byte
 881     NOT_LP64(assert((0xC0 & *ip) == 0xC0, "shouldn't have LDS and LES instructions"));
 882 
 883     int vex_opcode;
 884     // First byte
 885     if ((0xFF & *inst) == VEX_3bytes) {
 886       vex_opcode = VEX_OPCODE_MASK & *ip;
 887       ip++; // third byte
 888       is_64bit = ((VEX_W & *ip) == VEX_W);
 889     } else {
 890       vex_opcode = VEX_OPCODE_0F;
 891     }
 892     ip++; // opcode
 893     // To find the end of instruction (which == end_pc_operand).
 894     switch (vex_opcode) {
 895       case VEX_OPCODE_0F:
 896         switch (0xFF & *ip) {
 897         case 0x70: // pshufd r, r/a, #8
 898         case 0x71: // ps[rl|ra|ll]w r, #8
 899         case 0x72: // ps[rl|ra|ll]d r, #8
 900         case 0x73: // ps[rl|ra|ll]q r, #8
 901         case 0xC2: // cmp[ps|pd|ss|sd] r, r, r/a, #8
 902         case 0xC4: // pinsrw r, r, r/a, #8
 903         case 0xC5: // pextrw r/a, r, #8
 904         case 0xC6: // shufp[s|d] r, r, r/a, #8
 905           tail_size = 1;  // the imm8
 906           break;
 907         }
 908         break;
 909       case VEX_OPCODE_0F_3A:
 910         tail_size = 1;
 911         break;
 912     }
 913     ip++; // skip opcode
 914     debug_only(has_disp32 = true); // has both kinds of operands!
 915     break;
 916 
 917   case 0x62: // EVEX_4bytes
 918     assert((UseAVX > 0), "shouldn't have EVEX prefix");
 919     assert(ip == inst+1, "no prefixes allowed");
 920     // no EVEX collisions, all instructions that have 0x62 opcodes
 921     // have EVEX versions and are subopcodes of 0x66
 922     ip++; // skip P0 and exmaine W in P1
 923     is_64bit = ((VEX_W & *ip) == VEX_W);
 924     ip++; // move to P2
 925     ip++; // skip P2, move to opcode
 926     // To find the end of instruction (which == end_pc_operand).
 927     switch (0xFF & *ip) {
 928     case 0x61: // pcmpestri r, r/a, #8
 929     case 0x70: // pshufd r, r/a, #8
 930     case 0x73: // psrldq r, #8
 931       tail_size = 1;  // the imm8
 932       break;
 933     default:
 934       break;
 935     }
 936     ip++; // skip opcode
 937     debug_only(has_disp32 = true); // has both kinds of operands!
 938     break;
 939 
 940   case 0xD1: // sal a, 1; sar a, 1; shl a, 1; shr a, 1
 941   case 0xD3: // sal a, %cl; sar a, %cl; shl a, %cl; shr a, %cl
 942   case 0xD9: // fld_s a; fst_s a; fstp_s a; fldcw a
 943   case 0xDD: // fld_d a; fst_d a; fstp_d a
 944   case 0xDB: // fild_s a; fistp_s a; fld_x a; fstp_x a
 945   case 0xDF: // fild_d a; fistp_d a
 946   case 0xD8: // fadd_s a; fsubr_s a; fmul_s a; fdivr_s a; fcomp_s a
 947   case 0xDC: // fadd_d a; fsubr_d a; fmul_d a; fdivr_d a; fcomp_d a
 948   case 0xDE: // faddp_d a; fsubrp_d a; fmulp_d a; fdivrp_d a; fcompp_d a
 949     debug_only(has_disp32 = true);
 950     break;
 951 
 952   case 0xE8: // call rdisp32
 953   case 0xE9: // jmp  rdisp32
 954     if (which == end_pc_operand)  return ip + 4;
 955     assert(which == call32_operand, "call has no disp32 or imm");
 956     return ip;
 957 
 958   case 0xF0:                    // Lock
 959     assert(os::is_MP(), "only on MP");
 960     goto again_after_prefix;
 961 
 962   case 0xF3:                    // For SSE
 963   case 0xF2:                    // For SSE2
 964     switch (0xFF & *ip++) {
 965     case REX:
 966     case REX_B:
 967     case REX_X:
 968     case REX_XB:
 969     case REX_R:
 970     case REX_RB:
 971     case REX_RX:
 972     case REX_RXB:
 973     case REX_W:
 974     case REX_WB:
 975     case REX_WX:
 976     case REX_WXB:
 977     case REX_WR:
 978     case REX_WRB:
 979     case REX_WRX:
 980     case REX_WRXB:
 981       NOT_LP64(assert(false, "found 64bit prefix"));
 982       ip++;
 983     default:
 984       ip++;
 985     }
 986     debug_only(has_disp32 = true); // has both kinds of operands!
 987     break;
 988 
 989   default:
 990     ShouldNotReachHere();
 991 
 992 #undef REP8
 993 #undef REP16
 994   }
 995 
 996   assert(which != call32_operand, "instruction is not a call, jmp, or jcc");
 997 #ifdef _LP64
 998   assert(which != imm_operand, "instruction is not a movq reg, imm64");
 999 #else
1000   // assert(which != imm_operand || has_imm32, "instruction has no imm32 field");
1001   assert(which != imm_operand || has_disp32, "instruction has no imm32 field");
1002 #endif // LP64
1003   assert(which != disp32_operand || has_disp32, "instruction has no disp32 field");
1004 
1005   // parse the output of emit_operand
1006   int op2 = 0xFF & *ip++;
1007   int base = op2 & 0x07;
1008   int op3 = -1;
1009   const int b100 = 4;
1010   const int b101 = 5;
1011   if (base == b100 && (op2 >> 6) != 3) {
1012     op3 = 0xFF & *ip++;
1013     base = op3 & 0x07;   // refetch the base
1014   }
1015   // now ip points at the disp (if any)
1016 
1017   switch (op2 >> 6) {
1018   case 0:
1019     // [00 reg  100][ss index base]
1020     // [00 reg  100][00   100  esp]
1021     // [00 reg base]
1022     // [00 reg  100][ss index  101][disp32]
1023     // [00 reg  101]               [disp32]
1024 
1025     if (base == b101) {
1026       if (which == disp32_operand)
1027         return ip;              // caller wants the disp32
1028       ip += 4;                  // skip the disp32
1029     }
1030     break;
1031 
1032   case 1:
1033     // [01 reg  100][ss index base][disp8]
1034     // [01 reg  100][00   100  esp][disp8]
1035     // [01 reg base]               [disp8]
1036     ip += 1;                    // skip the disp8
1037     break;
1038 
1039   case 2:
1040     // [10 reg  100][ss index base][disp32]
1041     // [10 reg  100][00   100  esp][disp32]
1042     // [10 reg base]               [disp32]
1043     if (which == disp32_operand)
1044       return ip;                // caller wants the disp32
1045     ip += 4;                    // skip the disp32
1046     break;
1047 
1048   case 3:
1049     // [11 reg base]  (not a memory addressing mode)
1050     break;
1051   }
1052 
1053   if (which == end_pc_operand) {
1054     return ip + tail_size;
1055   }
1056 
1057 #ifdef _LP64
1058   assert(which == narrow_oop_operand && !is_64bit, "instruction is not a movl adr, imm32");
1059 #else
1060   assert(which == imm_operand, "instruction has only an imm field");
1061 #endif // LP64
1062   return ip;
1063 }
1064 
1065 address Assembler::locate_next_instruction(address inst) {
1066   // Secretly share code with locate_operand:
1067   return locate_operand(inst, end_pc_operand);
1068 }
1069 
1070 
1071 #ifdef ASSERT
1072 void Assembler::check_relocation(RelocationHolder const& rspec, int format) {
1073   address inst = inst_mark();
1074   assert(inst != NULL && inst < pc(), "must point to beginning of instruction");
1075   address opnd;
1076 
1077   Relocation* r = rspec.reloc();
1078   if (r->type() == relocInfo::none) {
1079     return;
1080   } else if (r->is_call() || format == call32_operand) {
1081     // assert(format == imm32_operand, "cannot specify a nonzero format");
1082     opnd = locate_operand(inst, call32_operand);
1083   } else if (r->is_data()) {
1084     assert(format == imm_operand || format == disp32_operand
1085            LP64_ONLY(|| format == narrow_oop_operand), "format ok");
1086     opnd = locate_operand(inst, (WhichOperand)format);
1087   } else {
1088     assert(format == imm_operand, "cannot specify a format");
1089     return;
1090   }
1091   assert(opnd == pc(), "must put operand where relocs can find it");
1092 }
1093 #endif // ASSERT
1094 
1095 void Assembler::emit_operand32(Register reg, Address adr) {
1096   assert(reg->encoding() < 8, "no extended registers");
1097   assert(!adr.base_needs_rex() && !adr.index_needs_rex(), "no extended registers");
1098   emit_operand(reg, adr._base, adr._index, adr._scale, adr._disp,
1099                adr._rspec);
1100 }
1101 
1102 void Assembler::emit_operand(Register reg, Address adr,
1103                              int rip_relative_correction) {
1104   emit_operand(reg, adr._base, adr._index, adr._scale, adr._disp,
1105                adr._rspec,
1106                rip_relative_correction);
1107 }
1108 
1109 void Assembler::emit_operand(XMMRegister reg, Address adr) {
1110   emit_operand(reg, adr._base, adr._index, adr._scale, adr._disp,
1111                adr._rspec);
1112 }
1113 
1114 // MMX operations
1115 void Assembler::emit_operand(MMXRegister reg, Address adr) {
1116   assert(!adr.base_needs_rex() && !adr.index_needs_rex(), "no extended registers");
1117   emit_operand((Register)reg, adr._base, adr._index, adr._scale, adr._disp, adr._rspec);
1118 }
1119 
1120 // work around gcc (3.2.1-7a) bug
1121 void Assembler::emit_operand(Address adr, MMXRegister reg) {
1122   assert(!adr.base_needs_rex() && !adr.index_needs_rex(), "no extended registers");
1123   emit_operand((Register)reg, adr._base, adr._index, adr._scale, adr._disp, adr._rspec);
1124 }
1125 
1126 
1127 void Assembler::emit_farith(int b1, int b2, int i) {
1128   assert(isByte(b1) && isByte(b2), "wrong opcode");
1129   assert(0 <= i &&  i < 8, "illegal stack offset");
1130   emit_int8(b1);
1131   emit_int8(b2 + i);
1132 }
1133 
1134 
1135 // Now the Assembler instructions (identical for 32/64 bits)
1136 
1137 void Assembler::adcl(Address dst, int32_t imm32) {
1138   InstructionMark im(this);
1139   prefix(dst);
1140   emit_arith_operand(0x81, rdx, dst, imm32);
1141 }
1142 
1143 void Assembler::adcl(Address dst, Register src) {
1144   InstructionMark im(this);
1145   prefix(dst, src);
1146   emit_int8(0x11);
1147   emit_operand(src, dst);
1148 }
1149 
1150 void Assembler::adcl(Register dst, int32_t imm32) {
1151   prefix(dst);
1152   emit_arith(0x81, 0xD0, dst, imm32);
1153 }
1154 
1155 void Assembler::adcl(Register dst, Address src) {
1156   InstructionMark im(this);
1157   prefix(src, dst);
1158   emit_int8(0x13);
1159   emit_operand(dst, src);
1160 }
1161 
1162 void Assembler::adcl(Register dst, Register src) {
1163   (void) prefix_and_encode(dst->encoding(), src->encoding());
1164   emit_arith(0x13, 0xC0, dst, src);
1165 }
1166 
1167 void Assembler::addl(Address dst, int32_t imm32) {
1168   InstructionMark im(this);
1169   prefix(dst);
1170   emit_arith_operand(0x81, rax, dst, imm32);
1171 }
1172 
1173 void Assembler::addl(Address dst, Register src) {
1174   InstructionMark im(this);
1175   prefix(dst, src);
1176   emit_int8(0x01);
1177   emit_operand(src, dst);
1178 }
1179 
1180 void Assembler::addl(Register dst, int32_t imm32) {
1181   prefix(dst);
1182   emit_arith(0x81, 0xC0, dst, imm32);
1183 }
1184 
1185 void Assembler::addl(Register dst, Address src) {
1186   InstructionMark im(this);
1187   prefix(src, dst);
1188   emit_int8(0x03);
1189   emit_operand(dst, src);
1190 }
1191 
1192 void Assembler::addl(Register dst, Register src) {
1193   (void) prefix_and_encode(dst->encoding(), src->encoding());
1194   emit_arith(0x03, 0xC0, dst, src);
1195 }
1196 
1197 void Assembler::addr_nop_4() {
1198   assert(UseAddressNop, "no CPU support");
1199   // 4 bytes: NOP DWORD PTR [EAX+0]
1200   emit_int8(0x0F);
1201   emit_int8(0x1F);
1202   emit_int8(0x40); // emit_rm(cbuf, 0x1, EAX_enc, EAX_enc);
1203   emit_int8(0);    // 8-bits offset (1 byte)
1204 }
1205 
1206 void Assembler::addr_nop_5() {
1207   assert(UseAddressNop, "no CPU support");
1208   // 5 bytes: NOP DWORD PTR [EAX+EAX*0+0] 8-bits offset
1209   emit_int8(0x0F);
1210   emit_int8(0x1F);
1211   emit_int8(0x44); // emit_rm(cbuf, 0x1, EAX_enc, 0x4);
1212   emit_int8(0x00); // emit_rm(cbuf, 0x0, EAX_enc, EAX_enc);
1213   emit_int8(0);    // 8-bits offset (1 byte)
1214 }
1215 
1216 void Assembler::addr_nop_7() {
1217   assert(UseAddressNop, "no CPU support");
1218   // 7 bytes: NOP DWORD PTR [EAX+0] 32-bits offset
1219   emit_int8(0x0F);
1220   emit_int8(0x1F);
1221   emit_int8((unsigned char)0x80);
1222                    // emit_rm(cbuf, 0x2, EAX_enc, EAX_enc);
1223   emit_int32(0);   // 32-bits offset (4 bytes)
1224 }
1225 
1226 void Assembler::addr_nop_8() {
1227   assert(UseAddressNop, "no CPU support");
1228   // 8 bytes: NOP DWORD PTR [EAX+EAX*0+0] 32-bits offset
1229   emit_int8(0x0F);
1230   emit_int8(0x1F);
1231   emit_int8((unsigned char)0x84);
1232                    // emit_rm(cbuf, 0x2, EAX_enc, 0x4);
1233   emit_int8(0x00); // emit_rm(cbuf, 0x0, EAX_enc, EAX_enc);
1234   emit_int32(0);   // 32-bits offset (4 bytes)
1235 }
1236 
1237 void Assembler::addsd(XMMRegister dst, XMMRegister src) {
1238   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1239   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
1240   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
1241   emit_int8(0x58);
1242   emit_int8((unsigned char)(0xC0 | encode));
1243 }
1244 
1245 void Assembler::addsd(XMMRegister dst, Address src) {
1246   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1247   InstructionMark im(this);
1248   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
1249   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
1250   simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
1251   emit_int8(0x58);
1252   emit_operand(dst, src);
1253 }
1254 
1255 void Assembler::addss(XMMRegister dst, XMMRegister src) {
1256   NOT_LP64(assert(VM_Version::supports_sse(), ""));
1257   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
1258   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
1259   emit_int8(0x58);
1260   emit_int8((unsigned char)(0xC0 | encode));
1261 }
1262 
1263 void Assembler::addss(XMMRegister dst, Address src) {
1264   NOT_LP64(assert(VM_Version::supports_sse(), ""));
1265   InstructionMark im(this);
1266   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
1267   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
1268   simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
1269   emit_int8(0x58);
1270   emit_operand(dst, src);
1271 }
1272 
1273 void Assembler::aesdec(XMMRegister dst, Address src) {
1274   assert(VM_Version::supports_aes(), "");
1275   InstructionMark im(this);
1276   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
1277   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
1278   emit_int8((unsigned char)0xDE);
1279   emit_operand(dst, src);
1280 }
1281 
1282 void Assembler::aesdec(XMMRegister dst, XMMRegister src) {
1283   assert(VM_Version::supports_aes(), "");
1284   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
1285   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
1286   emit_int8((unsigned char)0xDE);
1287   emit_int8(0xC0 | encode);
1288 }
1289 
1290 void Assembler::aesdeclast(XMMRegister dst, Address src) {
1291   assert(VM_Version::supports_aes(), "");
1292   InstructionMark im(this);
1293   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
1294   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
1295   emit_int8((unsigned char)0xDF);
1296   emit_operand(dst, src);
1297 }
1298 
1299 void Assembler::aesdeclast(XMMRegister dst, XMMRegister src) {
1300   assert(VM_Version::supports_aes(), "");
1301   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
1302   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
1303   emit_int8((unsigned char)0xDF);
1304   emit_int8((unsigned char)(0xC0 | encode));
1305 }
1306 
1307 void Assembler::aesenc(XMMRegister dst, Address src) {
1308   assert(VM_Version::supports_aes(), "");
1309   InstructionMark im(this);
1310   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
1311   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
1312   emit_int8((unsigned char)0xDC);
1313   emit_operand(dst, src);
1314 }
1315 
1316 void Assembler::aesenc(XMMRegister dst, XMMRegister src) {
1317   assert(VM_Version::supports_aes(), "");
1318   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
1319   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
1320   emit_int8((unsigned char)0xDC);
1321   emit_int8(0xC0 | encode);
1322 }
1323 
1324 void Assembler::aesenclast(XMMRegister dst, Address src) {
1325   assert(VM_Version::supports_aes(), "");
1326   InstructionMark im(this);
1327   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
1328   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
1329   emit_int8((unsigned char)0xDD);
1330   emit_operand(dst, src);
1331 }
1332 
1333 void Assembler::aesenclast(XMMRegister dst, XMMRegister src) {
1334   assert(VM_Version::supports_aes(), "");
1335   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
1336   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
1337   emit_int8((unsigned char)0xDD);
1338   emit_int8((unsigned char)(0xC0 | encode));
1339 }
1340 
1341 void Assembler::andl(Address dst, int32_t imm32) {
1342   InstructionMark im(this);
1343   prefix(dst);
1344   emit_int8((unsigned char)0x81);
1345   emit_operand(rsp, dst, 4);
1346   emit_int32(imm32);
1347 }
1348 
1349 void Assembler::andl(Register dst, int32_t imm32) {
1350   prefix(dst);
1351   emit_arith(0x81, 0xE0, dst, imm32);
1352 }
1353 
1354 void Assembler::andl(Register dst, Address src) {
1355   InstructionMark im(this);
1356   prefix(src, dst);
1357   emit_int8(0x23);
1358   emit_operand(dst, src);
1359 }
1360 
1361 void Assembler::andl(Register dst, Register src) {
1362   (void) prefix_and_encode(dst->encoding(), src->encoding());
1363   emit_arith(0x23, 0xC0, dst, src);
1364 }
1365 
1366 void Assembler::andnl(Register dst, Register src1, Register src2) {
1367   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1368   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
1369   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
1370   emit_int8((unsigned char)0xF2);
1371   emit_int8((unsigned char)(0xC0 | encode));
1372 }
1373 
1374 void Assembler::andnl(Register dst, Register src1, Address src2) {
1375   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1376   InstructionMark im(this);
1377   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
1378   vex_prefix(src2, src1->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
1379   emit_int8((unsigned char)0xF2);
1380   emit_operand(dst, src2);
1381 }
1382 
1383 void Assembler::bsfl(Register dst, Register src) {
1384   int encode = prefix_and_encode(dst->encoding(), src->encoding());
1385   emit_int8(0x0F);
1386   emit_int8((unsigned char)0xBC);
1387   emit_int8((unsigned char)(0xC0 | encode));
1388 }
1389 
1390 void Assembler::bsrl(Register dst, Register src) {
1391   int encode = prefix_and_encode(dst->encoding(), src->encoding());
1392   emit_int8(0x0F);
1393   emit_int8((unsigned char)0xBD);
1394   emit_int8((unsigned char)(0xC0 | encode));
1395 }
1396 
1397 void Assembler::bswapl(Register reg) { // bswap
1398   int encode = prefix_and_encode(reg->encoding());
1399   emit_int8(0x0F);
1400   emit_int8((unsigned char)(0xC8 | encode));
1401 }
1402 
1403 void Assembler::blsil(Register dst, Register src) {
1404   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1405   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
1406   int encode = vex_prefix_and_encode(rbx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
1407   emit_int8((unsigned char)0xF3);
1408   emit_int8((unsigned char)(0xC0 | encode));
1409 }
1410 
1411 void Assembler::blsil(Register dst, Address src) {
1412   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1413   InstructionMark im(this);
1414   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
1415   vex_prefix(src, dst->encoding(), rbx->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
1416   emit_int8((unsigned char)0xF3);
1417   emit_operand(rbx, src);
1418 }
1419 
1420 void Assembler::blsmskl(Register dst, Register src) {
1421   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1422   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
1423   int encode = vex_prefix_and_encode(rdx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
1424   emit_int8((unsigned char)0xF3);
1425   emit_int8((unsigned char)(0xC0 | encode));
1426 }
1427 
1428 void Assembler::blsmskl(Register dst, Address src) {
1429   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1430   InstructionMark im(this);
1431   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
1432   vex_prefix(src, dst->encoding(), rdx->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
1433   emit_int8((unsigned char)0xF3);
1434   emit_operand(rdx, src);
1435 }
1436 
1437 void Assembler::blsrl(Register dst, Register src) {
1438   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1439   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
1440   int encode = vex_prefix_and_encode(rcx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
1441   emit_int8((unsigned char)0xF3);
1442   emit_int8((unsigned char)(0xC0 | encode));
1443 }
1444 
1445 void Assembler::blsrl(Register dst, Address src) {
1446   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1447   InstructionMark im(this);
1448   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
1449   vex_prefix(src, dst->encoding(), rcx->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
1450   emit_int8((unsigned char)0xF3);
1451   emit_operand(rcx, src);
1452 }
1453 
1454 void Assembler::call(Label& L, relocInfo::relocType rtype) {
1455   // suspect disp32 is always good
1456   int operand = LP64_ONLY(disp32_operand) NOT_LP64(imm_operand);
1457 
1458   if (L.is_bound()) {
1459     const int long_size = 5;
1460     int offs = (int)( target(L) - pc() );
1461     assert(offs <= 0, "assembler error");
1462     InstructionMark im(this);
1463     // 1110 1000 #32-bit disp
1464     emit_int8((unsigned char)0xE8);
1465     emit_data(offs - long_size, rtype, operand);
1466   } else {
1467     InstructionMark im(this);
1468     // 1110 1000 #32-bit disp
1469     L.add_patch_at(code(), locator());
1470 
1471     emit_int8((unsigned char)0xE8);
1472     emit_data(int(0), rtype, operand);
1473   }
1474 }
1475 
1476 void Assembler::call(Register dst) {
1477   int encode = prefix_and_encode(dst->encoding());
1478   emit_int8((unsigned char)0xFF);
1479   emit_int8((unsigned char)(0xD0 | encode));
1480 }
1481 
1482 
1483 void Assembler::call(Address adr) {
1484   InstructionMark im(this);
1485   prefix(adr);
1486   emit_int8((unsigned char)0xFF);
1487   emit_operand(rdx, adr);
1488 }
1489 
1490 void Assembler::call_literal(address entry, RelocationHolder const& rspec) {
1491   assert(entry != NULL, "call most probably wrong");
1492   InstructionMark im(this);
1493   emit_int8((unsigned char)0xE8);
1494   intptr_t disp = entry - (pc() + sizeof(int32_t));
1495   assert(is_simm32(disp), "must be 32bit offset (call2)");
1496   // Technically, should use call32_operand, but this format is
1497   // implied by the fact that we're emitting a call instruction.
1498 
1499   int operand = LP64_ONLY(disp32_operand) NOT_LP64(call32_operand);
1500   emit_data((int) disp, rspec, operand);
1501 }
1502 
1503 void Assembler::cdql() {
1504   emit_int8((unsigned char)0x99);
1505 }
1506 
1507 void Assembler::cld() {
1508   emit_int8((unsigned char)0xFC);
1509 }
1510 
1511 void Assembler::cmovl(Condition cc, Register dst, Register src) {
1512   NOT_LP64(guarantee(VM_Version::supports_cmov(), "illegal instruction"));
1513   int encode = prefix_and_encode(dst->encoding(), src->encoding());
1514   emit_int8(0x0F);
1515   emit_int8(0x40 | cc);
1516   emit_int8((unsigned char)(0xC0 | encode));
1517 }
1518 
1519 
1520 void Assembler::cmovl(Condition cc, Register dst, Address src) {
1521   NOT_LP64(guarantee(VM_Version::supports_cmov(), "illegal instruction"));
1522   prefix(src, dst);
1523   emit_int8(0x0F);
1524   emit_int8(0x40 | cc);
1525   emit_operand(dst, src);
1526 }
1527 
1528 void Assembler::cmpb(Address dst, int imm8) {
1529   InstructionMark im(this);
1530   prefix(dst);
1531   emit_int8((unsigned char)0x80);
1532   emit_operand(rdi, dst, 1);
1533   emit_int8(imm8);
1534 }
1535 
1536 void Assembler::cmpl(Address dst, int32_t imm32) {
1537   InstructionMark im(this);
1538   prefix(dst);
1539   emit_int8((unsigned char)0x81);
1540   emit_operand(rdi, dst, 4);
1541   emit_int32(imm32);
1542 }
1543 
1544 void Assembler::cmpl(Register dst, int32_t imm32) {
1545   prefix(dst);
1546   emit_arith(0x81, 0xF8, dst, imm32);
1547 }
1548 
1549 void Assembler::cmpl(Register dst, Register src) {
1550   (void) prefix_and_encode(dst->encoding(), src->encoding());
1551   emit_arith(0x3B, 0xC0, dst, src);
1552 }
1553 
1554 void Assembler::cmpl(Register dst, Address  src) {
1555   InstructionMark im(this);
1556   prefix(src, dst);
1557   emit_int8((unsigned char)0x3B);
1558   emit_operand(dst, src);
1559 }
1560 
1561 void Assembler::cmpw(Address dst, int imm16) {
1562   InstructionMark im(this);
1563   assert(!dst.base_needs_rex() && !dst.index_needs_rex(), "no extended registers");
1564   emit_int8(0x66);
1565   emit_int8((unsigned char)0x81);
1566   emit_operand(rdi, dst, 2);
1567   emit_int16(imm16);
1568 }
1569 
1570 // The 32-bit cmpxchg compares the value at adr with the contents of rax,
1571 // and stores reg into adr if so; otherwise, the value at adr is loaded into rax,.
1572 // The ZF is set if the compared values were equal, and cleared otherwise.
1573 void Assembler::cmpxchgl(Register reg, Address adr) { // cmpxchg
1574   InstructionMark im(this);
1575   prefix(adr, reg);
1576   emit_int8(0x0F);
1577   emit_int8((unsigned char)0xB1);
1578   emit_operand(reg, adr);
1579 }
1580 
1581 // The 8-bit cmpxchg compares the value at adr with the contents of rax,
1582 // and stores reg into adr if so; otherwise, the value at adr is loaded into rax,.
1583 // The ZF is set if the compared values were equal, and cleared otherwise.
1584 void Assembler::cmpxchgb(Register reg, Address adr) { // cmpxchg
1585   InstructionMark im(this);
1586   prefix(adr, reg, true);
1587   emit_int8(0x0F);
1588   emit_int8((unsigned char)0xB0);
1589   emit_operand(reg, adr);
1590 }
1591 
1592 void Assembler::comisd(XMMRegister dst, Address src) {
1593   // NOTE: dbx seems to decode this as comiss even though the
1594   // 0x66 is there. Strangly ucomisd comes out correct
1595   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1596   InstructionMark im(this);
1597   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);;
1598   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
1599   simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
1600   emit_int8(0x2F);
1601   emit_operand(dst, src);
1602 }
1603 
1604 void Assembler::comisd(XMMRegister dst, XMMRegister src) {
1605   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1606   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1607   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
1608   emit_int8(0x2F);
1609   emit_int8((unsigned char)(0xC0 | encode));
1610 }
1611 
1612 void Assembler::comiss(XMMRegister dst, Address src) {
1613   NOT_LP64(assert(VM_Version::supports_sse(), ""));
1614   InstructionMark im(this);
1615   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1616   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
1617   simd_prefix(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
1618   emit_int8(0x2F);
1619   emit_operand(dst, src);
1620 }
1621 
1622 void Assembler::comiss(XMMRegister dst, XMMRegister src) {
1623   NOT_LP64(assert(VM_Version::supports_sse(), ""));
1624   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1625   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
1626   emit_int8(0x2F);
1627   emit_int8((unsigned char)(0xC0 | encode));
1628 }
1629 
1630 void Assembler::cpuid() {
1631   emit_int8(0x0F);
1632   emit_int8((unsigned char)0xA2);
1633 }
1634 
1635 // Opcode / Instruction                      Op /  En  64 - Bit Mode     Compat / Leg Mode Description                  Implemented
1636 // F2 0F 38 F0 / r       CRC32 r32, r / m8   RM        Valid             Valid             Accumulate CRC32 on r / m8.  v
1637 // F2 REX 0F 38 F0 / r   CRC32 r32, r / m8*  RM        Valid             N.E.              Accumulate CRC32 on r / m8.  -
1638 // F2 REX.W 0F 38 F0 / r CRC32 r64, r / m8   RM        Valid             N.E.              Accumulate CRC32 on r / m8.  -
1639 //
1640 // F2 0F 38 F1 / r       CRC32 r32, r / m16  RM        Valid             Valid             Accumulate CRC32 on r / m16. v
1641 //
1642 // F2 0F 38 F1 / r       CRC32 r32, r / m32  RM        Valid             Valid             Accumulate CRC32 on r / m32. v
1643 //
1644 // F2 REX.W 0F 38 F1 / r CRC32 r64, r / m64  RM        Valid             N.E.              Accumulate CRC32 on r / m64. v
1645 void Assembler::crc32(Register crc, Register v, int8_t sizeInBytes) {
1646   assert(VM_Version::supports_sse4_2(), "");
1647   int8_t w = 0x01;
1648   Prefix p = Prefix_EMPTY;
1649 
1650   emit_int8((int8_t)0xF2);
1651   switch (sizeInBytes) {
1652   case 1:
1653     w = 0;
1654     break;
1655   case 2:
1656   case 4:
1657     break;
1658   LP64_ONLY(case 8:)
1659     // This instruction is not valid in 32 bits
1660     // Note:
1661     // http://www.intel.com/content/dam/www/public/us/en/documents/manuals/64-ia-32-architectures-software-developer-instruction-set-reference-manual-325383.pdf
1662     //
1663     // Page B - 72   Vol. 2C says
1664     // qwreg2 to qwreg            1111 0010 : 0100 1R0B : 0000 1111 : 0011 1000 : 1111 0000 : 11 qwreg1 qwreg2
1665     // mem64 to qwreg             1111 0010 : 0100 1R0B : 0000 1111 : 0011 1000 : 1111 0000 : mod qwreg r / m
1666     //                                                                            F0!!!
1667     // while 3 - 208 Vol. 2A
1668     // F2 REX.W 0F 38 F1 / r       CRC32 r64, r / m64             RM         Valid      N.E.Accumulate CRC32 on r / m64.
1669     //
1670     // the 0 on a last bit is reserved for a different flavor of this instruction :
1671     // F2 REX.W 0F 38 F0 / r       CRC32 r64, r / m8              RM         Valid      N.E.Accumulate CRC32 on r / m8.
1672     p = REX_W;
1673     break;
1674   default:
1675     assert(0, "Unsupported value for a sizeInBytes argument");
1676     break;
1677   }
1678   LP64_ONLY(prefix(crc, v, p);)
1679   emit_int8((int8_t)0x0F);
1680   emit_int8(0x38);
1681   emit_int8((int8_t)(0xF0 | w));
1682   emit_int8(0xC0 | ((crc->encoding() & 0x7) << 3) | (v->encoding() & 7));
1683 }
1684 
1685 void Assembler::crc32(Register crc, Address adr, int8_t sizeInBytes) {
1686   assert(VM_Version::supports_sse4_2(), "");
1687   InstructionMark im(this);
1688   int8_t w = 0x01;
1689   Prefix p = Prefix_EMPTY;
1690 
1691   emit_int8((int8_t)0xF2);
1692   switch (sizeInBytes) {
1693   case 1:
1694     w = 0;
1695     break;
1696   case 2:
1697   case 4:
1698     break;
1699   LP64_ONLY(case 8:)
1700     // This instruction is not valid in 32 bits
1701     p = REX_W;
1702     break;
1703   default:
1704     assert(0, "Unsupported value for a sizeInBytes argument");
1705     break;
1706   }
1707   LP64_ONLY(prefix(crc, adr, p);)
1708   emit_int8((int8_t)0x0F);
1709   emit_int8(0x38);
1710   emit_int8((int8_t)(0xF0 | w));
1711   emit_operand(crc, adr);
1712 }
1713 
1714 void Assembler::cvtdq2pd(XMMRegister dst, XMMRegister src) {
1715   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1716   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
1717   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
1718   emit_int8((unsigned char)0xE6);
1719   emit_int8((unsigned char)(0xC0 | encode));
1720 }
1721 
1722 void Assembler::cvtdq2ps(XMMRegister dst, XMMRegister src) {
1723   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1724   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
1725   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
1726   emit_int8(0x5B);
1727   emit_int8((unsigned char)(0xC0 | encode));
1728 }
1729 
1730 void Assembler::cvtsd2ss(XMMRegister dst, XMMRegister src) {
1731   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1732   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
1733   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
1734   emit_int8(0x5A);
1735   emit_int8((unsigned char)(0xC0 | encode));
1736 }
1737 
1738 void Assembler::cvtsd2ss(XMMRegister dst, Address src) {
1739   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1740   InstructionMark im(this);
1741   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
1742   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
1743   simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
1744   emit_int8(0x5A);
1745   emit_operand(dst, src);
1746 }
1747 
1748 void Assembler::cvtsi2sdl(XMMRegister dst, Register src) {
1749   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1750   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1751   int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
1752   emit_int8(0x2A);
1753   emit_int8((unsigned char)(0xC0 | encode));
1754 }
1755 
1756 void Assembler::cvtsi2sdl(XMMRegister dst, Address src) {
1757   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1758   InstructionMark im(this);
1759   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1760   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
1761   simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
1762   emit_int8(0x2A);
1763   emit_operand(dst, src);
1764 }
1765 
1766 void Assembler::cvtsi2ssl(XMMRegister dst, Register src) {
1767   NOT_LP64(assert(VM_Version::supports_sse(), ""));
1768   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1769   int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
1770   emit_int8(0x2A);
1771   emit_int8((unsigned char)(0xC0 | encode));
1772 }
1773 
1774 void Assembler::cvtsi2ssl(XMMRegister dst, Address src) {
1775   NOT_LP64(assert(VM_Version::supports_sse(), ""));
1776   InstructionMark im(this);
1777   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1778   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
1779   simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
1780   emit_int8(0x2A);
1781   emit_operand(dst, src);
1782 }
1783 
1784 void Assembler::cvtsi2ssq(XMMRegister dst, Register src) {
1785   NOT_LP64(assert(VM_Version::supports_sse(), ""));
1786   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1787   int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
1788   emit_int8(0x2A);
1789   emit_int8((unsigned char)(0xC0 | encode));
1790 }
1791 
1792 void Assembler::cvtss2sd(XMMRegister dst, XMMRegister src) {
1793   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1794   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
1795   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
1796   emit_int8(0x5A);
1797   emit_int8((unsigned char)(0xC0 | encode));
1798 }
1799 
1800 void Assembler::cvtss2sd(XMMRegister dst, Address src) {
1801   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1802   InstructionMark im(this);
1803   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
1804   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
1805   simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
1806   emit_int8(0x5A);
1807   emit_operand(dst, src);
1808 }
1809 
1810 
1811 void Assembler::cvttsd2sil(Register dst, XMMRegister src) {
1812   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1813   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1814   int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
1815   emit_int8(0x2C);
1816   emit_int8((unsigned char)(0xC0 | encode));
1817 }
1818 
1819 void Assembler::cvttss2sil(Register dst, XMMRegister src) {
1820   NOT_LP64(assert(VM_Version::supports_sse(), ""));
1821   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1822   int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
1823   emit_int8(0x2C);
1824   emit_int8((unsigned char)(0xC0 | encode));
1825 }
1826 
1827 void Assembler::decl(Address dst) {
1828   // Don't use it directly. Use MacroAssembler::decrement() instead.
1829   InstructionMark im(this);
1830   prefix(dst);
1831   emit_int8((unsigned char)0xFF);
1832   emit_operand(rcx, dst);
1833 }
1834 
1835 void Assembler::divsd(XMMRegister dst, Address src) {
1836   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1837   InstructionMark im(this);
1838   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
1839   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
1840   simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
1841   emit_int8(0x5E);
1842   emit_operand(dst, src);
1843 }
1844 
1845 void Assembler::divsd(XMMRegister dst, XMMRegister src) {
1846   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1847   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
1848   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
1849   emit_int8(0x5E);
1850   emit_int8((unsigned char)(0xC0 | encode));
1851 }
1852 
1853 void Assembler::divss(XMMRegister dst, Address src) {
1854   NOT_LP64(assert(VM_Version::supports_sse(), ""));
1855   InstructionMark im(this);
1856   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
1857   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
1858   simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
1859   emit_int8(0x5E);
1860   emit_operand(dst, src);
1861 }
1862 
1863 void Assembler::divss(XMMRegister dst, XMMRegister src) {
1864   NOT_LP64(assert(VM_Version::supports_sse(), ""));
1865   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
1866   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
1867   emit_int8(0x5E);
1868   emit_int8((unsigned char)(0xC0 | encode));
1869 }
1870 
1871 void Assembler::emms() {
1872   NOT_LP64(assert(VM_Version::supports_mmx(), ""));
1873   emit_int8(0x0F);
1874   emit_int8(0x77);
1875 }
1876 
1877 void Assembler::hlt() {
1878   emit_int8((unsigned char)0xF4);
1879 }
1880 
1881 void Assembler::idivl(Register src) {
1882   int encode = prefix_and_encode(src->encoding());
1883   emit_int8((unsigned char)0xF7);
1884   emit_int8((unsigned char)(0xF8 | encode));
1885 }
1886 
1887 void Assembler::divl(Register src) { // Unsigned
1888   int encode = prefix_and_encode(src->encoding());
1889   emit_int8((unsigned char)0xF7);
1890   emit_int8((unsigned char)(0xF0 | encode));
1891 }
1892 
1893 void Assembler::imull(Register dst, Register src) {
1894   int encode = prefix_and_encode(dst->encoding(), src->encoding());
1895   emit_int8(0x0F);
1896   emit_int8((unsigned char)0xAF);
1897   emit_int8((unsigned char)(0xC0 | encode));
1898 }
1899 
1900 
1901 void Assembler::imull(Register dst, Register src, int value) {
1902   int encode = prefix_and_encode(dst->encoding(), src->encoding());
1903   if (is8bit(value)) {
1904     emit_int8(0x6B);
1905     emit_int8((unsigned char)(0xC0 | encode));
1906     emit_int8(value & 0xFF);
1907   } else {
1908     emit_int8(0x69);
1909     emit_int8((unsigned char)(0xC0 | encode));
1910     emit_int32(value);
1911   }
1912 }
1913 
1914 void Assembler::imull(Register dst, Address src) {
1915   InstructionMark im(this);
1916   prefix(src, dst);
1917   emit_int8(0x0F);
1918   emit_int8((unsigned char) 0xAF);
1919   emit_operand(dst, src);
1920 }
1921 
1922 
1923 void Assembler::incl(Address dst) {
1924   // Don't use it directly. Use MacroAssembler::increment() instead.
1925   InstructionMark im(this);
1926   prefix(dst);
1927   emit_int8((unsigned char)0xFF);
1928   emit_operand(rax, dst);
1929 }
1930 
1931 void Assembler::jcc(Condition cc, Label& L, bool maybe_short) {
1932   InstructionMark im(this);
1933   assert((0 <= cc) && (cc < 16), "illegal cc");
1934   if (L.is_bound()) {
1935     address dst = target(L);
1936     assert(dst != NULL, "jcc most probably wrong");
1937 
1938     const int short_size = 2;
1939     const int long_size = 6;
1940     intptr_t offs = (intptr_t)dst - (intptr_t)pc();
1941     if (maybe_short && is8bit(offs - short_size)) {
1942       // 0111 tttn #8-bit disp
1943       emit_int8(0x70 | cc);
1944       emit_int8((offs - short_size) & 0xFF);
1945     } else {
1946       // 0000 1111 1000 tttn #32-bit disp
1947       assert(is_simm32(offs - long_size),
1948              "must be 32bit offset (call4)");
1949       emit_int8(0x0F);
1950       emit_int8((unsigned char)(0x80 | cc));
1951       emit_int32(offs - long_size);
1952     }
1953   } else {
1954     // Note: could eliminate cond. jumps to this jump if condition
1955     //       is the same however, seems to be rather unlikely case.
1956     // Note: use jccb() if label to be bound is very close to get
1957     //       an 8-bit displacement
1958     L.add_patch_at(code(), locator());
1959     emit_int8(0x0F);
1960     emit_int8((unsigned char)(0x80 | cc));
1961     emit_int32(0);
1962   }
1963 }
1964 
1965 void Assembler::jccb(Condition cc, Label& L) {
1966   if (L.is_bound()) {
1967     const int short_size = 2;
1968     address entry = target(L);
1969 #ifdef ASSERT
1970     intptr_t dist = (intptr_t)entry - ((intptr_t)pc() + short_size);
1971     intptr_t delta = short_branch_delta();
1972     if (delta != 0) {
1973       dist += (dist < 0 ? (-delta) :delta);
1974     }
1975     assert(is8bit(dist), "Dispacement too large for a short jmp");
1976 #endif
1977     intptr_t offs = (intptr_t)entry - (intptr_t)pc();
1978     // 0111 tttn #8-bit disp
1979     emit_int8(0x70 | cc);
1980     emit_int8((offs - short_size) & 0xFF);
1981   } else {
1982     InstructionMark im(this);
1983     L.add_patch_at(code(), locator());
1984     emit_int8(0x70 | cc);
1985     emit_int8(0);
1986   }
1987 }
1988 
1989 void Assembler::jmp(Address adr) {
1990   InstructionMark im(this);
1991   prefix(adr);
1992   emit_int8((unsigned char)0xFF);
1993   emit_operand(rsp, adr);
1994 }
1995 
1996 void Assembler::jmp(Label& L, bool maybe_short) {
1997   if (L.is_bound()) {
1998     address entry = target(L);
1999     assert(entry != NULL, "jmp most probably wrong");
2000     InstructionMark im(this);
2001     const int short_size = 2;
2002     const int long_size = 5;
2003     intptr_t offs = entry - pc();
2004     if (maybe_short && is8bit(offs - short_size)) {
2005       emit_int8((unsigned char)0xEB);
2006       emit_int8((offs - short_size) & 0xFF);
2007     } else {
2008       emit_int8((unsigned char)0xE9);
2009       emit_int32(offs - long_size);
2010     }
2011   } else {
2012     // By default, forward jumps are always 32-bit displacements, since
2013     // we can't yet know where the label will be bound.  If you're sure that
2014     // the forward jump will not run beyond 256 bytes, use jmpb to
2015     // force an 8-bit displacement.
2016     InstructionMark im(this);
2017     L.add_patch_at(code(), locator());
2018     emit_int8((unsigned char)0xE9);
2019     emit_int32(0);
2020   }
2021 }
2022 
2023 void Assembler::jmp(Register entry) {
2024   int encode = prefix_and_encode(entry->encoding());
2025   emit_int8((unsigned char)0xFF);
2026   emit_int8((unsigned char)(0xE0 | encode));
2027 }
2028 
2029 void Assembler::jmp_literal(address dest, RelocationHolder const& rspec) {
2030   InstructionMark im(this);
2031   emit_int8((unsigned char)0xE9);
2032   assert(dest != NULL, "must have a target");
2033   intptr_t disp = dest - (pc() + sizeof(int32_t));
2034   assert(is_simm32(disp), "must be 32bit offset (jmp)");
2035   emit_data(disp, rspec.reloc(), call32_operand);
2036 }
2037 
2038 void Assembler::jmpb(Label& L) {
2039   if (L.is_bound()) {
2040     const int short_size = 2;
2041     address entry = target(L);
2042     assert(entry != NULL, "jmp most probably wrong");
2043 #ifdef ASSERT
2044     intptr_t dist = (intptr_t)entry - ((intptr_t)pc() + short_size);
2045     intptr_t delta = short_branch_delta();
2046     if (delta != 0) {
2047       dist += (dist < 0 ? (-delta) :delta);
2048     }
2049     assert(is8bit(dist), "Dispacement too large for a short jmp");
2050 #endif
2051     intptr_t offs = entry - pc();
2052     emit_int8((unsigned char)0xEB);
2053     emit_int8((offs - short_size) & 0xFF);
2054   } else {
2055     InstructionMark im(this);
2056     L.add_patch_at(code(), locator());
2057     emit_int8((unsigned char)0xEB);
2058     emit_int8(0);
2059   }
2060 }
2061 
2062 void Assembler::ldmxcsr( Address src) {
2063   NOT_LP64(assert(VM_Version::supports_sse(), ""));
2064   InstructionMark im(this);
2065   prefix(src);
2066   emit_int8(0x0F);
2067   emit_int8((unsigned char)0xAE);
2068   emit_operand(as_Register(2), src);
2069 }
2070 
2071 void Assembler::leal(Register dst, Address src) {
2072   InstructionMark im(this);
2073 #ifdef _LP64
2074   emit_int8(0x67); // addr32
2075   prefix(src, dst);
2076 #endif // LP64
2077   emit_int8((unsigned char)0x8D);
2078   emit_operand(dst, src);
2079 }
2080 
2081 void Assembler::lfence() {
2082   emit_int8(0x0F);
2083   emit_int8((unsigned char)0xAE);
2084   emit_int8((unsigned char)0xE8);
2085 }
2086 
2087 void Assembler::lock() {
2088   emit_int8((unsigned char)0xF0);
2089 }
2090 
2091 void Assembler::lzcntl(Register dst, Register src) {
2092   assert(VM_Version::supports_lzcnt(), "encoding is treated as BSR");
2093   emit_int8((unsigned char)0xF3);
2094   int encode = prefix_and_encode(dst->encoding(), src->encoding());
2095   emit_int8(0x0F);
2096   emit_int8((unsigned char)0xBD);
2097   emit_int8((unsigned char)(0xC0 | encode));
2098 }
2099 
2100 // Emit mfence instruction
2101 void Assembler::mfence() {
2102   NOT_LP64(assert(VM_Version::supports_sse2(), "unsupported");)
2103   emit_int8(0x0F);
2104   emit_int8((unsigned char)0xAE);
2105   emit_int8((unsigned char)0xF0);
2106 }
2107 
2108 void Assembler::mov(Register dst, Register src) {
2109   LP64_ONLY(movq(dst, src)) NOT_LP64(movl(dst, src));
2110 }
2111 
2112 void Assembler::movapd(XMMRegister dst, XMMRegister src) {
2113   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2114   int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_128bit;
2115   InstructionAttr attributes(vector_len, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
2116   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2117   emit_int8(0x28);
2118   emit_int8((unsigned char)(0xC0 | encode));
2119 }
2120 
2121 void Assembler::movaps(XMMRegister dst, XMMRegister src) {
2122   NOT_LP64(assert(VM_Version::supports_sse(), ""));
2123   int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_128bit;
2124   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
2125   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2126   emit_int8(0x28);
2127   emit_int8((unsigned char)(0xC0 | encode));
2128 }
2129 
2130 void Assembler::movlhps(XMMRegister dst, XMMRegister src) {
2131   NOT_LP64(assert(VM_Version::supports_sse(), ""));
2132   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2133   int encode = simd_prefix_and_encode(dst, src, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2134   emit_int8(0x16);
2135   emit_int8((unsigned char)(0xC0 | encode));
2136 }
2137 
2138 void Assembler::movb(Register dst, Address src) {
2139   NOT_LP64(assert(dst->has_byte_register(), "must have byte register"));
2140   InstructionMark im(this);
2141   prefix(src, dst, true);
2142   emit_int8((unsigned char)0x8A);
2143   emit_operand(dst, src);
2144 }
2145 
2146 void Assembler::movddup(XMMRegister dst, XMMRegister src) {
2147   NOT_LP64(assert(VM_Version::supports_sse3(), ""));
2148   int vector_len = VM_Version::supports_evex() ? AVX_512bit : AVX_128bit;
2149   InstructionAttr attributes(vector_len, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2150   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2151   emit_int8(0x12);
2152   emit_int8(0xC0 | encode);
2153 }
2154 
2155 void Assembler::kmovql(KRegister dst, KRegister src) {
2156   NOT_LP64(assert(VM_Version::supports_evex(), ""));
2157   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2158   int encode = kreg_prefix_and_encode(dst, knoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2159   emit_int8((unsigned char)0x90);
2160   emit_int8((unsigned char)(0xC0 | encode));
2161 }
2162 
2163 void Assembler::kmovql(KRegister dst, Address src) {
2164   NOT_LP64(assert(VM_Version::supports_evex(), ""));
2165   InstructionMark im(this);
2166   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2167   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2168   emit_int8((unsigned char)0x90);
2169   emit_operand((Register)dst, src);
2170 }
2171 
2172 void Assembler::kmovql(Address dst, KRegister src) {
2173   NOT_LP64(assert(VM_Version::supports_evex(), ""));
2174   InstructionMark im(this);
2175   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2176   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2177   emit_int8((unsigned char)0x90);
2178   emit_operand((Register)src, dst);
2179 }
2180 
2181 void Assembler::kmovql(KRegister dst, Register src) {
2182   NOT_LP64(assert(VM_Version::supports_evex(), ""));
2183   VexSimdPrefix pre = !_legacy_mode_bw ? VEX_SIMD_F2 : VEX_SIMD_NONE;
2184   InstructionAttr attributes(AVX_128bit, /* rex_w */ !_legacy_mode_bw, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2185   int encode = kreg_prefix_and_encode(dst, knoreg, src, pre, VEX_OPCODE_0F, &attributes);
2186   emit_int8((unsigned char)0x92);
2187   emit_int8((unsigned char)(0xC0 | encode));
2188 }
2189 
2190 void Assembler::kmovdl(KRegister dst, Register src) {
2191   NOT_LP64(assert(VM_Version::supports_evex(), ""));
2192   VexSimdPrefix pre = !_legacy_mode_bw ? VEX_SIMD_F2 : VEX_SIMD_NONE;
2193   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2194   int encode = kreg_prefix_and_encode(dst, knoreg, src, pre, VEX_OPCODE_0F, &attributes);
2195   emit_int8((unsigned char)0x92);
2196   emit_int8((unsigned char)(0xC0 | encode));
2197 }
2198 
2199 void Assembler::kmovwl(KRegister dst, Register src) {
2200   NOT_LP64(assert(VM_Version::supports_evex(), ""));
2201   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2202   int encode = kreg_prefix_and_encode(dst, knoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2203   emit_int8((unsigned char)0x92);
2204   emit_int8((unsigned char)(0xC0 | encode));
2205 }
2206 
2207 void Assembler::movb(Address dst, int imm8) {
2208   InstructionMark im(this);
2209    prefix(dst);
2210   emit_int8((unsigned char)0xC6);
2211   emit_operand(rax, dst, 1);
2212   emit_int8(imm8);
2213 }
2214 
2215 
2216 void Assembler::movb(Address dst, Register src) {
2217   assert(src->has_byte_register(), "must have byte register");
2218   InstructionMark im(this);
2219   prefix(dst, src, true);
2220   emit_int8((unsigned char)0x88);
2221   emit_operand(src, dst);
2222 }
2223 
2224 void Assembler::movdl(XMMRegister dst, Register src) {
2225   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2226   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2227   int encode = simd_prefix_and_encode(dst, xnoreg, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2228   emit_int8(0x6E);
2229   emit_int8((unsigned char)(0xC0 | encode));
2230 }
2231 
2232 void Assembler::movdl(Register dst, XMMRegister src) {
2233   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2234   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2235   // swap src/dst to get correct prefix
2236   int encode = simd_prefix_and_encode(src, xnoreg, as_XMMRegister(dst->encoding()), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2237   emit_int8(0x7E);
2238   emit_int8((unsigned char)(0xC0 | encode));
2239 }
2240 
2241 void Assembler::movdl(XMMRegister dst, Address src) {
2242   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2243   InstructionMark im(this);
2244   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2245   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
2246   simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2247   emit_int8(0x6E);
2248   emit_operand(dst, src);
2249 }
2250 
2251 void Assembler::movdl(Address dst, XMMRegister src) {
2252   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2253   InstructionMark im(this);
2254   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2255   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
2256   simd_prefix(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2257   emit_int8(0x7E);
2258   emit_operand(src, dst);
2259 }
2260 
2261 void Assembler::movdqa(XMMRegister dst, XMMRegister src) {
2262   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2263   int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_128bit;
2264   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2265   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2266   emit_int8(0x6F);
2267   emit_int8((unsigned char)(0xC0 | encode));
2268 }
2269 
2270 void Assembler::movdqa(XMMRegister dst, Address src) {
2271   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2272   InstructionMark im(this);
2273   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2274   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2275   simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2276   emit_int8(0x6F);
2277   emit_operand(dst, src);
2278 }
2279 
2280 void Assembler::movdqu(XMMRegister dst, Address src) {
2281   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2282   InstructionMark im(this);
2283   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
2284   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2285   simd_prefix(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2286   emit_int8(0x6F);
2287   emit_operand(dst, src);
2288 }
2289 
2290 void Assembler::movdqu(XMMRegister dst, XMMRegister src) {
2291   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2292   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2293   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2294   emit_int8(0x6F);
2295   emit_int8((unsigned char)(0xC0 | encode));
2296 }
2297 
2298 void Assembler::movdqu(Address dst, XMMRegister src) {
2299   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2300   InstructionMark im(this);
2301   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2302   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2303   simd_prefix(src, xnoreg, dst, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2304   emit_int8(0x7F);
2305   emit_operand(src, dst);
2306 }
2307 
2308 // Move Unaligned 256bit Vector
2309 void Assembler::vmovdqu(XMMRegister dst, XMMRegister src) {
2310   assert(UseAVX > 0, "");
2311   InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2312   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2313   emit_int8(0x6F);
2314   emit_int8((unsigned char)(0xC0 | encode));
2315 }
2316 
2317 void Assembler::vmovdqu(XMMRegister dst, Address src) {
2318   assert(UseAVX > 0, "");
2319   InstructionMark im(this);
2320   InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
2321   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2322   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2323   emit_int8(0x6F);
2324   emit_operand(dst, src);
2325 }
2326 
2327 void Assembler::vmovdqu(Address dst, XMMRegister src) {
2328   assert(UseAVX > 0, "");
2329   InstructionMark im(this);
2330   InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2331   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2332   // swap src<->dst for encoding
2333   assert(src != xnoreg, "sanity");
2334   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2335   emit_int8(0x7F);
2336   emit_operand(src, dst);
2337 }
2338 
2339 // Move Unaligned EVEX enabled Vector (programmable : 8,16,32,64)
2340 void Assembler::evmovdqul(XMMRegister dst, XMMRegister src, int vector_len) {
2341   assert(VM_Version::supports_evex(), "");
2342   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2343   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2344   emit_int8(0x6F);
2345   emit_int8((unsigned char)(0xC0 | encode));
2346 }
2347 
2348 void Assembler::evmovdqul(XMMRegister dst, Address src, int vector_len) {
2349   assert(VM_Version::supports_evex(), "");
2350   InstructionMark im(this);
2351   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
2352   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2353   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2354   emit_int8(0x6F);
2355   emit_operand(dst, src);
2356 }
2357 
2358 void Assembler::evmovdqul(Address dst, XMMRegister src, int vector_len) {
2359   assert(VM_Version::supports_evex(), "");
2360   assert(src != xnoreg, "sanity");
2361   InstructionMark im(this);
2362   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2363   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2364   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2365   emit_int8(0x7F);
2366   emit_operand(src, dst);
2367 }
2368 
2369 void Assembler::evmovdquq(XMMRegister dst, XMMRegister src, int vector_len) {
2370   assert(VM_Version::supports_evex(), "");
2371   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2372   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2373   emit_int8(0x6F);
2374   emit_int8((unsigned char)(0xC0 | encode));
2375 }
2376 
2377 void Assembler::evmovdquq(XMMRegister dst, Address src, int vector_len) {
2378   assert(VM_Version::supports_evex(), "");
2379   InstructionMark im(this);
2380   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
2381   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2382   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2383   emit_int8(0x6F);
2384   emit_operand(dst, src);
2385 }
2386 
2387 void Assembler::evmovdquq(Address dst, XMMRegister src, int vector_len) {
2388   assert(VM_Version::supports_evex(), "");
2389   assert(src != xnoreg, "sanity");
2390   InstructionMark im(this);
2391   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2392   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2393   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2394   emit_int8(0x7F);
2395   emit_operand(src, dst);
2396 }
2397 
2398 // Uses zero extension on 64bit
2399 
2400 void Assembler::movl(Register dst, int32_t imm32) {
2401   int encode = prefix_and_encode(dst->encoding());
2402   emit_int8((unsigned char)(0xB8 | encode));
2403   emit_int32(imm32);
2404 }
2405 
2406 void Assembler::movl(Register dst, Register src) {
2407   int encode = prefix_and_encode(dst->encoding(), src->encoding());
2408   emit_int8((unsigned char)0x8B);
2409   emit_int8((unsigned char)(0xC0 | encode));
2410 }
2411 
2412 void Assembler::movl(Register dst, Address src) {
2413   InstructionMark im(this);
2414   prefix(src, dst);
2415   emit_int8((unsigned char)0x8B);
2416   emit_operand(dst, src);
2417 }
2418 
2419 void Assembler::movl(Address dst, int32_t imm32) {
2420   InstructionMark im(this);
2421   prefix(dst);
2422   emit_int8((unsigned char)0xC7);
2423   emit_operand(rax, dst, 4);
2424   emit_int32(imm32);
2425 }
2426 
2427 void Assembler::movl(Address dst, Register src) {
2428   InstructionMark im(this);
2429   prefix(dst, src);
2430   emit_int8((unsigned char)0x89);
2431   emit_operand(src, dst);
2432 }
2433 
2434 // New cpus require to use movsd and movss to avoid partial register stall
2435 // when loading from memory. But for old Opteron use movlpd instead of movsd.
2436 // The selection is done in MacroAssembler::movdbl() and movflt().
2437 void Assembler::movlpd(XMMRegister dst, Address src) {
2438   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2439   InstructionMark im(this);
2440   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2441   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
2442   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2443   emit_int8(0x12);
2444   emit_operand(dst, src);
2445 }
2446 
2447 void Assembler::movq( MMXRegister dst, Address src ) {
2448   assert( VM_Version::supports_mmx(), "" );
2449   emit_int8(0x0F);
2450   emit_int8(0x6F);
2451   emit_operand(dst, src);
2452 }
2453 
2454 void Assembler::movq( Address dst, MMXRegister src ) {
2455   assert( VM_Version::supports_mmx(), "" );
2456   emit_int8(0x0F);
2457   emit_int8(0x7F);
2458   // workaround gcc (3.2.1-7a) bug
2459   // In that version of gcc with only an emit_operand(MMX, Address)
2460   // gcc will tail jump and try and reverse the parameters completely
2461   // obliterating dst in the process. By having a version available
2462   // that doesn't need to swap the args at the tail jump the bug is
2463   // avoided.
2464   emit_operand(dst, src);
2465 }
2466 
2467 void Assembler::movq(XMMRegister dst, Address src) {
2468   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2469   InstructionMark im(this);
2470   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2471   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
2472   simd_prefix(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2473   emit_int8(0x7E);
2474   emit_operand(dst, src);
2475 }
2476 
2477 void Assembler::movq(Address dst, XMMRegister src) {
2478   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2479   InstructionMark im(this);
2480   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2481   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
2482   simd_prefix(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2483   emit_int8((unsigned char)0xD6);
2484   emit_operand(src, dst);
2485 }
2486 
2487 void Assembler::movsbl(Register dst, Address src) { // movsxb
2488   InstructionMark im(this);
2489   prefix(src, dst);
2490   emit_int8(0x0F);
2491   emit_int8((unsigned char)0xBE);
2492   emit_operand(dst, src);
2493 }
2494 
2495 void Assembler::movsbl(Register dst, Register src) { // movsxb
2496   NOT_LP64(assert(src->has_byte_register(), "must have byte register"));
2497   int encode = prefix_and_encode(dst->encoding(), false, src->encoding(), true);
2498   emit_int8(0x0F);
2499   emit_int8((unsigned char)0xBE);
2500   emit_int8((unsigned char)(0xC0 | encode));
2501 }
2502 
2503 void Assembler::movsd(XMMRegister dst, XMMRegister src) {
2504   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2505   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2506   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2507   emit_int8(0x10);
2508   emit_int8((unsigned char)(0xC0 | encode));
2509 }
2510 
2511 void Assembler::movsd(XMMRegister dst, Address src) {
2512   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2513   InstructionMark im(this);
2514   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2515   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
2516   simd_prefix(dst, xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2517   emit_int8(0x10);
2518   emit_operand(dst, src);
2519 }
2520 
2521 void Assembler::movsd(Address dst, XMMRegister src) {
2522   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2523   InstructionMark im(this);
2524   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
2525   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
2526   simd_prefix(src, xnoreg, dst, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2527   emit_int8(0x11);
2528   emit_operand(src, dst);
2529 }
2530 
2531 void Assembler::movss(XMMRegister dst, XMMRegister src) {
2532   NOT_LP64(assert(VM_Version::supports_sse(), ""));
2533   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2534   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2535   emit_int8(0x10);
2536   emit_int8((unsigned char)(0xC0 | encode));
2537 }
2538 
2539 void Assembler::movss(XMMRegister dst, Address src) {
2540   NOT_LP64(assert(VM_Version::supports_sse(), ""));
2541   InstructionMark im(this);
2542   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2543   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
2544   simd_prefix(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2545   emit_int8(0x10);
2546   emit_operand(dst, src);
2547 }
2548 
2549 void Assembler::movss(Address dst, XMMRegister src) {
2550   NOT_LP64(assert(VM_Version::supports_sse(), ""));
2551   InstructionMark im(this);
2552   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
2553   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
2554   simd_prefix(src, xnoreg, dst, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2555   emit_int8(0x11);
2556   emit_operand(src, dst);
2557 }
2558 
2559 void Assembler::movswl(Register dst, Address src) { // movsxw
2560   InstructionMark im(this);
2561   prefix(src, dst);
2562   emit_int8(0x0F);
2563   emit_int8((unsigned char)0xBF);
2564   emit_operand(dst, src);
2565 }
2566 
2567 void Assembler::movswl(Register dst, Register src) { // movsxw
2568   int encode = prefix_and_encode(dst->encoding(), src->encoding());
2569   emit_int8(0x0F);
2570   emit_int8((unsigned char)0xBF);
2571   emit_int8((unsigned char)(0xC0 | encode));
2572 }
2573 
2574 void Assembler::movw(Address dst, int imm16) {
2575   InstructionMark im(this);
2576 
2577   emit_int8(0x66); // switch to 16-bit mode
2578   prefix(dst);
2579   emit_int8((unsigned char)0xC7);
2580   emit_operand(rax, dst, 2);
2581   emit_int16(imm16);
2582 }
2583 
2584 void Assembler::movw(Register dst, Address src) {
2585   InstructionMark im(this);
2586   emit_int8(0x66);
2587   prefix(src, dst);
2588   emit_int8((unsigned char)0x8B);
2589   emit_operand(dst, src);
2590 }
2591 
2592 void Assembler::movw(Address dst, Register src) {
2593   InstructionMark im(this);
2594   emit_int8(0x66);
2595   prefix(dst, src);
2596   emit_int8((unsigned char)0x89);
2597   emit_operand(src, dst);
2598 }
2599 
2600 void Assembler::movzbl(Register dst, Address src) { // movzxb
2601   InstructionMark im(this);
2602   prefix(src, dst);
2603   emit_int8(0x0F);
2604   emit_int8((unsigned char)0xB6);
2605   emit_operand(dst, src);
2606 }
2607 
2608 void Assembler::movzbl(Register dst, Register src) { // movzxb
2609   NOT_LP64(assert(src->has_byte_register(), "must have byte register"));
2610   int encode = prefix_and_encode(dst->encoding(), false, src->encoding(), true);
2611   emit_int8(0x0F);
2612   emit_int8((unsigned char)0xB6);
2613   emit_int8(0xC0 | encode);
2614 }
2615 
2616 void Assembler::movzwl(Register dst, Address src) { // movzxw
2617   InstructionMark im(this);
2618   prefix(src, dst);
2619   emit_int8(0x0F);
2620   emit_int8((unsigned char)0xB7);
2621   emit_operand(dst, src);
2622 }
2623 
2624 void Assembler::movzwl(Register dst, Register src) { // movzxw
2625   int encode = prefix_and_encode(dst->encoding(), src->encoding());
2626   emit_int8(0x0F);
2627   emit_int8((unsigned char)0xB7);
2628   emit_int8(0xC0 | encode);
2629 }
2630 
2631 void Assembler::mull(Address src) {
2632   InstructionMark im(this);
2633   prefix(src);
2634   emit_int8((unsigned char)0xF7);
2635   emit_operand(rsp, src);
2636 }
2637 
2638 void Assembler::mull(Register src) {
2639   int encode = prefix_and_encode(src->encoding());
2640   emit_int8((unsigned char)0xF7);
2641   emit_int8((unsigned char)(0xE0 | encode));
2642 }
2643 
2644 void Assembler::mulsd(XMMRegister dst, Address src) {
2645   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2646   InstructionMark im(this);
2647   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2648   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
2649   simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2650   emit_int8(0x59);
2651   emit_operand(dst, src);
2652 }
2653 
2654 void Assembler::mulsd(XMMRegister dst, XMMRegister src) {
2655   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2656   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2657   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2658   emit_int8(0x59);
2659   emit_int8((unsigned char)(0xC0 | encode));
2660 }
2661 
2662 void Assembler::mulss(XMMRegister dst, Address src) {
2663   NOT_LP64(assert(VM_Version::supports_sse(), ""));
2664   InstructionMark im(this);
2665   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2666   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
2667   simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2668   emit_int8(0x59);
2669   emit_operand(dst, src);
2670 }
2671 
2672 void Assembler::mulss(XMMRegister dst, XMMRegister src) {
2673   NOT_LP64(assert(VM_Version::supports_sse(), ""));
2674   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2675   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2676   emit_int8(0x59);
2677   emit_int8((unsigned char)(0xC0 | encode));
2678 }
2679 
2680 void Assembler::negl(Register dst) {
2681   int encode = prefix_and_encode(dst->encoding());
2682   emit_int8((unsigned char)0xF7);
2683   emit_int8((unsigned char)(0xD8 | encode));
2684 }
2685 
2686 void Assembler::nop(int i) {
2687 #ifdef ASSERT
2688   assert(i > 0, " ");
2689   // The fancy nops aren't currently recognized by debuggers making it a
2690   // pain to disassemble code while debugging. If asserts are on clearly
2691   // speed is not an issue so simply use the single byte traditional nop
2692   // to do alignment.
2693 
2694   for (; i > 0 ; i--) emit_int8((unsigned char)0x90);
2695   return;
2696 
2697 #endif // ASSERT
2698 
2699   if (UseAddressNop && VM_Version::is_intel()) {
2700     //
2701     // Using multi-bytes nops "0x0F 0x1F [address]" for Intel
2702     //  1: 0x90
2703     //  2: 0x66 0x90
2704     //  3: 0x66 0x66 0x90 (don't use "0x0F 0x1F 0x00" - need patching safe padding)
2705     //  4: 0x0F 0x1F 0x40 0x00
2706     //  5: 0x0F 0x1F 0x44 0x00 0x00
2707     //  6: 0x66 0x0F 0x1F 0x44 0x00 0x00
2708     //  7: 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00
2709     //  8: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
2710     //  9: 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
2711     // 10: 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
2712     // 11: 0x66 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
2713 
2714     // The rest coding is Intel specific - don't use consecutive address nops
2715 
2716     // 12: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
2717     // 13: 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
2718     // 14: 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
2719     // 15: 0x66 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
2720 
2721     while(i >= 15) {
2722       // For Intel don't generate consecutive addess nops (mix with regular nops)
2723       i -= 15;
2724       emit_int8(0x66);   // size prefix
2725       emit_int8(0x66);   // size prefix
2726       emit_int8(0x66);   // size prefix
2727       addr_nop_8();
2728       emit_int8(0x66);   // size prefix
2729       emit_int8(0x66);   // size prefix
2730       emit_int8(0x66);   // size prefix
2731       emit_int8((unsigned char)0x90);
2732                          // nop
2733     }
2734     switch (i) {
2735       case 14:
2736         emit_int8(0x66); // size prefix
2737       case 13:
2738         emit_int8(0x66); // size prefix
2739       case 12:
2740         addr_nop_8();
2741         emit_int8(0x66); // size prefix
2742         emit_int8(0x66); // size prefix
2743         emit_int8(0x66); // size prefix
2744         emit_int8((unsigned char)0x90);
2745                          // nop
2746         break;
2747       case 11:
2748         emit_int8(0x66); // size prefix
2749       case 10:
2750         emit_int8(0x66); // size prefix
2751       case 9:
2752         emit_int8(0x66); // size prefix
2753       case 8:
2754         addr_nop_8();
2755         break;
2756       case 7:
2757         addr_nop_7();
2758         break;
2759       case 6:
2760         emit_int8(0x66); // size prefix
2761       case 5:
2762         addr_nop_5();
2763         break;
2764       case 4:
2765         addr_nop_4();
2766         break;
2767       case 3:
2768         // Don't use "0x0F 0x1F 0x00" - need patching safe padding
2769         emit_int8(0x66); // size prefix
2770       case 2:
2771         emit_int8(0x66); // size prefix
2772       case 1:
2773         emit_int8((unsigned char)0x90);
2774                          // nop
2775         break;
2776       default:
2777         assert(i == 0, " ");
2778     }
2779     return;
2780   }
2781   if (UseAddressNop && VM_Version::is_amd()) {
2782     //
2783     // Using multi-bytes nops "0x0F 0x1F [address]" for AMD.
2784     //  1: 0x90
2785     //  2: 0x66 0x90
2786     //  3: 0x66 0x66 0x90 (don't use "0x0F 0x1F 0x00" - need patching safe padding)
2787     //  4: 0x0F 0x1F 0x40 0x00
2788     //  5: 0x0F 0x1F 0x44 0x00 0x00
2789     //  6: 0x66 0x0F 0x1F 0x44 0x00 0x00
2790     //  7: 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00
2791     //  8: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
2792     //  9: 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
2793     // 10: 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
2794     // 11: 0x66 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
2795 
2796     // The rest coding is AMD specific - use consecutive address nops
2797 
2798     // 12: 0x66 0x0F 0x1F 0x44 0x00 0x00 0x66 0x0F 0x1F 0x44 0x00 0x00
2799     // 13: 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00 0x66 0x0F 0x1F 0x44 0x00 0x00
2800     // 14: 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00
2801     // 15: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00
2802     // 16: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
2803     //     Size prefixes (0x66) are added for larger sizes
2804 
2805     while(i >= 22) {
2806       i -= 11;
2807       emit_int8(0x66); // size prefix
2808       emit_int8(0x66); // size prefix
2809       emit_int8(0x66); // size prefix
2810       addr_nop_8();
2811     }
2812     // Generate first nop for size between 21-12
2813     switch (i) {
2814       case 21:
2815         i -= 1;
2816         emit_int8(0x66); // size prefix
2817       case 20:
2818       case 19:
2819         i -= 1;
2820         emit_int8(0x66); // size prefix
2821       case 18:
2822       case 17:
2823         i -= 1;
2824         emit_int8(0x66); // size prefix
2825       case 16:
2826       case 15:
2827         i -= 8;
2828         addr_nop_8();
2829         break;
2830       case 14:
2831       case 13:
2832         i -= 7;
2833         addr_nop_7();
2834         break;
2835       case 12:
2836         i -= 6;
2837         emit_int8(0x66); // size prefix
2838         addr_nop_5();
2839         break;
2840       default:
2841         assert(i < 12, " ");
2842     }
2843 
2844     // Generate second nop for size between 11-1
2845     switch (i) {
2846       case 11:
2847         emit_int8(0x66); // size prefix
2848       case 10:
2849         emit_int8(0x66); // size prefix
2850       case 9:
2851         emit_int8(0x66); // size prefix
2852       case 8:
2853         addr_nop_8();
2854         break;
2855       case 7:
2856         addr_nop_7();
2857         break;
2858       case 6:
2859         emit_int8(0x66); // size prefix
2860       case 5:
2861         addr_nop_5();
2862         break;
2863       case 4:
2864         addr_nop_4();
2865         break;
2866       case 3:
2867         // Don't use "0x0F 0x1F 0x00" - need patching safe padding
2868         emit_int8(0x66); // size prefix
2869       case 2:
2870         emit_int8(0x66); // size prefix
2871       case 1:
2872         emit_int8((unsigned char)0x90);
2873                          // nop
2874         break;
2875       default:
2876         assert(i == 0, " ");
2877     }
2878     return;
2879   }
2880 
2881   // Using nops with size prefixes "0x66 0x90".
2882   // From AMD Optimization Guide:
2883   //  1: 0x90
2884   //  2: 0x66 0x90
2885   //  3: 0x66 0x66 0x90
2886   //  4: 0x66 0x66 0x66 0x90
2887   //  5: 0x66 0x66 0x90 0x66 0x90
2888   //  6: 0x66 0x66 0x90 0x66 0x66 0x90
2889   //  7: 0x66 0x66 0x66 0x90 0x66 0x66 0x90
2890   //  8: 0x66 0x66 0x66 0x90 0x66 0x66 0x66 0x90
2891   //  9: 0x66 0x66 0x90 0x66 0x66 0x90 0x66 0x66 0x90
2892   // 10: 0x66 0x66 0x66 0x90 0x66 0x66 0x90 0x66 0x66 0x90
2893   //
2894   while(i > 12) {
2895     i -= 4;
2896     emit_int8(0x66); // size prefix
2897     emit_int8(0x66);
2898     emit_int8(0x66);
2899     emit_int8((unsigned char)0x90);
2900                      // nop
2901   }
2902   // 1 - 12 nops
2903   if(i > 8) {
2904     if(i > 9) {
2905       i -= 1;
2906       emit_int8(0x66);
2907     }
2908     i -= 3;
2909     emit_int8(0x66);
2910     emit_int8(0x66);
2911     emit_int8((unsigned char)0x90);
2912   }
2913   // 1 - 8 nops
2914   if(i > 4) {
2915     if(i > 6) {
2916       i -= 1;
2917       emit_int8(0x66);
2918     }
2919     i -= 3;
2920     emit_int8(0x66);
2921     emit_int8(0x66);
2922     emit_int8((unsigned char)0x90);
2923   }
2924   switch (i) {
2925     case 4:
2926       emit_int8(0x66);
2927     case 3:
2928       emit_int8(0x66);
2929     case 2:
2930       emit_int8(0x66);
2931     case 1:
2932       emit_int8((unsigned char)0x90);
2933       break;
2934     default:
2935       assert(i == 0, " ");
2936   }
2937 }
2938 
2939 void Assembler::notl(Register dst) {
2940   int encode = prefix_and_encode(dst->encoding());
2941   emit_int8((unsigned char)0xF7);
2942   emit_int8((unsigned char)(0xD0 | encode));
2943 }
2944 
2945 void Assembler::orl(Address dst, int32_t imm32) {
2946   InstructionMark im(this);
2947   prefix(dst);
2948   emit_arith_operand(0x81, rcx, dst, imm32);
2949 }
2950 
2951 void Assembler::orl(Register dst, int32_t imm32) {
2952   prefix(dst);
2953   emit_arith(0x81, 0xC8, dst, imm32);
2954 }
2955 
2956 void Assembler::orl(Register dst, Address src) {
2957   InstructionMark im(this);
2958   prefix(src, dst);
2959   emit_int8(0x0B);
2960   emit_operand(dst, src);
2961 }
2962 
2963 void Assembler::orl(Register dst, Register src) {
2964   (void) prefix_and_encode(dst->encoding(), src->encoding());
2965   emit_arith(0x0B, 0xC0, dst, src);
2966 }
2967 
2968 void Assembler::orl(Address dst, Register src) {
2969   InstructionMark im(this);
2970   prefix(dst, src);
2971   emit_int8(0x09);
2972   emit_operand(src, dst);
2973 }
2974 
2975 void Assembler::packuswb(XMMRegister dst, Address src) {
2976   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2977   assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
2978   InstructionMark im(this);
2979   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
2980   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
2981   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2982   emit_int8(0x67);
2983   emit_operand(dst, src);
2984 }
2985 
2986 void Assembler::packuswb(XMMRegister dst, XMMRegister src) {
2987   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2988   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
2989   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2990   emit_int8(0x67);
2991   emit_int8((unsigned char)(0xC0 | encode));
2992 }
2993 
2994 void Assembler::vpackuswb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
2995   assert(UseAVX > 0, "some form of AVX must be enabled");
2996   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
2997   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
2998   int encode = vex_prefix_and_encode(dst->encoding(), nds_enc, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2999   emit_int8(0x67);
3000   emit_int8((unsigned char)(0xC0 | encode));
3001 }
3002 
3003 void Assembler::vpermq(XMMRegister dst, XMMRegister src, int imm8, int vector_len) {
3004   assert(VM_Version::supports_avx2(), "");
3005   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
3006   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3007   emit_int8(0x00);
3008   emit_int8(0xC0 | encode);
3009   emit_int8(imm8);
3010 }
3011 
3012 void Assembler::pause() {
3013   emit_int8((unsigned char)0xF3);
3014   emit_int8((unsigned char)0x90);
3015 }
3016 
3017 void Assembler::pcmpestri(XMMRegister dst, Address src, int imm8) {
3018   assert(VM_Version::supports_sse4_2(), "");
3019   InstructionMark im(this);
3020   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3021   simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3022   emit_int8(0x61);
3023   emit_operand(dst, src);
3024   emit_int8(imm8);
3025 }
3026 
3027 void Assembler::pcmpestri(XMMRegister dst, XMMRegister src, int imm8) {
3028   assert(VM_Version::supports_sse4_2(), "");
3029   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3030   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3031   emit_int8(0x61);
3032   emit_int8((unsigned char)(0xC0 | encode));
3033   emit_int8(imm8);
3034 }
3035 
3036 void Assembler::pextrd(Register dst, XMMRegister src, int imm8) {
3037   assert(VM_Version::supports_sse4_1(), "");
3038   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
3039   int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3040   emit_int8(0x16);
3041   emit_int8((unsigned char)(0xC0 | encode));
3042   emit_int8(imm8);
3043 }
3044 
3045 void Assembler::pextrq(Register dst, XMMRegister src, int imm8) {
3046   assert(VM_Version::supports_sse4_1(), "");
3047   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
3048   int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3049   emit_int8(0x16);
3050   emit_int8((unsigned char)(0xC0 | encode));
3051   emit_int8(imm8);
3052 }
3053 
3054 void Assembler::pextrw(Register dst, XMMRegister src, int imm8) {
3055   assert(VM_Version::supports_sse2(), "");
3056   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
3057   int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3058   emit_int8((unsigned char)0xC5);
3059   emit_int8((unsigned char)(0xC0 | encode));
3060   emit_int8(imm8);
3061 }
3062 
3063 void Assembler::pinsrd(XMMRegister dst, Register src, int imm8) {
3064   assert(VM_Version::supports_sse4_1(), "");
3065   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
3066   int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3067   emit_int8(0x22);
3068   emit_int8((unsigned char)(0xC0 | encode));
3069   emit_int8(imm8);
3070 }
3071 
3072 void Assembler::pinsrq(XMMRegister dst, Register src, int imm8) {
3073   assert(VM_Version::supports_sse4_1(), "");
3074   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
3075   int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3076   emit_int8(0x22);
3077   emit_int8((unsigned char)(0xC0 | encode));
3078   emit_int8(imm8);
3079 }
3080 
3081 void Assembler::pinsrw(XMMRegister dst, Register src, int imm8) {
3082   assert(VM_Version::supports_sse2(), "");
3083   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
3084   int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3085   emit_int8((unsigned char)0xC4);
3086   emit_int8((unsigned char)(0xC0 | encode));
3087   emit_int8(imm8);
3088 }
3089 
3090 void Assembler::pmovzxbw(XMMRegister dst, Address src) {
3091   assert(VM_Version::supports_sse4_1(), "");
3092   InstructionMark im(this);
3093   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
3094   attributes.set_address_attributes(/* tuple_type */ EVEX_HVM, /* input_size_in_bits */ EVEX_NObit);
3095   simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3096   emit_int8(0x30);
3097   emit_operand(dst, src);
3098 }
3099 
3100 void Assembler::pmovzxbw(XMMRegister dst, XMMRegister src) {
3101   assert(VM_Version::supports_sse4_1(), "");
3102   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
3103   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3104   emit_int8(0x30);
3105   emit_int8((unsigned char)(0xC0 | encode));
3106 }
3107 
3108 // generic
3109 void Assembler::pop(Register dst) {
3110   int encode = prefix_and_encode(dst->encoding());
3111   emit_int8(0x58 | encode);
3112 }
3113 
3114 void Assembler::popcntl(Register dst, Address src) {
3115   assert(VM_Version::supports_popcnt(), "must support");
3116   InstructionMark im(this);
3117   emit_int8((unsigned char)0xF3);
3118   prefix(src, dst);
3119   emit_int8(0x0F);
3120   emit_int8((unsigned char)0xB8);
3121   emit_operand(dst, src);
3122 }
3123 
3124 void Assembler::popcntl(Register dst, Register src) {
3125   assert(VM_Version::supports_popcnt(), "must support");
3126   emit_int8((unsigned char)0xF3);
3127   int encode = prefix_and_encode(dst->encoding(), src->encoding());
3128   emit_int8(0x0F);
3129   emit_int8((unsigned char)0xB8);
3130   emit_int8((unsigned char)(0xC0 | encode));
3131 }
3132 
3133 void Assembler::popf() {
3134   emit_int8((unsigned char)0x9D);
3135 }
3136 
3137 #ifndef _LP64 // no 32bit push/pop on amd64
3138 void Assembler::popl(Address dst) {
3139   // NOTE: this will adjust stack by 8byte on 64bits
3140   InstructionMark im(this);
3141   prefix(dst);
3142   emit_int8((unsigned char)0x8F);
3143   emit_operand(rax, dst);
3144 }
3145 #endif
3146 
3147 void Assembler::prefetch_prefix(Address src) {
3148   prefix(src);
3149   emit_int8(0x0F);
3150 }
3151 
3152 void Assembler::prefetchnta(Address src) {
3153   NOT_LP64(assert(VM_Version::supports_sse(), "must support"));
3154   InstructionMark im(this);
3155   prefetch_prefix(src);
3156   emit_int8(0x18);
3157   emit_operand(rax, src); // 0, src
3158 }
3159 
3160 void Assembler::prefetchr(Address src) {
3161   assert(VM_Version::supports_3dnow_prefetch(), "must support");
3162   InstructionMark im(this);
3163   prefetch_prefix(src);
3164   emit_int8(0x0D);
3165   emit_operand(rax, src); // 0, src
3166 }
3167 
3168 void Assembler::prefetcht0(Address src) {
3169   NOT_LP64(assert(VM_Version::supports_sse(), "must support"));
3170   InstructionMark im(this);
3171   prefetch_prefix(src);
3172   emit_int8(0x18);
3173   emit_operand(rcx, src); // 1, src
3174 }
3175 
3176 void Assembler::prefetcht1(Address src) {
3177   NOT_LP64(assert(VM_Version::supports_sse(), "must support"));
3178   InstructionMark im(this);
3179   prefetch_prefix(src);
3180   emit_int8(0x18);
3181   emit_operand(rdx, src); // 2, src
3182 }
3183 
3184 void Assembler::prefetcht2(Address src) {
3185   NOT_LP64(assert(VM_Version::supports_sse(), "must support"));
3186   InstructionMark im(this);
3187   prefetch_prefix(src);
3188   emit_int8(0x18);
3189   emit_operand(rbx, src); // 3, src
3190 }
3191 
3192 void Assembler::prefetchw(Address src) {
3193   assert(VM_Version::supports_3dnow_prefetch(), "must support");
3194   InstructionMark im(this);
3195   prefetch_prefix(src);
3196   emit_int8(0x0D);
3197   emit_operand(rcx, src); // 1, src
3198 }
3199 
3200 void Assembler::prefix(Prefix p) {
3201   emit_int8(p);
3202 }
3203 
3204 void Assembler::pshufb(XMMRegister dst, XMMRegister src) {
3205   assert(VM_Version::supports_ssse3(), "");
3206   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ false);
3207   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3208   emit_int8(0x00);
3209   emit_int8((unsigned char)(0xC0 | encode));
3210 }
3211 
3212 void Assembler::pshufb(XMMRegister dst, Address src) {
3213   assert(VM_Version::supports_ssse3(), "");
3214   InstructionMark im(this);
3215   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ false);
3216   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
3217   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3218   emit_int8(0x00);
3219   emit_operand(dst, src);
3220 }
3221 
3222 void Assembler::pshufd(XMMRegister dst, XMMRegister src, int mode) {
3223   assert(isByte(mode), "invalid value");
3224   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3225   int vector_len = VM_Version::supports_evex() ? AVX_512bit : AVX_128bit;
3226   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
3227   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3228   emit_int8(0x70);
3229   emit_int8((unsigned char)(0xC0 | encode));
3230   emit_int8(mode & 0xFF);
3231 }
3232 
3233 void Assembler::pshufd(XMMRegister dst, Address src, int mode) {
3234   assert(isByte(mode), "invalid value");
3235   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3236   assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
3237   InstructionMark im(this);
3238   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
3239   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
3240   simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3241   emit_int8(0x70);
3242   emit_operand(dst, src);
3243   emit_int8(mode & 0xFF);
3244 }
3245 
3246 void Assembler::pshuflw(XMMRegister dst, XMMRegister src, int mode) {
3247   assert(isByte(mode), "invalid value");
3248   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3249   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ false);
3250   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
3251   emit_int8(0x70);
3252   emit_int8((unsigned char)(0xC0 | encode));
3253   emit_int8(mode & 0xFF);
3254 }
3255 
3256 void Assembler::pshuflw(XMMRegister dst, Address src, int mode) {
3257   assert(isByte(mode), "invalid value");
3258   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3259   assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
3260   InstructionMark im(this);
3261   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ false);
3262   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
3263   simd_prefix(dst, xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
3264   emit_int8(0x70);
3265   emit_operand(dst, src);
3266   emit_int8(mode & 0xFF);
3267 }
3268 
3269 void Assembler::psrldq(XMMRegister dst, int shift) {
3270   // Shift left 128 bit value in dst XMMRegister by shift number of bytes.
3271   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3272   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
3273   // XMM3 is for /3 encoding: 66 0F 73 /3 ib
3274   int encode = simd_prefix_and_encode(xmm3, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3275   emit_int8(0x73);
3276   emit_int8((unsigned char)(0xC0 | encode));
3277   emit_int8(shift);
3278 }
3279 
3280 void Assembler::pslldq(XMMRegister dst, int shift) {
3281   // Shift left 128 bit value in dst XMMRegister by shift number of bytes.
3282   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3283   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
3284   // XMM7 is for /7 encoding: 66 0F 73 /7 ib
3285   int encode = simd_prefix_and_encode(xmm7, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3286   emit_int8(0x73);
3287   emit_int8((unsigned char)(0xC0 | encode));
3288   emit_int8(shift);
3289 }
3290 
3291 void Assembler::ptest(XMMRegister dst, Address src) {
3292   assert(VM_Version::supports_sse4_1(), "");
3293   assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
3294   InstructionMark im(this);
3295   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3296   simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3297   emit_int8(0x17);
3298   emit_operand(dst, src);
3299 }
3300 
3301 void Assembler::ptest(XMMRegister dst, XMMRegister src) {
3302   assert(VM_Version::supports_sse4_1(), "");
3303   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3304   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3305   emit_int8(0x17);
3306   emit_int8((unsigned char)(0xC0 | encode));
3307 }
3308 
3309 void Assembler::vptest(XMMRegister dst, Address src) {
3310   assert(VM_Version::supports_avx(), "");
3311   InstructionMark im(this);
3312   InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3313   assert(dst != xnoreg, "sanity");
3314   // swap src<->dst for encoding
3315   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3316   emit_int8(0x17);
3317   emit_operand(dst, src);
3318 }
3319 
3320 void Assembler::vptest(XMMRegister dst, XMMRegister src) {
3321   assert(VM_Version::supports_avx(), "");
3322   InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3323   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3324   emit_int8(0x17);
3325   emit_int8((unsigned char)(0xC0 | encode));
3326 }
3327 
3328 void Assembler::punpcklbw(XMMRegister dst, Address src) {
3329   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3330   assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
3331   InstructionMark im(this);
3332   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_vlbw, /* no_mask_reg */ false, /* uses_vl */ true);
3333   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
3334   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3335   emit_int8(0x60);
3336   emit_operand(dst, src);
3337 }
3338 
3339 void Assembler::punpcklbw(XMMRegister dst, XMMRegister src) {
3340   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3341   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_vlbw, /* no_mask_reg */ false, /* uses_vl */ true);
3342   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3343   emit_int8(0x60);
3344   emit_int8((unsigned char)(0xC0 | encode));
3345 }
3346 
3347 void Assembler::punpckldq(XMMRegister dst, Address src) {
3348   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3349   assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
3350   InstructionMark im(this);
3351   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
3352   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
3353   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3354   emit_int8(0x62);
3355   emit_operand(dst, src);
3356 }
3357 
3358 void Assembler::punpckldq(XMMRegister dst, XMMRegister src) {
3359   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3360   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
3361   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3362   emit_int8(0x62);
3363   emit_int8((unsigned char)(0xC0 | encode));
3364 }
3365 
3366 void Assembler::punpcklqdq(XMMRegister dst, XMMRegister src) {
3367   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3368   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
3369   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3370   emit_int8(0x6C);
3371   emit_int8((unsigned char)(0xC0 | encode));
3372 }
3373 
3374 void Assembler::push(int32_t imm32) {
3375   // in 64bits we push 64bits onto the stack but only
3376   // take a 32bit immediate
3377   emit_int8(0x68);
3378   emit_int32(imm32);
3379 }
3380 
3381 void Assembler::push(Register src) {
3382   int encode = prefix_and_encode(src->encoding());
3383 
3384   emit_int8(0x50 | encode);
3385 }
3386 
3387 void Assembler::pushf() {
3388   emit_int8((unsigned char)0x9C);
3389 }
3390 
3391 #ifndef _LP64 // no 32bit push/pop on amd64
3392 void Assembler::pushl(Address src) {
3393   // Note this will push 64bit on 64bit
3394   InstructionMark im(this);
3395   prefix(src);
3396   emit_int8((unsigned char)0xFF);
3397   emit_operand(rsi, src);
3398 }
3399 #endif
3400 
3401 void Assembler::rcll(Register dst, int imm8) {
3402   assert(isShiftCount(imm8), "illegal shift count");
3403   int encode = prefix_and_encode(dst->encoding());
3404   if (imm8 == 1) {
3405     emit_int8((unsigned char)0xD1);
3406     emit_int8((unsigned char)(0xD0 | encode));
3407   } else {
3408     emit_int8((unsigned char)0xC1);
3409     emit_int8((unsigned char)0xD0 | encode);
3410     emit_int8(imm8);
3411   }
3412 }
3413 
3414 void Assembler::rcpps(XMMRegister dst, XMMRegister src) {
3415   NOT_LP64(assert(VM_Version::supports_sse(), ""));
3416   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3417   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
3418   emit_int8(0x53);
3419   emit_int8((unsigned char)(0xC0 | encode));
3420 }
3421 
3422 void Assembler::rcpss(XMMRegister dst, XMMRegister src) {
3423   NOT_LP64(assert(VM_Version::supports_sse(), ""));
3424   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3425   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
3426   emit_int8(0x53);
3427   emit_int8((unsigned char)(0xC0 | encode));
3428 }
3429 
3430 void Assembler::rdtsc() {
3431   emit_int8((unsigned char)0x0F);
3432   emit_int8((unsigned char)0x31);
3433 }
3434 
3435 // copies data from [esi] to [edi] using rcx pointer sized words
3436 // generic
3437 void Assembler::rep_mov() {
3438   emit_int8((unsigned char)0xF3);
3439   // MOVSQ
3440   LP64_ONLY(prefix(REX_W));
3441   emit_int8((unsigned char)0xA5);
3442 }
3443 
3444 // sets rcx bytes with rax, value at [edi]
3445 void Assembler::rep_stosb() {
3446   emit_int8((unsigned char)0xF3); // REP
3447   LP64_ONLY(prefix(REX_W));
3448   emit_int8((unsigned char)0xAA); // STOSB
3449 }
3450 
3451 // sets rcx pointer sized words with rax, value at [edi]
3452 // generic
3453 void Assembler::rep_stos() {
3454   emit_int8((unsigned char)0xF3); // REP
3455   LP64_ONLY(prefix(REX_W));       // LP64:STOSQ, LP32:STOSD
3456   emit_int8((unsigned char)0xAB);
3457 }
3458 
3459 // scans rcx pointer sized words at [edi] for occurance of rax,
3460 // generic
3461 void Assembler::repne_scan() { // repne_scan
3462   emit_int8((unsigned char)0xF2);
3463   // SCASQ
3464   LP64_ONLY(prefix(REX_W));
3465   emit_int8((unsigned char)0xAF);
3466 }
3467 
3468 #ifdef _LP64
3469 // scans rcx 4 byte words at [edi] for occurance of rax,
3470 // generic
3471 void Assembler::repne_scanl() { // repne_scan
3472   emit_int8((unsigned char)0xF2);
3473   // SCASL
3474   emit_int8((unsigned char)0xAF);
3475 }
3476 #endif
3477 
3478 void Assembler::ret(int imm16) {
3479   if (imm16 == 0) {
3480     emit_int8((unsigned char)0xC3);
3481   } else {
3482     emit_int8((unsigned char)0xC2);
3483     emit_int16(imm16);
3484   }
3485 }
3486 
3487 void Assembler::sahf() {
3488 #ifdef _LP64
3489   // Not supported in 64bit mode
3490   ShouldNotReachHere();
3491 #endif
3492   emit_int8((unsigned char)0x9E);
3493 }
3494 
3495 void Assembler::sarl(Register dst, int imm8) {
3496   int encode = prefix_and_encode(dst->encoding());
3497   assert(isShiftCount(imm8), "illegal shift count");
3498   if (imm8 == 1) {
3499     emit_int8((unsigned char)0xD1);
3500     emit_int8((unsigned char)(0xF8 | encode));
3501   } else {
3502     emit_int8((unsigned char)0xC1);
3503     emit_int8((unsigned char)(0xF8 | encode));
3504     emit_int8(imm8);
3505   }
3506 }
3507 
3508 void Assembler::sarl(Register dst) {
3509   int encode = prefix_and_encode(dst->encoding());
3510   emit_int8((unsigned char)0xD3);
3511   emit_int8((unsigned char)(0xF8 | encode));
3512 }
3513 
3514 void Assembler::sbbl(Address dst, int32_t imm32) {
3515   InstructionMark im(this);
3516   prefix(dst);
3517   emit_arith_operand(0x81, rbx, dst, imm32);
3518 }
3519 
3520 void Assembler::sbbl(Register dst, int32_t imm32) {
3521   prefix(dst);
3522   emit_arith(0x81, 0xD8, dst, imm32);
3523 }
3524 
3525 
3526 void Assembler::sbbl(Register dst, Address src) {
3527   InstructionMark im(this);
3528   prefix(src, dst);
3529   emit_int8(0x1B);
3530   emit_operand(dst, src);
3531 }
3532 
3533 void Assembler::sbbl(Register dst, Register src) {
3534   (void) prefix_and_encode(dst->encoding(), src->encoding());
3535   emit_arith(0x1B, 0xC0, dst, src);
3536 }
3537 
3538 void Assembler::setb(Condition cc, Register dst) {
3539   assert(0 <= cc && cc < 16, "illegal cc");
3540   int encode = prefix_and_encode(dst->encoding(), true);
3541   emit_int8(0x0F);
3542   emit_int8((unsigned char)0x90 | cc);
3543   emit_int8((unsigned char)(0xC0 | encode));
3544 }
3545 
3546 void Assembler::shll(Register dst, int imm8) {
3547   assert(isShiftCount(imm8), "illegal shift count");
3548   int encode = prefix_and_encode(dst->encoding());
3549   if (imm8 == 1 ) {
3550     emit_int8((unsigned char)0xD1);
3551     emit_int8((unsigned char)(0xE0 | encode));
3552   } else {
3553     emit_int8((unsigned char)0xC1);
3554     emit_int8((unsigned char)(0xE0 | encode));
3555     emit_int8(imm8);
3556   }
3557 }
3558 
3559 void Assembler::shll(Register dst) {
3560   int encode = prefix_and_encode(dst->encoding());
3561   emit_int8((unsigned char)0xD3);
3562   emit_int8((unsigned char)(0xE0 | encode));
3563 }
3564 
3565 void Assembler::shrl(Register dst, int imm8) {
3566   assert(isShiftCount(imm8), "illegal shift count");
3567   int encode = prefix_and_encode(dst->encoding());
3568   emit_int8((unsigned char)0xC1);
3569   emit_int8((unsigned char)(0xE8 | encode));
3570   emit_int8(imm8);
3571 }
3572 
3573 void Assembler::shrl(Register dst) {
3574   int encode = prefix_and_encode(dst->encoding());
3575   emit_int8((unsigned char)0xD3);
3576   emit_int8((unsigned char)(0xE8 | encode));
3577 }
3578 
3579 // copies a single word from [esi] to [edi]
3580 void Assembler::smovl() {
3581   emit_int8((unsigned char)0xA5);
3582 }
3583 
3584 void Assembler::sqrtsd(XMMRegister dst, XMMRegister src) {
3585   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3586   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3587   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
3588   emit_int8(0x51);
3589   emit_int8((unsigned char)(0xC0 | encode));
3590 }
3591 
3592 void Assembler::sqrtsd(XMMRegister dst, Address src) {
3593   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3594   InstructionMark im(this);
3595   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3596   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
3597   simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
3598   emit_int8(0x51);
3599   emit_operand(dst, src);
3600 }
3601 
3602 void Assembler::sqrtss(XMMRegister dst, XMMRegister src) {
3603   NOT_LP64(assert(VM_Version::supports_sse(), ""));
3604   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3605   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
3606   emit_int8(0x51);
3607   emit_int8((unsigned char)(0xC0 | encode));
3608 }
3609 
3610 void Assembler::std() {
3611   emit_int8((unsigned char)0xFD);
3612 }
3613 
3614 void Assembler::sqrtss(XMMRegister dst, Address src) {
3615   NOT_LP64(assert(VM_Version::supports_sse(), ""));
3616   InstructionMark im(this);
3617   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3618   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
3619   simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
3620   emit_int8(0x51);
3621   emit_operand(dst, src);
3622 }
3623 
3624 void Assembler::stmxcsr( Address dst) {
3625   NOT_LP64(assert(VM_Version::supports_sse(), ""));
3626   InstructionMark im(this);
3627   prefix(dst);
3628   emit_int8(0x0F);
3629   emit_int8((unsigned char)0xAE);
3630   emit_operand(as_Register(3), dst);
3631 }
3632 
3633 void Assembler::subl(Address dst, int32_t imm32) {
3634   InstructionMark im(this);
3635   prefix(dst);
3636   emit_arith_operand(0x81, rbp, dst, imm32);
3637 }
3638 
3639 void Assembler::subl(Address dst, Register src) {
3640   InstructionMark im(this);
3641   prefix(dst, src);
3642   emit_int8(0x29);
3643   emit_operand(src, dst);
3644 }
3645 
3646 void Assembler::subl(Register dst, int32_t imm32) {
3647   prefix(dst);
3648   emit_arith(0x81, 0xE8, dst, imm32);
3649 }
3650 
3651 // Force generation of a 4 byte immediate value even if it fits into 8bit
3652 void Assembler::subl_imm32(Register dst, int32_t imm32) {
3653   prefix(dst);
3654   emit_arith_imm32(0x81, 0xE8, dst, imm32);
3655 }
3656 
3657 void Assembler::subl(Register dst, Address src) {
3658   InstructionMark im(this);
3659   prefix(src, dst);
3660   emit_int8(0x2B);
3661   emit_operand(dst, src);
3662 }
3663 
3664 void Assembler::subl(Register dst, Register src) {
3665   (void) prefix_and_encode(dst->encoding(), src->encoding());
3666   emit_arith(0x2B, 0xC0, dst, src);
3667 }
3668 
3669 void Assembler::subsd(XMMRegister dst, XMMRegister src) {
3670   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3671   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3672   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
3673   emit_int8(0x5C);
3674   emit_int8((unsigned char)(0xC0 | encode));
3675 }
3676 
3677 void Assembler::subsd(XMMRegister dst, Address src) {
3678   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3679   InstructionMark im(this);
3680   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3681   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
3682   simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
3683   emit_int8(0x5C);
3684   emit_operand(dst, src);
3685 }
3686 
3687 void Assembler::subss(XMMRegister dst, XMMRegister src) {
3688   NOT_LP64(assert(VM_Version::supports_sse(), ""));
3689   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3690   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
3691   emit_int8(0x5C);
3692   emit_int8((unsigned char)(0xC0 | encode));
3693 }
3694 
3695 void Assembler::subss(XMMRegister dst, Address src) {
3696   NOT_LP64(assert(VM_Version::supports_sse(), ""));
3697   InstructionMark im(this);
3698   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3699   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
3700   simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
3701   emit_int8(0x5C);
3702   emit_operand(dst, src);
3703 }
3704 
3705 void Assembler::testb(Register dst, int imm8) {
3706   NOT_LP64(assert(dst->has_byte_register(), "must have byte register"));
3707   (void) prefix_and_encode(dst->encoding(), true);
3708   emit_arith_b(0xF6, 0xC0, dst, imm8);
3709 }
3710 
3711 void Assembler::testl(Register dst, int32_t imm32) {
3712   // not using emit_arith because test
3713   // doesn't support sign-extension of
3714   // 8bit operands
3715   int encode = dst->encoding();
3716   if (encode == 0) {
3717     emit_int8((unsigned char)0xA9);
3718   } else {
3719     encode = prefix_and_encode(encode);
3720     emit_int8((unsigned char)0xF7);
3721     emit_int8((unsigned char)(0xC0 | encode));
3722   }
3723   emit_int32(imm32);
3724 }
3725 
3726 void Assembler::testl(Register dst, Register src) {
3727   (void) prefix_and_encode(dst->encoding(), src->encoding());
3728   emit_arith(0x85, 0xC0, dst, src);
3729 }
3730 
3731 void Assembler::testl(Register dst, Address src) {
3732   InstructionMark im(this);
3733   prefix(src, dst);
3734   emit_int8((unsigned char)0x85);
3735   emit_operand(dst, src);
3736 }
3737 
3738 void Assembler::tzcntl(Register dst, Register src) {
3739   assert(VM_Version::supports_bmi1(), "tzcnt instruction not supported");
3740   emit_int8((unsigned char)0xF3);
3741   int encode = prefix_and_encode(dst->encoding(), src->encoding());
3742   emit_int8(0x0F);
3743   emit_int8((unsigned char)0xBC);
3744   emit_int8((unsigned char)0xC0 | encode);
3745 }
3746 
3747 void Assembler::tzcntq(Register dst, Register src) {
3748   assert(VM_Version::supports_bmi1(), "tzcnt instruction not supported");
3749   emit_int8((unsigned char)0xF3);
3750   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
3751   emit_int8(0x0F);
3752   emit_int8((unsigned char)0xBC);
3753   emit_int8((unsigned char)(0xC0 | encode));
3754 }
3755 
3756 void Assembler::ucomisd(XMMRegister dst, Address src) {
3757   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3758   InstructionMark im(this);
3759   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3760   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
3761   simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3762   emit_int8(0x2E);
3763   emit_operand(dst, src);
3764 }
3765 
3766 void Assembler::ucomisd(XMMRegister dst, XMMRegister src) {
3767   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3768   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3769   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3770   emit_int8(0x2E);
3771   emit_int8((unsigned char)(0xC0 | encode));
3772 }
3773 
3774 void Assembler::ucomiss(XMMRegister dst, Address src) {
3775   NOT_LP64(assert(VM_Version::supports_sse(), ""));
3776   InstructionMark im(this);
3777   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3778   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
3779   simd_prefix(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
3780   emit_int8(0x2E);
3781   emit_operand(dst, src);
3782 }
3783 
3784 void Assembler::ucomiss(XMMRegister dst, XMMRegister src) {
3785   NOT_LP64(assert(VM_Version::supports_sse(), ""));
3786   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3787   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
3788   emit_int8(0x2E);
3789   emit_int8((unsigned char)(0xC0 | encode));
3790 }
3791 
3792 void Assembler::xabort(int8_t imm8) {
3793   emit_int8((unsigned char)0xC6);
3794   emit_int8((unsigned char)0xF8);
3795   emit_int8((unsigned char)(imm8 & 0xFF));
3796 }
3797 
3798 void Assembler::xaddl(Address dst, Register src) {
3799   InstructionMark im(this);
3800   prefix(dst, src);
3801   emit_int8(0x0F);
3802   emit_int8((unsigned char)0xC1);
3803   emit_operand(src, dst);
3804 }
3805 
3806 void Assembler::xbegin(Label& abort, relocInfo::relocType rtype) {
3807   InstructionMark im(this);
3808   relocate(rtype);
3809   if (abort.is_bound()) {
3810     address entry = target(abort);
3811     assert(entry != NULL, "abort entry NULL");
3812     intptr_t offset = entry - pc();
3813     emit_int8((unsigned char)0xC7);
3814     emit_int8((unsigned char)0xF8);
3815     emit_int32(offset - 6); // 2 opcode + 4 address
3816   } else {
3817     abort.add_patch_at(code(), locator());
3818     emit_int8((unsigned char)0xC7);
3819     emit_int8((unsigned char)0xF8);
3820     emit_int32(0);
3821   }
3822 }
3823 
3824 void Assembler::xchgl(Register dst, Address src) { // xchg
3825   InstructionMark im(this);
3826   prefix(src, dst);
3827   emit_int8((unsigned char)0x87);
3828   emit_operand(dst, src);
3829 }
3830 
3831 void Assembler::xchgl(Register dst, Register src) {
3832   int encode = prefix_and_encode(dst->encoding(), src->encoding());
3833   emit_int8((unsigned char)0x87);
3834   emit_int8((unsigned char)(0xC0 | encode));
3835 }
3836 
3837 void Assembler::xend() {
3838   emit_int8((unsigned char)0x0F);
3839   emit_int8((unsigned char)0x01);
3840   emit_int8((unsigned char)0xD5);
3841 }
3842 
3843 void Assembler::xgetbv() {
3844   emit_int8(0x0F);
3845   emit_int8(0x01);
3846   emit_int8((unsigned char)0xD0);
3847 }
3848 
3849 void Assembler::xorl(Register dst, int32_t imm32) {
3850   prefix(dst);
3851   emit_arith(0x81, 0xF0, dst, imm32);
3852 }
3853 
3854 void Assembler::xorl(Register dst, Address src) {
3855   InstructionMark im(this);
3856   prefix(src, dst);
3857   emit_int8(0x33);
3858   emit_operand(dst, src);
3859 }
3860 
3861 void Assembler::xorl(Register dst, Register src) {
3862   (void) prefix_and_encode(dst->encoding(), src->encoding());
3863   emit_arith(0x33, 0xC0, dst, src);
3864 }
3865 
3866 
3867 // AVX 3-operands scalar float-point arithmetic instructions
3868 
3869 void Assembler::vaddsd(XMMRegister dst, XMMRegister nds, Address src) {
3870   assert(VM_Version::supports_avx(), "");
3871   InstructionMark im(this);
3872   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3873   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
3874   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
3875   vex_prefix(src, nds_enc, dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
3876   emit_int8(0x58);
3877   emit_operand(dst, src);
3878 }
3879 
3880 void Assembler::vaddsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
3881   assert(VM_Version::supports_avx(), "");
3882   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3883   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
3884   int encode = vex_prefix_and_encode(dst->encoding(), nds_enc, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
3885   emit_int8(0x58);
3886   emit_int8((unsigned char)(0xC0 | encode));
3887 }
3888 
3889 void Assembler::vaddss(XMMRegister dst, XMMRegister nds, Address src) {
3890   assert(VM_Version::supports_avx(), "");
3891   InstructionMark im(this);
3892   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3893   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
3894   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
3895   vex_prefix(src, nds_enc, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
3896   emit_int8(0x58);
3897   emit_operand(dst, src);
3898 }
3899 
3900 void Assembler::vaddss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
3901   assert(VM_Version::supports_avx(), "");
3902   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3903   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
3904   int encode = vex_prefix_and_encode(dst->encoding(), nds_enc, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
3905   emit_int8(0x58);
3906   emit_int8((unsigned char)(0xC0 | encode));
3907 }
3908 
3909 void Assembler::vdivsd(XMMRegister dst, XMMRegister nds, Address src) {
3910   assert(VM_Version::supports_avx(), "");
3911   InstructionMark im(this);
3912   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3913   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
3914   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
3915   vex_prefix(src, nds_enc, dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
3916   emit_int8(0x5E);
3917   emit_operand(dst, src);
3918 }
3919 
3920 void Assembler::vdivsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
3921   assert(VM_Version::supports_avx(), "");
3922   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3923   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
3924   int encode = vex_prefix_and_encode(dst->encoding(), nds_enc, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
3925   emit_int8(0x5E);
3926   emit_int8((unsigned char)(0xC0 | encode));
3927 }
3928 
3929 void Assembler::vdivss(XMMRegister dst, XMMRegister nds, Address src) {
3930   assert(VM_Version::supports_avx(), "");
3931   InstructionMark im(this);
3932   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3933   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
3934   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
3935   vex_prefix(src, nds_enc, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
3936   emit_int8(0x5E);
3937   emit_operand(dst, src);
3938 }
3939 
3940 void Assembler::vdivss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
3941   assert(VM_Version::supports_avx(), "");
3942   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3943   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
3944   int encode = vex_prefix_and_encode(dst->encoding(), nds_enc, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
3945   emit_int8(0x5E);
3946   emit_int8((unsigned char)(0xC0 | encode));
3947 }
3948 
3949 void Assembler::vmulsd(XMMRegister dst, XMMRegister nds, Address src) {
3950   assert(VM_Version::supports_avx(), "");
3951   InstructionMark im(this);
3952   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3953   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
3954   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
3955   vex_prefix(src, nds_enc, dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
3956   emit_int8(0x59);
3957   emit_operand(dst, src);
3958 }
3959 
3960 void Assembler::vmulsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
3961   assert(VM_Version::supports_avx(), "");
3962   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3963   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
3964   int encode = vex_prefix_and_encode(dst->encoding(), nds_enc, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
3965   emit_int8(0x59);
3966   emit_int8((unsigned char)(0xC0 | encode));
3967 }
3968 
3969 void Assembler::vmulss(XMMRegister dst, XMMRegister nds, Address src) {
3970   assert(VM_Version::supports_avx(), "");
3971   InstructionMark im(this);
3972   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3973   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
3974   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
3975   vex_prefix(src, nds_enc, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
3976   emit_int8(0x59);
3977   emit_operand(dst, src);
3978 }
3979 
3980 void Assembler::vmulss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
3981   assert(VM_Version::supports_avx(), "");
3982   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3983   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
3984   int encode = vex_prefix_and_encode(dst->encoding(), nds_enc, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
3985   emit_int8(0x59);
3986   emit_int8((unsigned char)(0xC0 | encode));
3987 }
3988 
3989 void Assembler::vsubsd(XMMRegister dst, XMMRegister nds, Address src) {
3990   assert(VM_Version::supports_avx(), "");
3991   InstructionMark im(this);
3992   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3993   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
3994   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
3995   vex_prefix(src, nds_enc, dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
3996   emit_int8(0x5C);
3997   emit_operand(dst, src);
3998 }
3999 
4000 void Assembler::vsubsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
4001   assert(VM_Version::supports_avx(), "");
4002   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4003   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
4004   int encode = vex_prefix_and_encode(dst->encoding(), nds_enc, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
4005   emit_int8(0x5C);
4006   emit_int8((unsigned char)(0xC0 | encode));
4007 }
4008 
4009 void Assembler::vsubss(XMMRegister dst, XMMRegister nds, Address src) {
4010   assert(VM_Version::supports_avx(), "");
4011   InstructionMark im(this);
4012   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4013   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
4014   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
4015   vex_prefix(src, nds_enc, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
4016   emit_int8(0x5C);
4017   emit_operand(dst, src);
4018 }
4019 
4020 void Assembler::vsubss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
4021   assert(VM_Version::supports_avx(), "");
4022   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4023   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
4024   int encode = vex_prefix_and_encode(dst->encoding(), nds_enc, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
4025   emit_int8(0x5C);
4026   emit_int8((unsigned char)(0xC0 | encode));
4027 }
4028 
4029 //====================VECTOR ARITHMETIC=====================================
4030 
4031 // Float-point vector arithmetic
4032 
4033 void Assembler::addpd(XMMRegister dst, XMMRegister src) {
4034   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4035   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4036   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4037   emit_int8(0x58);
4038   emit_int8((unsigned char)(0xC0 | encode));
4039 }
4040 
4041 void Assembler::addps(XMMRegister dst, XMMRegister src) {
4042   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4043   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4044   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
4045   emit_int8(0x58);
4046   emit_int8((unsigned char)(0xC0 | encode));
4047 }
4048 
4049 void Assembler::vaddpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4050   assert(VM_Version::supports_avx(), "");
4051   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4052   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
4053   int encode = vex_prefix_and_encode(dst->encoding(), nds_enc, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4054   emit_int8(0x58);
4055   emit_int8((unsigned char)(0xC0 | encode));
4056 }
4057 
4058 void Assembler::vaddps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4059   assert(VM_Version::supports_avx(), "");
4060   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4061   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
4062   int encode = vex_prefix_and_encode(dst->encoding(), nds_enc, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
4063   emit_int8(0x58);
4064   emit_int8((unsigned char)(0xC0 | encode));
4065 }
4066 
4067 void Assembler::vaddpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4068   assert(VM_Version::supports_avx(), "");
4069   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4070   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
4071   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
4072   vex_prefix(src, nds_enc, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4073   emit_int8(0x58);
4074   emit_operand(dst, src);
4075 }
4076 
4077 void Assembler::vaddps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4078   assert(VM_Version::supports_avx(), "");
4079   InstructionMark im(this);
4080   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4081   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
4082   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
4083   vex_prefix(src, nds_enc, dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
4084   emit_int8(0x58);
4085   emit_operand(dst, src);
4086 }
4087 
4088 void Assembler::subpd(XMMRegister dst, XMMRegister src) {
4089   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4090   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4091   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4092   emit_int8(0x5C);
4093   emit_int8((unsigned char)(0xC0 | encode));
4094 }
4095 
4096 void Assembler::subps(XMMRegister dst, XMMRegister src) {
4097   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4098   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4099   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
4100   emit_int8(0x5C);
4101   emit_int8((unsigned char)(0xC0 | encode));
4102 }
4103 
4104 void Assembler::vsubpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4105   assert(VM_Version::supports_avx(), "");
4106   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4107   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
4108   int encode = vex_prefix_and_encode(dst->encoding(), nds_enc, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4109   emit_int8(0x5C);
4110   emit_int8((unsigned char)(0xC0 | encode));
4111 }
4112 
4113 void Assembler::vsubps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4114   assert(VM_Version::supports_avx(), "");
4115   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4116   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
4117   int encode = vex_prefix_and_encode(dst->encoding(), nds_enc, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
4118   emit_int8(0x5C);
4119   emit_int8((unsigned char)(0xC0 | encode));
4120 }
4121 
4122 void Assembler::vsubpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4123   assert(VM_Version::supports_avx(), "");
4124   InstructionMark im(this);
4125   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4126   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
4127   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
4128   vex_prefix(src, nds_enc, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4129   emit_int8(0x5C);
4130   emit_operand(dst, src);
4131 }
4132 
4133 void Assembler::vsubps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4134   assert(VM_Version::supports_avx(), "");
4135   InstructionMark im(this);
4136   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4137   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
4138   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
4139   vex_prefix(src, nds_enc, dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
4140   emit_int8(0x5C);
4141   emit_operand(dst, src);
4142 }
4143 
4144 void Assembler::mulpd(XMMRegister dst, XMMRegister src) {
4145   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4146   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4147   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4148   emit_int8(0x59);
4149   emit_int8((unsigned char)(0xC0 | encode));
4150 }
4151 
4152 void Assembler::mulpd(XMMRegister dst, Address src) {
4153   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4154   InstructionMark im(this);
4155   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4156   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
4157   simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4158   emit_int8(0x59);
4159   emit_operand(dst, src);
4160 }
4161 
4162 void Assembler::mulps(XMMRegister dst, XMMRegister src) {
4163   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4164   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4165   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
4166   emit_int8(0x59);
4167   emit_int8((unsigned char)(0xC0 | encode));
4168 }
4169 
4170 void Assembler::vmulpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4171   assert(VM_Version::supports_avx(), "");
4172   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4173   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
4174   int encode = vex_prefix_and_encode(dst->encoding(), nds_enc, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4175   emit_int8(0x59);
4176   emit_int8((unsigned char)(0xC0 | encode));
4177 }
4178 
4179 void Assembler::vmulps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4180   assert(VM_Version::supports_avx(), "");
4181   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4182   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
4183   int encode = vex_prefix_and_encode(dst->encoding(), nds_enc, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
4184   emit_int8(0x59);
4185   emit_int8((unsigned char)(0xC0 | encode));
4186 }
4187 
4188 void Assembler::vmulpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4189   assert(VM_Version::supports_avx(), "");
4190   InstructionMark im(this);
4191   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4192   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
4193   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
4194   vex_prefix(src, nds_enc, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4195   emit_int8(0x59);
4196   emit_operand(dst, src);
4197 }
4198 
4199 void Assembler::vmulps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4200   assert(VM_Version::supports_avx(), "");
4201   InstructionMark im(this);
4202   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4203   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
4204   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
4205   vex_prefix(src, nds_enc, dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
4206   emit_int8(0x59);
4207   emit_operand(dst, src);
4208 }
4209 
4210 void Assembler::divpd(XMMRegister dst, XMMRegister src) {
4211   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4212   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4213   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4214   emit_int8(0x5E);
4215   emit_int8((unsigned char)(0xC0 | encode));
4216 }
4217 
4218 void Assembler::divps(XMMRegister dst, XMMRegister src) {
4219   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4220   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4221   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
4222   emit_int8(0x5E);
4223   emit_int8((unsigned char)(0xC0 | encode));
4224 }
4225 
4226 void Assembler::vdivpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4227   assert(VM_Version::supports_avx(), "");
4228   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4229   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
4230   int encode = vex_prefix_and_encode(dst->encoding(), nds_enc, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4231   emit_int8(0x5E);
4232   emit_int8((unsigned char)(0xC0 | encode));
4233 }
4234 
4235 void Assembler::vdivps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4236   assert(VM_Version::supports_avx(), "");
4237   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4238   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
4239   int encode = vex_prefix_and_encode(dst->encoding(), nds_enc, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
4240   emit_int8(0x5E);
4241   emit_int8((unsigned char)(0xC0 | encode));
4242 }
4243 
4244 void Assembler::vdivpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4245   assert(VM_Version::supports_avx(), "");
4246   InstructionMark im(this);
4247   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4248   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
4249   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
4250   vex_prefix(src, nds_enc, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4251   emit_int8(0x5E);
4252   emit_operand(dst, src);
4253 }
4254 
4255 void Assembler::vdivps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4256   assert(VM_Version::supports_avx(), "");
4257   InstructionMark im(this);
4258   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4259   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
4260   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
4261   vex_prefix(src, nds_enc, dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
4262   emit_int8(0x5E);
4263   emit_operand(dst, src);
4264 }
4265 
4266 void Assembler::vsqrtpd(XMMRegister dst, XMMRegister src, int vector_len) {
4267   assert(VM_Version::supports_avx(), "");
4268   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4269   int nds_enc = 0;
4270   int encode = vex_prefix_and_encode(dst->encoding(), nds_enc, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4271   emit_int8(0x51);
4272   emit_int8((unsigned char)(0xC0 | encode));
4273 }
4274 
4275 void Assembler::vsqrtpd(XMMRegister dst, Address src, int vector_len) {
4276   assert(VM_Version::supports_avx(), "");
4277   InstructionMark im(this);
4278   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4279   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
4280   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4281   emit_int8(0x51);
4282   emit_operand(dst, src);
4283 }
4284 
4285 void Assembler::andpd(XMMRegister dst, XMMRegister src) {
4286   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4287   InstructionAttr attributes(AVX_128bit, /* rex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
4288   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4289   emit_int8(0x54);
4290   emit_int8((unsigned char)(0xC0 | encode));
4291 }
4292 
4293 void Assembler::andps(XMMRegister dst, XMMRegister src) {
4294   NOT_LP64(assert(VM_Version::supports_sse(), ""));
4295   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
4296   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
4297   emit_int8(0x54);
4298   emit_int8((unsigned char)(0xC0 | encode));
4299 }
4300 
4301 void Assembler::andps(XMMRegister dst, Address src) {
4302   NOT_LP64(assert(VM_Version::supports_sse(), ""));
4303   InstructionMark im(this);
4304   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
4305   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
4306   simd_prefix(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
4307   emit_int8(0x54);
4308   emit_operand(dst, src);
4309 }
4310 
4311 void Assembler::andpd(XMMRegister dst, Address src) {
4312   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4313   InstructionMark im(this);
4314   InstructionAttr attributes(AVX_128bit, /* rex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
4315   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
4316   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4317   emit_int8(0x54);
4318   emit_operand(dst, src);
4319 }
4320 
4321 void Assembler::vandpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4322   assert(VM_Version::supports_avx(), "");
4323   InstructionAttr attributes(vector_len, /* vex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
4324   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
4325   int encode = vex_prefix_and_encode(dst->encoding(), nds_enc, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4326   emit_int8(0x54);
4327   emit_int8((unsigned char)(0xC0 | encode));
4328 }
4329 
4330 void Assembler::vandps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4331   assert(VM_Version::supports_avx(), "");
4332   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
4333   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
4334   int encode = vex_prefix_and_encode(dst->encoding(), nds_enc, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
4335   emit_int8(0x54);
4336   emit_int8((unsigned char)(0xC0 | encode));
4337 }
4338 
4339 void Assembler::vandpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4340   assert(VM_Version::supports_avx(), "");
4341   InstructionMark im(this);
4342   InstructionAttr attributes(vector_len, /* vex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
4343   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
4344   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
4345   vex_prefix(src, nds_enc, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4346   emit_int8(0x54);
4347   emit_operand(dst, src);
4348 }
4349 
4350 void Assembler::vandps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4351   assert(VM_Version::supports_avx(), "");
4352   InstructionMark im(this);
4353   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
4354   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
4355   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
4356   vex_prefix(src, nds_enc, dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
4357   emit_int8(0x54);
4358   emit_operand(dst, src);
4359 }
4360 
4361 void Assembler::unpckhpd(XMMRegister dst, XMMRegister src) {
4362   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4363   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4364   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4365   emit_int8(0x15);
4366   emit_int8((unsigned char)(0xC0 | encode));
4367 }
4368 
4369 void Assembler::unpcklpd(XMMRegister dst, XMMRegister src) {
4370   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4371   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4372   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4373   emit_int8(0x14);
4374   emit_int8((unsigned char)(0xC0 | encode));
4375 }
4376 
4377 void Assembler::xorpd(XMMRegister dst, XMMRegister src) {
4378   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4379   InstructionAttr attributes(AVX_128bit, /* rex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
4380   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4381   emit_int8(0x57);
4382   emit_int8((unsigned char)(0xC0 | encode));
4383 }
4384 
4385 void Assembler::xorps(XMMRegister dst, XMMRegister src) {
4386   NOT_LP64(assert(VM_Version::supports_sse(), ""));
4387   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
4388   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
4389   emit_int8(0x57);
4390   emit_int8((unsigned char)(0xC0 | encode));
4391 }
4392 
4393 void Assembler::xorpd(XMMRegister dst, Address src) {
4394   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4395   InstructionMark im(this);
4396   InstructionAttr attributes(AVX_128bit, /* rex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
4397   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
4398   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4399   emit_int8(0x57);
4400   emit_operand(dst, src);
4401 }
4402 
4403 void Assembler::xorps(XMMRegister dst, Address src) {
4404   NOT_LP64(assert(VM_Version::supports_sse(), ""));
4405   InstructionMark im(this);
4406   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
4407   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
4408   simd_prefix(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
4409   emit_int8(0x57);
4410   emit_operand(dst, src);
4411 }
4412 
4413 void Assembler::vxorpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4414   assert(VM_Version::supports_avx(), "");
4415   InstructionAttr attributes(vector_len, /* vex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
4416   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
4417   int encode = vex_prefix_and_encode(dst->encoding(), nds_enc, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4418   emit_int8(0x57);
4419   emit_int8((unsigned char)(0xC0 | encode));
4420 }
4421 
4422 void Assembler::vxorps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4423   assert(VM_Version::supports_avx(), "");
4424   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
4425   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
4426   int encode = vex_prefix_and_encode(dst->encoding(), nds_enc, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
4427   emit_int8(0x57);
4428   emit_int8((unsigned char)(0xC0 | encode));
4429 }
4430 
4431 void Assembler::vxorpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4432   assert(VM_Version::supports_avx(), "");
4433   InstructionMark im(this);
4434   InstructionAttr attributes(vector_len, /* vex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
4435   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
4436   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
4437   vex_prefix(src, nds_enc, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4438   emit_int8(0x57);
4439   emit_operand(dst, src);
4440 }
4441 
4442 void Assembler::vxorps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4443   assert(VM_Version::supports_avx(), "");
4444   InstructionMark im(this);
4445   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
4446   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
4447   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
4448   vex_prefix(src, nds_enc, dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
4449   emit_int8(0x57);
4450   emit_operand(dst, src);
4451 }
4452 
4453 // Integer vector arithmetic
4454 void Assembler::vphaddw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4455   assert(VM_Version::supports_avx() && (vector_len == 0) ||
4456          VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
4457   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
4458   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
4459   int encode = vex_prefix_and_encode(dst->encoding(), nds_enc, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4460   emit_int8(0x01);
4461   emit_int8((unsigned char)(0xC0 | encode));
4462 }
4463 
4464 void Assembler::vphaddd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4465   assert(VM_Version::supports_avx() && (vector_len == 0) ||
4466          VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
4467   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
4468   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
4469   int encode = vex_prefix_and_encode(dst->encoding(), nds_enc, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4470   emit_int8(0x02);
4471   emit_int8((unsigned char)(0xC0 | encode));
4472 }
4473 
4474 void Assembler::paddb(XMMRegister dst, XMMRegister src) {
4475   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4476   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
4477   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4478   emit_int8((unsigned char)0xFC);
4479   emit_int8((unsigned char)(0xC0 | encode));
4480 }
4481 
4482 void Assembler::paddw(XMMRegister dst, XMMRegister src) {
4483   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4484   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
4485   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4486   emit_int8((unsigned char)0xFD);
4487   emit_int8((unsigned char)(0xC0 | encode));
4488 }
4489 
4490 void Assembler::paddd(XMMRegister dst, XMMRegister src) {
4491   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4492   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4493   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4494   emit_int8((unsigned char)0xFE);
4495   emit_int8((unsigned char)(0xC0 | encode));
4496 }
4497 
4498 void Assembler::paddq(XMMRegister dst, XMMRegister src) {
4499   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4500   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4501   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4502   emit_int8((unsigned char)0xD4);
4503   emit_int8((unsigned char)(0xC0 | encode));
4504 }
4505 
4506 void Assembler::phaddw(XMMRegister dst, XMMRegister src) {
4507   NOT_LP64(assert(VM_Version::supports_sse3(), ""));
4508   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
4509   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4510   emit_int8(0x01);
4511   emit_int8((unsigned char)(0xC0 | encode));
4512 }
4513 
4514 void Assembler::phaddd(XMMRegister dst, XMMRegister src) {
4515   NOT_LP64(assert(VM_Version::supports_sse3(), ""));
4516   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
4517   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4518   emit_int8(0x02);
4519   emit_int8((unsigned char)(0xC0 | encode));
4520 }
4521 
4522 void Assembler::vpaddb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4523   assert(UseAVX > 0, "requires some form of AVX");
4524   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
4525   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
4526   int encode = vex_prefix_and_encode(dst->encoding(), nds_enc, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4527   emit_int8((unsigned char)0xFC);
4528   emit_int8((unsigned char)(0xC0 | encode));
4529 }
4530 
4531 void Assembler::vpaddw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4532   assert(UseAVX > 0, "requires some form of AVX");
4533   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
4534   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
4535   int encode = vex_prefix_and_encode(dst->encoding(), nds_enc, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4536   emit_int8((unsigned char)0xFD);
4537   emit_int8((unsigned char)(0xC0 | encode));
4538 }
4539 
4540 void Assembler::vpaddd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4541   assert(UseAVX > 0, "requires some form of AVX");
4542   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4543   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
4544   int encode = vex_prefix_and_encode(dst->encoding(), nds_enc, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4545   emit_int8((unsigned char)0xFE);
4546   emit_int8((unsigned char)(0xC0 | encode));
4547 }
4548 
4549 void Assembler::vpaddq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4550   assert(UseAVX > 0, "requires some form of AVX");
4551   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4552   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
4553   int encode = vex_prefix_and_encode(dst->encoding(), nds_enc, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4554   emit_int8((unsigned char)0xD4);
4555   emit_int8((unsigned char)(0xC0 | encode));
4556 }
4557 
4558 void Assembler::vpaddb(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4559   assert(UseAVX > 0, "requires some form of AVX");
4560   InstructionMark im(this);
4561   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
4562   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
4563   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
4564   vex_prefix(src, nds_enc, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4565   emit_int8((unsigned char)0xFC);
4566   emit_operand(dst, src);
4567 }
4568 
4569 void Assembler::vpaddw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4570   assert(UseAVX > 0, "requires some form of AVX");
4571   InstructionMark im(this);
4572   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
4573   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
4574   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
4575   vex_prefix(src, nds_enc, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4576   emit_int8((unsigned char)0xFD);
4577   emit_operand(dst, src);
4578 }
4579 
4580 void Assembler::vpaddd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4581   assert(UseAVX > 0, "requires some form of AVX");
4582   InstructionMark im(this);
4583   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4584   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
4585   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
4586   vex_prefix(src, nds_enc, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4587   emit_int8((unsigned char)0xFE);
4588   emit_operand(dst, src);
4589 }
4590 
4591 void Assembler::vpaddq(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4592   assert(UseAVX > 0, "requires some form of AVX");
4593   InstructionMark im(this);
4594   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4595   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
4596   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
4597   vex_prefix(src, nds_enc, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4598   emit_int8((unsigned char)0xD4);
4599   emit_operand(dst, src);
4600 }
4601 
4602 void Assembler::psubb(XMMRegister dst, XMMRegister src) {
4603   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4604   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
4605   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4606   emit_int8((unsigned char)0xF8);
4607   emit_int8((unsigned char)(0xC0 | encode));
4608 }
4609 
4610 void Assembler::psubw(XMMRegister dst, XMMRegister src) {
4611   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4612   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
4613   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4614   emit_int8((unsigned char)0xF9);
4615   emit_int8((unsigned char)(0xC0 | encode));
4616 }
4617 
4618 void Assembler::psubd(XMMRegister dst, XMMRegister src) {
4619   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4620   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4621   emit_int8((unsigned char)0xFA);
4622   emit_int8((unsigned char)(0xC0 | encode));
4623 }
4624 
4625 void Assembler::psubq(XMMRegister dst, XMMRegister src) {
4626   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4627   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4628   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4629   emit_int8((unsigned char)0xFB);
4630   emit_int8((unsigned char)(0xC0 | encode));
4631 }
4632 
4633 void Assembler::vpsubb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4634   assert(UseAVX > 0, "requires some form of AVX");
4635   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
4636   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
4637   int encode = vex_prefix_and_encode(dst->encoding(), nds_enc, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4638   emit_int8((unsigned char)0xF8);
4639   emit_int8((unsigned char)(0xC0 | encode));
4640 }
4641 
4642 void Assembler::vpsubw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4643   assert(UseAVX > 0, "requires some form of AVX");
4644   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
4645   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
4646   int encode = vex_prefix_and_encode(dst->encoding(), nds_enc, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4647   emit_int8((unsigned char)0xF9);
4648   emit_int8((unsigned char)(0xC0 | encode));
4649 }
4650 
4651 void Assembler::vpsubd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4652   assert(UseAVX > 0, "requires some form of AVX");
4653   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4654   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
4655   int encode = vex_prefix_and_encode(dst->encoding(), nds_enc, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4656   emit_int8((unsigned char)0xFA);
4657   emit_int8((unsigned char)(0xC0 | encode));
4658 }
4659 
4660 void Assembler::vpsubq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4661   assert(UseAVX > 0, "requires some form of AVX");
4662   InstructionAttr attributes(vector_len, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4663   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
4664   int encode = vex_prefix_and_encode(dst->encoding(), nds_enc, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4665   emit_int8((unsigned char)0xFB);
4666   emit_int8((unsigned char)(0xC0 | encode));
4667 }
4668 
4669 void Assembler::vpsubb(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4670   assert(UseAVX > 0, "requires some form of AVX");
4671   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
4672   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
4673   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
4674   vex_prefix(src, nds_enc, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4675   emit_int8((unsigned char)0xF8);
4676   emit_operand(dst, src);
4677 }
4678 
4679 void Assembler::vpsubw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4680   assert(UseAVX > 0, "requires some form of AVX");
4681   InstructionMark im(this);
4682   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
4683   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
4684   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
4685   vex_prefix(src, nds_enc, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4686   emit_int8((unsigned char)0xF9);
4687   emit_operand(dst, src);
4688 }
4689 
4690 void Assembler::vpsubd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4691   assert(UseAVX > 0, "requires some form of AVX");
4692   InstructionMark im(this);
4693   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4694   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
4695   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
4696   vex_prefix(src, nds_enc, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4697   emit_int8((unsigned char)0xFA);
4698   emit_operand(dst, src);
4699 }
4700 
4701 void Assembler::vpsubq(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4702   assert(UseAVX > 0, "requires some form of AVX");
4703   InstructionMark im(this);
4704   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4705   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
4706   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
4707   vex_prefix(src, nds_enc, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4708   emit_int8((unsigned char)0xFB);
4709   emit_operand(dst, src);
4710 }
4711 
4712 void Assembler::pmullw(XMMRegister dst, XMMRegister src) {
4713   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4714   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
4715   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4716   emit_int8((unsigned char)0xD5);
4717   emit_int8((unsigned char)(0xC0 | encode));
4718 }
4719 
4720 void Assembler::pmulld(XMMRegister dst, XMMRegister src) {
4721   assert(VM_Version::supports_sse4_1(), "");
4722   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4723   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4724   emit_int8(0x40);
4725   emit_int8((unsigned char)(0xC0 | encode));
4726 }
4727 
4728 void Assembler::vpmullw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4729   assert(UseAVX > 0, "requires some form of AVX");
4730   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
4731   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
4732   int encode = vex_prefix_and_encode(dst->encoding(), nds_enc, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4733   emit_int8((unsigned char)0xD5);
4734   emit_int8((unsigned char)(0xC0 | encode));
4735 }
4736 
4737 void Assembler::vpmulld(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4738   assert(UseAVX > 0, "requires some form of AVX");
4739   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4740   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
4741   int encode = vex_prefix_and_encode(dst->encoding(), nds_enc, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4742   emit_int8(0x40);
4743   emit_int8((unsigned char)(0xC0 | encode));
4744 }
4745 
4746 void Assembler::vpmullq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4747   assert(UseAVX > 2, "requires some form of AVX");
4748   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
4749   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
4750   int encode = vex_prefix_and_encode(dst->encoding(), nds_enc, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4751   emit_int8(0x40);
4752   emit_int8((unsigned char)(0xC0 | encode));
4753 }
4754 
4755 void Assembler::vpmullw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4756   assert(UseAVX > 0, "requires some form of AVX");
4757   InstructionMark im(this);
4758   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
4759   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
4760   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
4761   vex_prefix(src, nds_enc, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4762   emit_int8((unsigned char)0xD5);
4763   emit_operand(dst, src);
4764 }
4765 
4766 void Assembler::vpmulld(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4767   assert(UseAVX > 0, "requires some form of AVX");
4768   InstructionMark im(this);
4769   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4770   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
4771   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
4772   vex_prefix(src, nds_enc, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4773   emit_int8(0x40);
4774   emit_operand(dst, src);
4775 }
4776 
4777 void Assembler::vpmullq(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4778   assert(UseAVX > 0, "requires some form of AVX");
4779   InstructionMark im(this);
4780   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
4781   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
4782   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
4783   vex_prefix(src, nds_enc, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4784   emit_int8(0x40);
4785   emit_operand(dst, src);
4786 }
4787 
4788 // Shift packed integers left by specified number of bits.
4789 void Assembler::psllw(XMMRegister dst, int shift) {
4790   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4791   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
4792   // XMM6 is for /6 encoding: 66 0F 71 /6 ib
4793   int encode = simd_prefix_and_encode(xmm6, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4794   emit_int8(0x71);
4795   emit_int8((unsigned char)(0xC0 | encode));
4796   emit_int8(shift & 0xFF);
4797 }
4798 
4799 void Assembler::pslld(XMMRegister dst, int shift) {
4800   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4801   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4802   // XMM6 is for /6 encoding: 66 0F 72 /6 ib
4803   int encode = simd_prefix_and_encode(xmm6, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4804   emit_int8(0x72);
4805   emit_int8((unsigned char)(0xC0 | encode));
4806   emit_int8(shift & 0xFF);
4807 }
4808 
4809 void Assembler::psllq(XMMRegister dst, int shift) {
4810   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4811   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4812   // XMM6 is for /6 encoding: 66 0F 73 /6 ib
4813   int encode = simd_prefix_and_encode(xmm6, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4814   emit_int8(0x73);
4815   emit_int8((unsigned char)(0xC0 | encode));
4816   emit_int8(shift & 0xFF);
4817 }
4818 
4819 void Assembler::psllw(XMMRegister dst, XMMRegister shift) {
4820   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4821   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
4822   int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4823   emit_int8((unsigned char)0xF1);
4824   emit_int8((unsigned char)(0xC0 | encode));
4825 }
4826 
4827 void Assembler::pslld(XMMRegister dst, XMMRegister shift) {
4828   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4829   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4830   int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4831   emit_int8((unsigned char)0xF2);
4832   emit_int8((unsigned char)(0xC0 | encode));
4833 }
4834 
4835 void Assembler::psllq(XMMRegister dst, XMMRegister shift) {
4836   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4837   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4838   int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4839   emit_int8((unsigned char)0xF3);
4840   emit_int8((unsigned char)(0xC0 | encode));
4841 }
4842 
4843 void Assembler::vpsllw(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
4844   assert(UseAVX > 0, "requires some form of AVX");
4845   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
4846   // XMM6 is for /6 encoding: 66 0F 71 /6 ib
4847   int encode = vex_prefix_and_encode(xmm6->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4848   emit_int8(0x71);
4849   emit_int8((unsigned char)(0xC0 | encode));
4850   emit_int8(shift & 0xFF);
4851 }
4852 
4853 void Assembler::vpslld(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
4854   assert(UseAVX > 0, "requires some form of AVX");
4855   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4856   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4857   // XMM6 is for /6 encoding: 66 0F 72 /6 ib
4858   int encode = vex_prefix_and_encode(xmm6->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4859   emit_int8(0x72);
4860   emit_int8((unsigned char)(0xC0 | encode));
4861   emit_int8(shift & 0xFF);
4862 }
4863 
4864 void Assembler::vpsllq(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
4865   assert(UseAVX > 0, "requires some form of AVX");
4866   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4867   // XMM6 is for /6 encoding: 66 0F 73 /6 ib
4868   int encode = vex_prefix_and_encode(xmm6->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4869   emit_int8(0x73);
4870   emit_int8((unsigned char)(0xC0 | encode));
4871   emit_int8(shift & 0xFF);
4872 }
4873 
4874 void Assembler::vpsllw(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
4875   assert(UseAVX > 0, "requires some form of AVX");
4876   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
4877   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4878   emit_int8((unsigned char)0xF1);
4879   emit_int8((unsigned char)(0xC0 | encode));
4880 }
4881 
4882 void Assembler::vpslld(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
4883   assert(UseAVX > 0, "requires some form of AVX");
4884   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4885   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4886   emit_int8((unsigned char)0xF2);
4887   emit_int8((unsigned char)(0xC0 | encode));
4888 }
4889 
4890 void Assembler::vpsllq(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
4891   assert(UseAVX > 0, "requires some form of AVX");
4892   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4893   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4894   emit_int8((unsigned char)0xF3);
4895   emit_int8((unsigned char)(0xC0 | encode));
4896 }
4897 
4898 // Shift packed integers logically right by specified number of bits.
4899 void Assembler::psrlw(XMMRegister dst, int shift) {
4900   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4901   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
4902   // XMM2 is for /2 encoding: 66 0F 71 /2 ib
4903   int encode = simd_prefix_and_encode(xmm2, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4904   emit_int8(0x71);
4905   emit_int8((unsigned char)(0xC0 | encode));
4906   emit_int8(shift & 0xFF);
4907 }
4908 
4909 void Assembler::psrld(XMMRegister dst, int shift) {
4910   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4911   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4912   // XMM2 is for /2 encoding: 66 0F 72 /2 ib
4913   int encode = simd_prefix_and_encode(xmm2, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4914   emit_int8(0x72);
4915   emit_int8((unsigned char)(0xC0 | encode));
4916   emit_int8(shift & 0xFF);
4917 }
4918 
4919 void Assembler::psrlq(XMMRegister dst, int shift) {
4920   // Do not confuse it with psrldq SSE2 instruction which
4921   // shifts 128 bit value in xmm register by number of bytes.
4922   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4923   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4924   // XMM2 is for /2 encoding: 66 0F 73 /2 ib
4925   int encode = simd_prefix_and_encode(xmm2, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4926   emit_int8(0x73);
4927   emit_int8((unsigned char)(0xC0 | encode));
4928   emit_int8(shift & 0xFF);
4929 }
4930 
4931 void Assembler::psrlw(XMMRegister dst, XMMRegister shift) {
4932   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4933   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
4934   int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4935   emit_int8((unsigned char)0xD1);
4936   emit_int8((unsigned char)(0xC0 | encode));
4937 }
4938 
4939 void Assembler::psrld(XMMRegister dst, XMMRegister shift) {
4940   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4941   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4942   int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4943   emit_int8((unsigned char)0xD2);
4944   emit_int8((unsigned char)(0xC0 | encode));
4945 }
4946 
4947 void Assembler::psrlq(XMMRegister dst, XMMRegister shift) {
4948   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4949   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4950   int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4951   emit_int8((unsigned char)0xD3);
4952   emit_int8((unsigned char)(0xC0 | encode));
4953 }
4954 
4955 void Assembler::vpsrlw(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
4956   assert(UseAVX > 0, "requires some form of AVX");
4957   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
4958   // XMM2 is for /2 encoding: 66 0F 71 /2 ib
4959   int encode = vex_prefix_and_encode(xmm2->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4960   emit_int8(0x71);
4961   emit_int8((unsigned char)(0xC0 | encode));
4962   emit_int8(shift & 0xFF);
4963 }
4964 
4965 void Assembler::vpsrld(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
4966   assert(UseAVX > 0, "requires some form of AVX");
4967   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4968   // XMM2 is for /2 encoding: 66 0F 72 /2 ib
4969   int encode = vex_prefix_and_encode(xmm2->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4970   emit_int8(0x72);
4971   emit_int8((unsigned char)(0xC0 | encode));
4972   emit_int8(shift & 0xFF);
4973 }
4974 
4975 void Assembler::vpsrlq(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
4976   assert(UseAVX > 0, "requires some form of AVX");
4977   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4978   // XMM2 is for /2 encoding: 66 0F 73 /2 ib
4979   int encode = vex_prefix_and_encode(xmm2->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4980   emit_int8(0x73);
4981   emit_int8((unsigned char)(0xC0 | encode));
4982   emit_int8(shift & 0xFF);
4983 }
4984 
4985 void Assembler::vpsrlw(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
4986   assert(UseAVX > 0, "requires some form of AVX");
4987   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
4988   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4989   emit_int8((unsigned char)0xD1);
4990   emit_int8((unsigned char)(0xC0 | encode));
4991 }
4992 
4993 void Assembler::vpsrld(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
4994   assert(UseAVX > 0, "requires some form of AVX");
4995   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4996   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4997   emit_int8((unsigned char)0xD2);
4998   emit_int8((unsigned char)(0xC0 | encode));
4999 }
5000 
5001 void Assembler::vpsrlq(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
5002   assert(UseAVX > 0, "requires some form of AVX");
5003   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5004   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5005   emit_int8((unsigned char)0xD3);
5006   emit_int8((unsigned char)(0xC0 | encode));
5007 }
5008 
5009 // Shift packed integers arithmetically right by specified number of bits.
5010 void Assembler::psraw(XMMRegister dst, int shift) {
5011   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5012   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
5013   // XMM4 is for /4 encoding: 66 0F 71 /4 ib
5014   int encode = simd_prefix_and_encode(xmm4, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5015   emit_int8(0x71);
5016   emit_int8((unsigned char)(0xC0 | encode));
5017   emit_int8(shift & 0xFF);
5018 }
5019 
5020 void Assembler::psrad(XMMRegister dst, int shift) {
5021   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5022   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5023   // XMM4 is for /4 encoding: 66 0F 72 /4 ib
5024   int encode = simd_prefix_and_encode(xmm4, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5025   emit_int8(0x72);
5026   emit_int8((unsigned char)(0xC0 | encode));
5027   emit_int8(shift & 0xFF);
5028 }
5029 
5030 void Assembler::psraw(XMMRegister dst, XMMRegister shift) {
5031   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5032   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
5033   int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5034   emit_int8((unsigned char)0xE1);
5035   emit_int8((unsigned char)(0xC0 | encode));
5036 }
5037 
5038 void Assembler::psrad(XMMRegister dst, XMMRegister shift) {
5039   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5040   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5041   int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5042   emit_int8((unsigned char)0xE2);
5043   emit_int8((unsigned char)(0xC0 | encode));
5044 }
5045 
5046 void Assembler::vpsraw(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
5047   assert(UseAVX > 0, "requires some form of AVX");
5048   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
5049   // XMM4 is for /4 encoding: 66 0F 71 /4 ib
5050   int encode = vex_prefix_and_encode(xmm4->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5051   emit_int8(0x71);
5052   emit_int8((unsigned char)(0xC0 | encode));
5053   emit_int8(shift & 0xFF);
5054 }
5055 
5056 void Assembler::vpsrad(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
5057   assert(UseAVX > 0, "requires some form of AVX");
5058   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5059   // XMM4 is for /4 encoding: 66 0F 71 /4 ib
5060   int encode = vex_prefix_and_encode(xmm4->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5061   emit_int8(0x72);
5062   emit_int8((unsigned char)(0xC0 | encode));
5063   emit_int8(shift & 0xFF);
5064 }
5065 
5066 void Assembler::vpsraw(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
5067   assert(UseAVX > 0, "requires some form of AVX");
5068   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
5069   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5070   emit_int8((unsigned char)0xE1);
5071   emit_int8((unsigned char)(0xC0 | encode));
5072 }
5073 
5074 void Assembler::vpsrad(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
5075   assert(UseAVX > 0, "requires some form of AVX");
5076   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5077   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5078   emit_int8((unsigned char)0xE2);
5079   emit_int8((unsigned char)(0xC0 | encode));
5080 }
5081 
5082 
5083 // logical operations packed integers
5084 void Assembler::pand(XMMRegister dst, XMMRegister src) {
5085   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5086   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5087   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5088   emit_int8((unsigned char)0xDB);
5089   emit_int8((unsigned char)(0xC0 | encode));
5090 }
5091 
5092 void Assembler::vpand(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5093   assert(UseAVX > 0, "requires some form of AVX");
5094   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5095   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
5096   int encode = vex_prefix_and_encode(dst->encoding(), nds_enc, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5097   emit_int8((unsigned char)0xDB);
5098   emit_int8((unsigned char)(0xC0 | encode));
5099 }
5100 
5101 void Assembler::vpand(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5102   assert(UseAVX > 0, "requires some form of AVX");
5103   InstructionMark im(this);
5104   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5105   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
5106   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
5107   vex_prefix(src, nds_enc, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5108   emit_int8((unsigned char)0xDB);
5109   emit_operand(dst, src);
5110 }
5111 
5112 void Assembler::pandn(XMMRegister dst, XMMRegister src) {
5113   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5114   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5115   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5116   emit_int8((unsigned char)0xDF);
5117   emit_int8((unsigned char)(0xC0 | encode));
5118 }
5119 
5120 void Assembler::por(XMMRegister dst, XMMRegister src) {
5121   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5122   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5123   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5124   emit_int8((unsigned char)0xEB);
5125   emit_int8((unsigned char)(0xC0 | encode));
5126 }
5127 
5128 void Assembler::vpor(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5129   assert(UseAVX > 0, "requires some form of AVX");
5130   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5131   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
5132   int encode = vex_prefix_and_encode(dst->encoding(), nds_enc, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5133   emit_int8((unsigned char)0xEB);
5134   emit_int8((unsigned char)(0xC0 | encode));
5135 }
5136 
5137 void Assembler::vpor(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5138   assert(UseAVX > 0, "requires some form of AVX");
5139   InstructionMark im(this);
5140   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5141   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
5142   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
5143   vex_prefix(src, nds_enc, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5144   emit_int8((unsigned char)0xEB);
5145   emit_operand(dst, src);
5146 }
5147 
5148 void Assembler::pxor(XMMRegister dst, XMMRegister src) {
5149   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5150   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5151   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5152   emit_int8((unsigned char)0xEF);
5153   emit_int8((unsigned char)(0xC0 | encode));
5154 }
5155 
5156 void Assembler::vpxor(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5157   assert(UseAVX > 0, "requires some form of AVX");
5158   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5159   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
5160   int encode = vex_prefix_and_encode(dst->encoding(), nds_enc, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5161   emit_int8((unsigned char)0xEF);
5162   emit_int8((unsigned char)(0xC0 | encode));
5163 }
5164 
5165 void Assembler::vpxor(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5166   assert(UseAVX > 0, "requires some form of AVX");
5167   InstructionMark im(this);
5168   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5169   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
5170   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
5171   vex_prefix(src, nds_enc, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5172   emit_int8((unsigned char)0xEF);
5173   emit_operand(dst, src);
5174 }
5175 
5176 
5177 void Assembler::vinsertf128h(XMMRegister dst, XMMRegister nds, XMMRegister src) {
5178   assert(VM_Version::supports_avx(), "");
5179   int vector_len = VM_Version::supports_evex() ? AVX_512bit : AVX_256bit;
5180   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
5181   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
5182   int encode = vex_prefix_and_encode(dst->encoding(), nds_enc, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
5183   emit_int8(0x18);
5184   emit_int8((unsigned char)(0xC0 | encode));
5185   // 0x00 - insert into lower 128 bits
5186   // 0x01 - insert into upper 128 bits
5187   emit_int8(0x01);
5188 }
5189 
5190 void Assembler::vinsertf64x4h(XMMRegister dst, XMMRegister nds, XMMRegister src, int value) {
5191   assert(VM_Version::supports_evex(), "");
5192   InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
5193   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
5194   int encode = vex_prefix_and_encode(dst->encoding(), nds_enc, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
5195   emit_int8(0x1A);
5196   emit_int8((unsigned char)(0xC0 | encode));
5197   // 0x00 - insert into lower 256 bits
5198   // 0x01 - insert into upper 256 bits
5199   emit_int8(value & 0x01);
5200 }
5201 
5202 void Assembler::vinsertf64x4h(XMMRegister dst, Address src, int value) {
5203   assert(VM_Version::supports_evex(), "");
5204   assert(dst != xnoreg, "sanity");
5205   InstructionMark im(this);
5206   InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
5207   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_64bit);
5208   // swap src<->dst for encoding
5209   vex_prefix(src, dst->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
5210   emit_int8(0x1A);
5211   emit_operand(dst, src);
5212   // 0x00 - insert into lower 256 bits
5213   // 0x01 - insert into upper 128 bits
5214   emit_int8(value & 0x01);
5215 }
5216 
5217 void Assembler::vinsertf32x4h(XMMRegister dst, XMMRegister nds, XMMRegister src, int value) {
5218   assert(VM_Version::supports_evex(), "");
5219   InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
5220   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
5221   int encode = vex_prefix_and_encode(dst->encoding(), nds_enc, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
5222   emit_int8(0x18);
5223   emit_int8((unsigned char)(0xC0 | encode));
5224   // 0x00 - insert into q0 128 bits (0..127)
5225   // 0x01 - insert into q1 128 bits (128..255)
5226   // 0x02 - insert into q2 128 bits (256..383)
5227   // 0x03 - insert into q3 128 bits (384..511)
5228   emit_int8(value & 0x3);
5229 }
5230 
5231 void Assembler::vinsertf32x4h(XMMRegister dst, Address src, int value) {
5232   assert(VM_Version::supports_avx(), "");
5233   assert(dst != xnoreg, "sanity");
5234   int vector_len = VM_Version::supports_evex() ? AVX_512bit : AVX_256bit;
5235   InstructionMark im(this);
5236   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
5237   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
5238   // swap src<->dst for encoding
5239   vex_prefix(src, dst->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
5240   emit_int8(0x18);
5241   emit_operand(dst, src);
5242   // 0x00 - insert into q0 128 bits (0..127)
5243   // 0x01 - insert into q1 128 bits (128..255)
5244   // 0x02 - insert into q2 128 bits (256..383)
5245   // 0x03 - insert into q3 128 bits (384..511)
5246   emit_int8(value & 0x3);
5247 }
5248 
5249 void Assembler::vinsertf128h(XMMRegister dst, Address src) {
5250   assert(VM_Version::supports_avx(), "");
5251   assert(dst != xnoreg, "sanity");
5252   int vector_len = VM_Version::supports_evex() ? AVX_512bit : AVX_256bit;
5253   InstructionMark im(this);
5254   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
5255   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
5256   // swap src<->dst for encoding
5257   vex_prefix(src, dst->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
5258   emit_int8(0x18);
5259   emit_operand(dst, src);
5260   // 0x01 - insert into upper 128 bits
5261   emit_int8(0x01);
5262 }
5263 
5264 void Assembler::vextractf128h(XMMRegister dst, XMMRegister src) {
5265   assert(VM_Version::supports_avx(), "");
5266   int vector_len = VM_Version::supports_evex() ? AVX_512bit : AVX_256bit;
5267   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
5268   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
5269   emit_int8(0x19);
5270   emit_int8((unsigned char)(0xC0 | encode));
5271   // 0x00 - insert into lower 128 bits
5272   // 0x01 - insert into upper 128 bits
5273   emit_int8(0x01);
5274 }
5275 
5276 void Assembler::vextractf128h(Address dst, XMMRegister src) {
5277   assert(VM_Version::supports_avx(), "");
5278   assert(src != xnoreg, "sanity");
5279   int vector_len = VM_Version::supports_evex() ? AVX_512bit : AVX_256bit;
5280   InstructionMark im(this);
5281   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
5282   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
5283   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
5284   emit_int8(0x19);
5285   emit_operand(src, dst);
5286   // 0x01 - extract from upper 128 bits
5287   emit_int8(0x01);
5288 }
5289 
5290 void Assembler::vinserti128h(XMMRegister dst, XMMRegister nds, XMMRegister src) {
5291   assert(VM_Version::supports_avx2(), "");
5292   int vector_len = VM_Version::supports_evex() ? AVX_512bit : AVX_256bit;
5293   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
5294   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
5295   int encode = vex_prefix_and_encode(dst->encoding(), nds_enc, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
5296   emit_int8(0x38);
5297   emit_int8((unsigned char)(0xC0 | encode));
5298   // 0x00 - insert into lower 128 bits
5299   // 0x01 - insert into upper 128 bits
5300   emit_int8(0x01);
5301 }
5302 
5303 void Assembler::vinserti64x4h(XMMRegister dst, XMMRegister nds, XMMRegister src, int value) {
5304   assert(VM_Version::supports_evex(), "");
5305   InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
5306   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
5307   int encode = vex_prefix_and_encode(dst->encoding(), nds_enc, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
5308   emit_int8(0x38);
5309   emit_int8((unsigned char)(0xC0 | encode));
5310   // 0x00 - insert into lower 256 bits
5311   // 0x01 - insert into upper 256 bits
5312   emit_int8(value & 0x01);
5313 }
5314 
5315 void Assembler::vinserti128h(XMMRegister dst, Address src) {
5316   assert(VM_Version::supports_avx2(), "");
5317   assert(dst != xnoreg, "sanity");
5318   int vector_len = VM_Version::supports_evex() ? AVX_512bit : AVX_256bit;
5319   InstructionMark im(this);
5320   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
5321   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
5322   // swap src<->dst for encoding
5323   vex_prefix(src, dst->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
5324   emit_int8(0x38);
5325   emit_operand(dst, src);
5326   // 0x01 - insert into upper 128 bits
5327   emit_int8(0x01);
5328 }
5329 
5330 void Assembler::vextracti128h(XMMRegister dst, XMMRegister src) {
5331   assert(VM_Version::supports_avx(), "");
5332   int vector_len = VM_Version::supports_evex() ? AVX_512bit : AVX_256bit;
5333   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
5334   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
5335   emit_int8(0x39);
5336   emit_int8((unsigned char)(0xC0 | encode));
5337   // 0x00 - insert into lower 128 bits
5338   // 0x01 - insert into upper 128 bits
5339   emit_int8(0x01);
5340 }
5341 
5342 void Assembler::vextracti128h(Address dst, XMMRegister src) {
5343   assert(VM_Version::supports_avx2(), "");
5344   assert(src != xnoreg, "sanity");
5345   int vector_len = VM_Version::supports_evex() ? AVX_512bit : AVX_256bit;
5346   InstructionMark im(this);
5347   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
5348   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
5349   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
5350   emit_int8(0x39);
5351   emit_operand(src, dst);
5352   // 0x01 - extract from upper 128 bits
5353   emit_int8(0x01);
5354 }
5355 
5356 void Assembler::vextracti64x4h(XMMRegister dst, XMMRegister src, int value) {
5357   assert(VM_Version::supports_evex(), "");
5358   InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
5359   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
5360   emit_int8(0x3B);
5361   emit_int8((unsigned char)(0xC0 | encode));
5362   // 0x00 - extract from lower 256 bits
5363   // 0x01 - extract from upper 256 bits
5364   emit_int8(value & 0x01);
5365 }
5366 
5367 void Assembler::vextracti64x2h(XMMRegister dst, XMMRegister src, int value) {
5368   assert(VM_Version::supports_evex(), "");
5369   InstructionAttr attributes(AVX_512bit, /* vex_w */ !_legacy_mode_dq, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
5370   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
5371   emit_int8(0x39);
5372   emit_int8((unsigned char)(0xC0 | encode));
5373   // 0x01 - extract from bits 255:128
5374   // 0x02 - extract from bits 383:256
5375   // 0x03 - extract from bits 511:384
5376   emit_int8(value & 0x3);
5377 }
5378 
5379 void Assembler::vextractf64x4h(XMMRegister dst, XMMRegister src, int value) {
5380   assert(VM_Version::supports_evex(), "");
5381   InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
5382   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
5383   emit_int8(0x1B);
5384   emit_int8((unsigned char)(0xC0 | encode));
5385   // 0x00 - extract from lower 256 bits
5386   // 0x01 - extract from upper 256 bits
5387   emit_int8(value & 0x1);
5388 }
5389 
5390 void Assembler::vextractf64x4h(Address dst, XMMRegister src, int value) {
5391   assert(VM_Version::supports_evex(), "");
5392   assert(src != xnoreg, "sanity");
5393   InstructionMark im(this);
5394   InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
5395   attributes.set_address_attributes(/* tuple_type */ EVEX_T4,/* input_size_in_bits */  EVEX_64bit);
5396   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
5397   emit_int8(0x1B);
5398   emit_operand(src, dst);
5399   // 0x00 - extract from lower 256 bits
5400   // 0x01 - extract from upper 256 bits
5401   emit_int8(value & 0x01);
5402 }
5403 
5404 void Assembler::vextractf32x4h(XMMRegister dst, XMMRegister src, int value) {
5405   assert(VM_Version::supports_avx(), "");
5406   int vector_len = VM_Version::supports_evex() ? AVX_512bit : AVX_256bit;
5407   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
5408   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
5409   emit_int8(0x19);
5410   emit_int8((unsigned char)(0xC0 | encode));
5411   // 0x00 - extract from bits 127:0
5412   // 0x01 - extract from bits 255:128
5413   // 0x02 - extract from bits 383:256
5414   // 0x03 - extract from bits 511:384
5415   emit_int8(value & 0x3);
5416 }
5417 
5418 void Assembler::vextractf32x4h(Address dst, XMMRegister src, int value) {
5419   assert(VM_Version::supports_evex(), "");
5420   assert(src != xnoreg, "sanity");
5421   InstructionMark im(this);
5422   InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
5423   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
5424   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
5425   emit_int8(0x19);
5426   emit_operand(src, dst);
5427   // 0x00 - extract from bits 127:0
5428   // 0x01 - extract from bits 255:128
5429   // 0x02 - extract from bits 383:256
5430   // 0x03 - extract from bits 511:384
5431   emit_int8(value & 0x3);
5432 }
5433 
5434 void Assembler::vextractf64x2h(XMMRegister dst, XMMRegister src, int value) {
5435   assert(VM_Version::supports_evex(), "");
5436   InstructionAttr attributes(AVX_512bit, /* vex_w */ !_legacy_mode_dq, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
5437   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
5438   emit_int8(0x19);
5439   emit_int8((unsigned char)(0xC0 | encode));
5440   // 0x01 - extract from bits 255:128
5441   // 0x02 - extract from bits 383:256
5442   // 0x03 - extract from bits 511:384
5443   emit_int8(value & 0x3);
5444 }
5445 
5446 // duplicate 4-bytes integer data from src into 8 locations in dest
5447 void Assembler::vpbroadcastd(XMMRegister dst, XMMRegister src) {
5448   assert(UseAVX > 1, "");
5449   InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5450   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5451   emit_int8(0x58);
5452   emit_int8((unsigned char)(0xC0 | encode));
5453 }
5454 
5455 // duplicate 1-byte integer data from src into 16||32|64 locations in dest : requires AVX512BW and AVX512VL
5456 void Assembler::evpbroadcastb(XMMRegister dst, XMMRegister src, int vector_len) {
5457   assert(UseAVX > 1, "");
5458   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5459   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5460   emit_int8(0x78);
5461   emit_int8((unsigned char)(0xC0 | encode));
5462 }
5463 
5464 void Assembler::evpbroadcastb(XMMRegister dst, Address src, int vector_len) {
5465   assert(UseAVX > 1, "");
5466   assert(dst != xnoreg, "sanity");
5467   InstructionMark im(this);
5468   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5469   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_8bit);
5470   // swap src<->dst for encoding
5471   vex_prefix(src, dst->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5472   emit_int8(0x78);
5473   emit_operand(dst, src);
5474 }
5475 
5476 // duplicate 2-byte integer data from src into 8|16||32 locations in dest : requires AVX512BW and AVX512VL
5477 void Assembler::evpbroadcastw(XMMRegister dst, XMMRegister src, int vector_len) {
5478   assert(UseAVX > 1, "");
5479   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5480   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5481   emit_int8(0x79);
5482   emit_int8((unsigned char)(0xC0 | encode));
5483 }
5484 
5485 void Assembler::evpbroadcastw(XMMRegister dst, Address src, int vector_len) {
5486   assert(UseAVX > 1, "");
5487   assert(dst != xnoreg, "sanity");
5488   InstructionMark im(this);
5489   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5490   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_16bit);
5491   // swap src<->dst for encoding
5492   vex_prefix(src, dst->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5493   emit_int8(0x79);
5494   emit_operand(dst, src);
5495 }
5496 
5497 // duplicate 4-byte integer data from src into 4|8|16 locations in dest : requires AVX512VL
5498 void Assembler::evpbroadcastd(XMMRegister dst, XMMRegister src, int vector_len) {
5499   assert(UseAVX > 1, "");
5500   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5501   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5502   emit_int8(0x58);
5503   emit_int8((unsigned char)(0xC0 | encode));
5504 }
5505 
5506 void Assembler::evpbroadcastd(XMMRegister dst, Address src, int vector_len) {
5507   assert(UseAVX > 1, "");
5508   assert(dst != xnoreg, "sanity");
5509   InstructionMark im(this);
5510   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5511   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
5512   // swap src<->dst for encoding
5513   vex_prefix(src, dst->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5514   emit_int8(0x58);
5515   emit_operand(dst, src);
5516 }
5517 
5518 // duplicate 8-byte integer data from src into 4|8|16 locations in dest : requires AVX512VL
5519 void Assembler::evpbroadcastq(XMMRegister dst, XMMRegister src, int vector_len) {
5520   assert(UseAVX > 1, "");
5521   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5522   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5523   emit_int8(0x59);
5524   emit_int8((unsigned char)(0xC0 | encode));
5525 }
5526 
5527 void Assembler::evpbroadcastq(XMMRegister dst, Address src, int vector_len) {
5528   assert(UseAVX > 1, "");
5529   assert(dst != xnoreg, "sanity");
5530   InstructionMark im(this);
5531   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5532   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
5533   // swap src<->dst for encoding
5534   vex_prefix(src, dst->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5535   emit_int8(0x59);
5536   emit_operand(dst, src);
5537 }
5538 
5539 // duplicate single precision fp from src into 4|8|16 locations in dest : requires AVX512VL
5540 void Assembler::evpbroadcastss(XMMRegister dst, XMMRegister src, int vector_len) {
5541   assert(UseAVX > 1, "");
5542   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5543   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5544   emit_int8(0x18);
5545   emit_int8((unsigned char)(0xC0 | encode));
5546 }
5547 
5548 void Assembler::evpbroadcastss(XMMRegister dst, Address src, int vector_len) {
5549   assert(UseAVX > 1, "");
5550   assert(dst != xnoreg, "sanity");
5551   InstructionMark im(this);
5552   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5553   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
5554   // swap src<->dst for encoding
5555   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5556   emit_int8(0x18);
5557   emit_operand(dst, src);
5558 }
5559 
5560 // duplicate double precision fp from src into 2|4|8 locations in dest : requires AVX512VL
5561 void Assembler::evpbroadcastsd(XMMRegister dst, XMMRegister src, int vector_len) {
5562   assert(UseAVX > 1, "");
5563   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5564   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5565   emit_int8(0x19);
5566   emit_int8((unsigned char)(0xC0 | encode));
5567 }
5568 
5569 void Assembler::evpbroadcastsd(XMMRegister dst, Address src, int vector_len) {
5570   assert(UseAVX > 1, "");
5571   assert(dst != xnoreg, "sanity");
5572   InstructionMark im(this);
5573   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5574   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
5575   // swap src<->dst for encoding
5576   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5577   emit_int8(0x19);
5578   emit_operand(dst, src);
5579 }
5580 
5581 // duplicate 1-byte integer data from src into 16||32|64 locations in dest : requires AVX512BW and AVX512VL
5582 void Assembler::evpbroadcastb(XMMRegister dst, Register src, int vector_len) {
5583   assert(VM_Version::supports_evex(), "");
5584   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5585   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5586   emit_int8(0x7A);
5587   emit_int8((unsigned char)(0xC0 | encode));
5588 }
5589 
5590 // duplicate 2-byte integer data from src into 8|16||32 locations in dest : requires AVX512BW and AVX512VL
5591 void Assembler::evpbroadcastw(XMMRegister dst, Register src, int vector_len) {
5592   assert(VM_Version::supports_evex(), "");
5593   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5594   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5595   emit_int8(0x7B);
5596   emit_int8((unsigned char)(0xC0 | encode));
5597 }
5598 
5599 // duplicate 4-byte integer data from src into 4|8|16 locations in dest : requires AVX512VL
5600 void Assembler::evpbroadcastd(XMMRegister dst, Register src, int vector_len) {
5601   assert(VM_Version::supports_evex(), "");
5602   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5603   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5604   emit_int8(0x7C);
5605   emit_int8((unsigned char)(0xC0 | encode));
5606 }
5607 
5608 // duplicate 8-byte integer data from src into 4|8|16 locations in dest : requires AVX512VL
5609 void Assembler::evpbroadcastq(XMMRegister dst, Register src, int vector_len) {
5610   assert(VM_Version::supports_evex(), "");
5611   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5612   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5613   emit_int8(0x7C);
5614   emit_int8((unsigned char)(0xC0 | encode));
5615 }
5616 
5617 // Carry-Less Multiplication Quadword
5618 void Assembler::pclmulqdq(XMMRegister dst, XMMRegister src, int mask) {
5619   assert(VM_Version::supports_clmul(), "");
5620   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
5621   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
5622   emit_int8(0x44);
5623   emit_int8((unsigned char)(0xC0 | encode));
5624   emit_int8((unsigned char)mask);
5625 }
5626 
5627 // Carry-Less Multiplication Quadword
5628 void Assembler::vpclmulqdq(XMMRegister dst, XMMRegister nds, XMMRegister src, int mask) {
5629   assert(VM_Version::supports_avx() && VM_Version::supports_clmul(), "");
5630   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
5631   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
5632   int encode = vex_prefix_and_encode(dst->encoding(), nds_enc, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
5633   emit_int8(0x44);
5634   emit_int8((unsigned char)(0xC0 | encode));
5635   emit_int8((unsigned char)mask);
5636 }
5637 
5638 void Assembler::vzeroupper() {
5639   assert(VM_Version::supports_avx(), "");
5640   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
5641   (void)vex_prefix_and_encode(0, 0, 0, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
5642   emit_int8(0x77);
5643 }
5644 
5645 
5646 #ifndef _LP64
5647 // 32bit only pieces of the assembler
5648 
5649 void Assembler::cmp_literal32(Register src1, int32_t imm32, RelocationHolder const& rspec) {
5650   // NO PREFIX AS NEVER 64BIT
5651   InstructionMark im(this);
5652   emit_int8((unsigned char)0x81);
5653   emit_int8((unsigned char)(0xF8 | src1->encoding()));
5654   emit_data(imm32, rspec, 0);
5655 }
5656 
5657 void Assembler::cmp_literal32(Address src1, int32_t imm32, RelocationHolder const& rspec) {
5658   // NO PREFIX AS NEVER 64BIT (not even 32bit versions of 64bit regs
5659   InstructionMark im(this);
5660   emit_int8((unsigned char)0x81);
5661   emit_operand(rdi, src1);
5662   emit_data(imm32, rspec, 0);
5663 }
5664 
5665 // The 64-bit (32bit platform) cmpxchg compares the value at adr with the contents of rdx:rax,
5666 // and stores rcx:rbx into adr if so; otherwise, the value at adr is loaded
5667 // into rdx:rax.  The ZF is set if the compared values were equal, and cleared otherwise.
5668 void Assembler::cmpxchg8(Address adr) {
5669   InstructionMark im(this);
5670   emit_int8(0x0F);
5671   emit_int8((unsigned char)0xC7);
5672   emit_operand(rcx, adr);
5673 }
5674 
5675 void Assembler::decl(Register dst) {
5676   // Don't use it directly. Use MacroAssembler::decrementl() instead.
5677  emit_int8(0x48 | dst->encoding());
5678 }
5679 
5680 #endif // _LP64
5681 
5682 // 64bit typically doesn't use the x87 but needs to for the trig funcs
5683 
5684 void Assembler::fabs() {
5685   emit_int8((unsigned char)0xD9);
5686   emit_int8((unsigned char)0xE1);
5687 }
5688 
5689 void Assembler::fadd(int i) {
5690   emit_farith(0xD8, 0xC0, i);
5691 }
5692 
5693 void Assembler::fadd_d(Address src) {
5694   InstructionMark im(this);
5695   emit_int8((unsigned char)0xDC);
5696   emit_operand32(rax, src);
5697 }
5698 
5699 void Assembler::fadd_s(Address src) {
5700   InstructionMark im(this);
5701   emit_int8((unsigned char)0xD8);
5702   emit_operand32(rax, src);
5703 }
5704 
5705 void Assembler::fadda(int i) {
5706   emit_farith(0xDC, 0xC0, i);
5707 }
5708 
5709 void Assembler::faddp(int i) {
5710   emit_farith(0xDE, 0xC0, i);
5711 }
5712 
5713 void Assembler::fchs() {
5714   emit_int8((unsigned char)0xD9);
5715   emit_int8((unsigned char)0xE0);
5716 }
5717 
5718 void Assembler::fcom(int i) {
5719   emit_farith(0xD8, 0xD0, i);
5720 }
5721 
5722 void Assembler::fcomp(int i) {
5723   emit_farith(0xD8, 0xD8, i);
5724 }
5725 
5726 void Assembler::fcomp_d(Address src) {
5727   InstructionMark im(this);
5728   emit_int8((unsigned char)0xDC);
5729   emit_operand32(rbx, src);
5730 }
5731 
5732 void Assembler::fcomp_s(Address src) {
5733   InstructionMark im(this);
5734   emit_int8((unsigned char)0xD8);
5735   emit_operand32(rbx, src);
5736 }
5737 
5738 void Assembler::fcompp() {
5739   emit_int8((unsigned char)0xDE);
5740   emit_int8((unsigned char)0xD9);
5741 }
5742 
5743 void Assembler::fcos() {
5744   emit_int8((unsigned char)0xD9);
5745   emit_int8((unsigned char)0xFF);
5746 }
5747 
5748 void Assembler::fdecstp() {
5749   emit_int8((unsigned char)0xD9);
5750   emit_int8((unsigned char)0xF6);
5751 }
5752 
5753 void Assembler::fdiv(int i) {
5754   emit_farith(0xD8, 0xF0, i);
5755 }
5756 
5757 void Assembler::fdiv_d(Address src) {
5758   InstructionMark im(this);
5759   emit_int8((unsigned char)0xDC);
5760   emit_operand32(rsi, src);
5761 }
5762 
5763 void Assembler::fdiv_s(Address src) {
5764   InstructionMark im(this);
5765   emit_int8((unsigned char)0xD8);
5766   emit_operand32(rsi, src);
5767 }
5768 
5769 void Assembler::fdiva(int i) {
5770   emit_farith(0xDC, 0xF8, i);
5771 }
5772 
5773 // Note: The Intel manual (Pentium Processor User's Manual, Vol.3, 1994)
5774 //       is erroneous for some of the floating-point instructions below.
5775 
5776 void Assembler::fdivp(int i) {
5777   emit_farith(0xDE, 0xF8, i);                    // ST(0) <- ST(0) / ST(1) and pop (Intel manual wrong)
5778 }
5779 
5780 void Assembler::fdivr(int i) {
5781   emit_farith(0xD8, 0xF8, i);
5782 }
5783 
5784 void Assembler::fdivr_d(Address src) {
5785   InstructionMark im(this);
5786   emit_int8((unsigned char)0xDC);
5787   emit_operand32(rdi, src);
5788 }
5789 
5790 void Assembler::fdivr_s(Address src) {
5791   InstructionMark im(this);
5792   emit_int8((unsigned char)0xD8);
5793   emit_operand32(rdi, src);
5794 }
5795 
5796 void Assembler::fdivra(int i) {
5797   emit_farith(0xDC, 0xF0, i);
5798 }
5799 
5800 void Assembler::fdivrp(int i) {
5801   emit_farith(0xDE, 0xF0, i);                    // ST(0) <- ST(1) / ST(0) and pop (Intel manual wrong)
5802 }
5803 
5804 void Assembler::ffree(int i) {
5805   emit_farith(0xDD, 0xC0, i);
5806 }
5807 
5808 void Assembler::fild_d(Address adr) {
5809   InstructionMark im(this);
5810   emit_int8((unsigned char)0xDF);
5811   emit_operand32(rbp, adr);
5812 }
5813 
5814 void Assembler::fild_s(Address adr) {
5815   InstructionMark im(this);
5816   emit_int8((unsigned char)0xDB);
5817   emit_operand32(rax, adr);
5818 }
5819 
5820 void Assembler::fincstp() {
5821   emit_int8((unsigned char)0xD9);
5822   emit_int8((unsigned char)0xF7);
5823 }
5824 
5825 void Assembler::finit() {
5826   emit_int8((unsigned char)0x9B);
5827   emit_int8((unsigned char)0xDB);
5828   emit_int8((unsigned char)0xE3);
5829 }
5830 
5831 void Assembler::fist_s(Address adr) {
5832   InstructionMark im(this);
5833   emit_int8((unsigned char)0xDB);
5834   emit_operand32(rdx, adr);
5835 }
5836 
5837 void Assembler::fistp_d(Address adr) {
5838   InstructionMark im(this);
5839   emit_int8((unsigned char)0xDF);
5840   emit_operand32(rdi, adr);
5841 }
5842 
5843 void Assembler::fistp_s(Address adr) {
5844   InstructionMark im(this);
5845   emit_int8((unsigned char)0xDB);
5846   emit_operand32(rbx, adr);
5847 }
5848 
5849 void Assembler::fld1() {
5850   emit_int8((unsigned char)0xD9);
5851   emit_int8((unsigned char)0xE8);
5852 }
5853 
5854 void Assembler::fld_d(Address adr) {
5855   InstructionMark im(this);
5856   emit_int8((unsigned char)0xDD);
5857   emit_operand32(rax, adr);
5858 }
5859 
5860 void Assembler::fld_s(Address adr) {
5861   InstructionMark im(this);
5862   emit_int8((unsigned char)0xD9);
5863   emit_operand32(rax, adr);
5864 }
5865 
5866 
5867 void Assembler::fld_s(int index) {
5868   emit_farith(0xD9, 0xC0, index);
5869 }
5870 
5871 void Assembler::fld_x(Address adr) {
5872   InstructionMark im(this);
5873   emit_int8((unsigned char)0xDB);
5874   emit_operand32(rbp, adr);
5875 }
5876 
5877 void Assembler::fldcw(Address src) {
5878   InstructionMark im(this);
5879   emit_int8((unsigned char)0xD9);
5880   emit_operand32(rbp, src);
5881 }
5882 
5883 void Assembler::fldenv(Address src) {
5884   InstructionMark im(this);
5885   emit_int8((unsigned char)0xD9);
5886   emit_operand32(rsp, src);
5887 }
5888 
5889 void Assembler::fldlg2() {
5890   emit_int8((unsigned char)0xD9);
5891   emit_int8((unsigned char)0xEC);
5892 }
5893 
5894 void Assembler::fldln2() {
5895   emit_int8((unsigned char)0xD9);
5896   emit_int8((unsigned char)0xED);
5897 }
5898 
5899 void Assembler::fldz() {
5900   emit_int8((unsigned char)0xD9);
5901   emit_int8((unsigned char)0xEE);
5902 }
5903 
5904 void Assembler::flog() {
5905   fldln2();
5906   fxch();
5907   fyl2x();
5908 }
5909 
5910 void Assembler::flog10() {
5911   fldlg2();
5912   fxch();
5913   fyl2x();
5914 }
5915 
5916 void Assembler::fmul(int i) {
5917   emit_farith(0xD8, 0xC8, i);
5918 }
5919 
5920 void Assembler::fmul_d(Address src) {
5921   InstructionMark im(this);
5922   emit_int8((unsigned char)0xDC);
5923   emit_operand32(rcx, src);
5924 }
5925 
5926 void Assembler::fmul_s(Address src) {
5927   InstructionMark im(this);
5928   emit_int8((unsigned char)0xD8);
5929   emit_operand32(rcx, src);
5930 }
5931 
5932 void Assembler::fmula(int i) {
5933   emit_farith(0xDC, 0xC8, i);
5934 }
5935 
5936 void Assembler::fmulp(int i) {
5937   emit_farith(0xDE, 0xC8, i);
5938 }
5939 
5940 void Assembler::fnsave(Address dst) {
5941   InstructionMark im(this);
5942   emit_int8((unsigned char)0xDD);
5943   emit_operand32(rsi, dst);
5944 }
5945 
5946 void Assembler::fnstcw(Address src) {
5947   InstructionMark im(this);
5948   emit_int8((unsigned char)0x9B);
5949   emit_int8((unsigned char)0xD9);
5950   emit_operand32(rdi, src);
5951 }
5952 
5953 void Assembler::fnstsw_ax() {
5954   emit_int8((unsigned char)0xDF);
5955   emit_int8((unsigned char)0xE0);
5956 }
5957 
5958 void Assembler::fprem() {
5959   emit_int8((unsigned char)0xD9);
5960   emit_int8((unsigned char)0xF8);
5961 }
5962 
5963 void Assembler::fprem1() {
5964   emit_int8((unsigned char)0xD9);
5965   emit_int8((unsigned char)0xF5);
5966 }
5967 
5968 void Assembler::frstor(Address src) {
5969   InstructionMark im(this);
5970   emit_int8((unsigned char)0xDD);
5971   emit_operand32(rsp, src);
5972 }
5973 
5974 void Assembler::fsin() {
5975   emit_int8((unsigned char)0xD9);
5976   emit_int8((unsigned char)0xFE);
5977 }
5978 
5979 void Assembler::fsqrt() {
5980   emit_int8((unsigned char)0xD9);
5981   emit_int8((unsigned char)0xFA);
5982 }
5983 
5984 void Assembler::fst_d(Address adr) {
5985   InstructionMark im(this);
5986   emit_int8((unsigned char)0xDD);
5987   emit_operand32(rdx, adr);
5988 }
5989 
5990 void Assembler::fst_s(Address adr) {
5991   InstructionMark im(this);
5992   emit_int8((unsigned char)0xD9);
5993   emit_operand32(rdx, adr);
5994 }
5995 
5996 void Assembler::fstp_d(Address adr) {
5997   InstructionMark im(this);
5998   emit_int8((unsigned char)0xDD);
5999   emit_operand32(rbx, adr);
6000 }
6001 
6002 void Assembler::fstp_d(int index) {
6003   emit_farith(0xDD, 0xD8, index);
6004 }
6005 
6006 void Assembler::fstp_s(Address adr) {
6007   InstructionMark im(this);
6008   emit_int8((unsigned char)0xD9);
6009   emit_operand32(rbx, adr);
6010 }
6011 
6012 void Assembler::fstp_x(Address adr) {
6013   InstructionMark im(this);
6014   emit_int8((unsigned char)0xDB);
6015   emit_operand32(rdi, adr);
6016 }
6017 
6018 void Assembler::fsub(int i) {
6019   emit_farith(0xD8, 0xE0, i);
6020 }
6021 
6022 void Assembler::fsub_d(Address src) {
6023   InstructionMark im(this);
6024   emit_int8((unsigned char)0xDC);
6025   emit_operand32(rsp, src);
6026 }
6027 
6028 void Assembler::fsub_s(Address src) {
6029   InstructionMark im(this);
6030   emit_int8((unsigned char)0xD8);
6031   emit_operand32(rsp, src);
6032 }
6033 
6034 void Assembler::fsuba(int i) {
6035   emit_farith(0xDC, 0xE8, i);
6036 }
6037 
6038 void Assembler::fsubp(int i) {
6039   emit_farith(0xDE, 0xE8, i);                    // ST(0) <- ST(0) - ST(1) and pop (Intel manual wrong)
6040 }
6041 
6042 void Assembler::fsubr(int i) {
6043   emit_farith(0xD8, 0xE8, i);
6044 }
6045 
6046 void Assembler::fsubr_d(Address src) {
6047   InstructionMark im(this);
6048   emit_int8((unsigned char)0xDC);
6049   emit_operand32(rbp, src);
6050 }
6051 
6052 void Assembler::fsubr_s(Address src) {
6053   InstructionMark im(this);
6054   emit_int8((unsigned char)0xD8);
6055   emit_operand32(rbp, src);
6056 }
6057 
6058 void Assembler::fsubra(int i) {
6059   emit_farith(0xDC, 0xE0, i);
6060 }
6061 
6062 void Assembler::fsubrp(int i) {
6063   emit_farith(0xDE, 0xE0, i);                    // ST(0) <- ST(1) - ST(0) and pop (Intel manual wrong)
6064 }
6065 
6066 void Assembler::ftan() {
6067   emit_int8((unsigned char)0xD9);
6068   emit_int8((unsigned char)0xF2);
6069   emit_int8((unsigned char)0xDD);
6070   emit_int8((unsigned char)0xD8);
6071 }
6072 
6073 void Assembler::ftst() {
6074   emit_int8((unsigned char)0xD9);
6075   emit_int8((unsigned char)0xE4);
6076 }
6077 
6078 void Assembler::fucomi(int i) {
6079   // make sure the instruction is supported (introduced for P6, together with cmov)
6080   guarantee(VM_Version::supports_cmov(), "illegal instruction");
6081   emit_farith(0xDB, 0xE8, i);
6082 }
6083 
6084 void Assembler::fucomip(int i) {
6085   // make sure the instruction is supported (introduced for P6, together with cmov)
6086   guarantee(VM_Version::supports_cmov(), "illegal instruction");
6087   emit_farith(0xDF, 0xE8, i);
6088 }
6089 
6090 void Assembler::fwait() {
6091   emit_int8((unsigned char)0x9B);
6092 }
6093 
6094 void Assembler::fxch(int i) {
6095   emit_farith(0xD9, 0xC8, i);
6096 }
6097 
6098 void Assembler::fyl2x() {
6099   emit_int8((unsigned char)0xD9);
6100   emit_int8((unsigned char)0xF1);
6101 }
6102 
6103 void Assembler::frndint() {
6104   emit_int8((unsigned char)0xD9);
6105   emit_int8((unsigned char)0xFC);
6106 }
6107 
6108 void Assembler::f2xm1() {
6109   emit_int8((unsigned char)0xD9);
6110   emit_int8((unsigned char)0xF0);
6111 }
6112 
6113 void Assembler::fldl2e() {
6114   emit_int8((unsigned char)0xD9);
6115   emit_int8((unsigned char)0xEA);
6116 }
6117 
6118 // SSE SIMD prefix byte values corresponding to VexSimdPrefix encoding.
6119 static int simd_pre[4] = { 0, 0x66, 0xF3, 0xF2 };
6120 // SSE opcode second byte values (first is 0x0F) corresponding to VexOpcode encoding.
6121 static int simd_opc[4] = { 0,    0, 0x38, 0x3A };
6122 
6123 // Generate SSE legacy REX prefix and SIMD opcode based on VEX encoding.
6124 void Assembler::rex_prefix(Address adr, XMMRegister xreg, VexSimdPrefix pre, VexOpcode opc, bool rex_w) {
6125   if (pre > 0) {
6126     emit_int8(simd_pre[pre]);
6127   }
6128   if (rex_w) {
6129     prefixq(adr, xreg);
6130   } else {
6131     prefix(adr, xreg);
6132   }
6133   if (opc > 0) {
6134     emit_int8(0x0F);
6135     int opc2 = simd_opc[opc];
6136     if (opc2 > 0) {
6137       emit_int8(opc2);
6138     }
6139   }
6140 }
6141 
6142 int Assembler::rex_prefix_and_encode(int dst_enc, int src_enc, VexSimdPrefix pre, VexOpcode opc, bool rex_w) {
6143   if (pre > 0) {
6144     emit_int8(simd_pre[pre]);
6145   }
6146   int encode = (rex_w) ? prefixq_and_encode(dst_enc, src_enc) : prefix_and_encode(dst_enc, src_enc);
6147   if (opc > 0) {
6148     emit_int8(0x0F);
6149     int opc2 = simd_opc[opc];
6150     if (opc2 > 0) {
6151       emit_int8(opc2);
6152     }
6153   }
6154   return encode;
6155 }
6156 
6157 
6158 void Assembler::vex_prefix(bool vex_r, bool vex_b, bool vex_x, int nds_enc, VexSimdPrefix pre, VexOpcode opc) {
6159   int vector_len = _attributes->get_vector_len();
6160   bool vex_w = _attributes->is_rex_vex_w();
6161   if (vex_b || vex_x || vex_w || (opc == VEX_OPCODE_0F_38) || (opc == VEX_OPCODE_0F_3A)) {
6162     prefix(VEX_3bytes);
6163 
6164     int byte1 = (vex_r ? VEX_R : 0) | (vex_x ? VEX_X : 0) | (vex_b ? VEX_B : 0);
6165     byte1 = (~byte1) & 0xE0;
6166     byte1 |= opc;
6167     emit_int8(byte1);
6168 
6169     int byte2 = ((~nds_enc) & 0xf) << 3;
6170     byte2 |= (vex_w ? VEX_W : 0) | ((vector_len > 0) ? 4 : 0) | pre;
6171     emit_int8(byte2);
6172   } else {
6173     prefix(VEX_2bytes);
6174 
6175     int byte1 = vex_r ? VEX_R : 0;
6176     byte1 = (~byte1) & 0x80;
6177     byte1 |= ((~nds_enc) & 0xf) << 3;
6178     byte1 |= ((vector_len > 0 ) ? 4 : 0) | pre;
6179     emit_int8(byte1);
6180   }
6181 }
6182 
6183 // This is a 4 byte encoding
6184 void Assembler::evex_prefix(bool vex_r, bool vex_b, bool vex_x, bool evex_r, bool evex_v, int nds_enc, VexSimdPrefix pre, VexOpcode opc){
6185   // EVEX 0x62 prefix
6186   prefix(EVEX_4bytes);
6187   bool vex_w = _attributes->is_rex_vex_w();
6188   int evex_encoding = (vex_w ? VEX_W : 0);
6189   // EVEX.b is not currently used for broadcast of single element or data rounding modes
6190   _attributes->set_evex_encoding(evex_encoding);
6191 
6192   // P0: byte 2, initialized to RXBR`00mm
6193   // instead of not'd
6194   int byte2 = (vex_r ? VEX_R : 0) | (vex_x ? VEX_X : 0) | (vex_b ? VEX_B : 0) | (evex_r ? EVEX_Rb : 0);
6195   byte2 = (~byte2) & 0xF0;
6196   // confine opc opcode extensions in mm bits to lower two bits
6197   // of form {0F, 0F_38, 0F_3A}
6198   byte2 |= opc;
6199   emit_int8(byte2);
6200 
6201   // P1: byte 3 as Wvvvv1pp
6202   int byte3 = ((~nds_enc) & 0xf) << 3;
6203   // p[10] is always 1
6204   byte3 |= EVEX_F;
6205   byte3 |= (vex_w & 1) << 7;
6206   // confine pre opcode extensions in pp bits to lower two bits
6207   // of form {66, F3, F2}
6208   byte3 |= pre;
6209   emit_int8(byte3);
6210 
6211   // P2: byte 4 as zL'Lbv'aaa
6212   int byte4 = (_attributes->is_no_reg_mask()) ? 0 : 1; // kregs are implemented in the low 3 bits as aaa (hard code k1, it will be initialized for now)
6213   // EVEX.v` for extending EVEX.vvvv or VIDX
6214   byte4 |= (evex_v ? 0: EVEX_V);
6215   // third EXEC.b for broadcast actions
6216   byte4 |= (_attributes->is_extended_context() ? EVEX_Rb : 0);
6217   // fourth EVEX.L'L for vector length : 0 is 128, 1 is 256, 2 is 512, currently we do not support 1024
6218   byte4 |= ((_attributes->get_vector_len())& 0x3) << 5;
6219   // last is EVEX.z for zero/merge actions
6220   byte4 |= (_attributes->is_clear_context() ? EVEX_Z : 0);
6221   emit_int8(byte4);
6222 }
6223 
6224 void Assembler::vex_prefix(Address adr, int nds_enc, int xreg_enc, VexSimdPrefix pre, VexOpcode opc, InstructionAttr *attributes) {
6225   bool vex_r = ((xreg_enc & 8) == 8) ? 1 : 0;
6226   bool vex_b = adr.base_needs_rex();
6227   bool vex_x = adr.index_needs_rex();
6228   set_attributes(attributes);
6229   attributes->set_current_assembler(this);
6230 
6231   // if vector length is turned off, revert to AVX for vectors smaller than 512-bit
6232   if ((UseAVX > 2) && _legacy_mode_vl && attributes->uses_vl()) {
6233     switch (attributes->get_vector_len()) {
6234     case AVX_128bit:
6235     case AVX_256bit:
6236       attributes->set_is_legacy_mode();
6237       break;
6238     }
6239   }
6240 
6241   if ((UseAVX > 2) && !attributes->is_legacy_mode())
6242   {
6243     bool evex_r = (xreg_enc >= 16);
6244     bool evex_v = (nds_enc >= 16);
6245     attributes->set_is_evex_instruction();
6246     evex_prefix(vex_r, vex_b, vex_x, evex_r, evex_v, nds_enc, pre, opc);
6247   } else {
6248     vex_prefix(vex_r, vex_b, vex_x, nds_enc, pre, opc);
6249   }
6250 }
6251 
6252 int Assembler::vex_prefix_and_encode(int dst_enc, int nds_enc, int src_enc, VexSimdPrefix pre, VexOpcode opc, InstructionAttr *attributes) {
6253   bool vex_r = ((dst_enc & 8) == 8) ? 1 : 0;
6254   bool vex_b = ((src_enc & 8) == 8) ? 1 : 0;
6255   bool vex_x = false;
6256   set_attributes(attributes);
6257   attributes->set_current_assembler(this);
6258 
6259   // if vector length is turned off, revert to AVX for vectors smaller than 512-bit
6260   if ((UseAVX > 2) && _legacy_mode_vl && attributes->uses_vl()) {
6261     switch (attributes->get_vector_len()) {
6262     case AVX_128bit:
6263     case AVX_256bit:
6264       if ((dst_enc >= 16) | (nds_enc >= 16) | (src_enc >= 16)) {
6265         // up propagate arithmetic instructions to meet RA requirements
6266         attributes->set_vector_len(AVX_512bit);
6267       } else {
6268         attributes->set_is_legacy_mode();
6269       }
6270       break;
6271     }
6272   }
6273 
6274   if ((UseAVX > 2) && !attributes->is_legacy_mode())
6275   {
6276     bool evex_r = (dst_enc >= 16);
6277     bool evex_v = (nds_enc >= 16);
6278     // can use vex_x as bank extender on rm encoding
6279     vex_x = (src_enc >= 16);
6280     attributes->set_is_evex_instruction();
6281     evex_prefix(vex_r, vex_b, vex_x, evex_r, evex_v, nds_enc, pre, opc);
6282   } else {
6283     vex_prefix(vex_r, vex_b, vex_x, nds_enc, pre, opc);
6284   }
6285 
6286   // return modrm byte components for operands
6287   return (((dst_enc & 7) << 3) | (src_enc & 7));
6288 }
6289 
6290 
6291 void Assembler::simd_prefix(XMMRegister xreg, XMMRegister nds, Address adr, VexSimdPrefix pre,
6292                             VexOpcode opc, InstructionAttr *attributes) {
6293   if (UseAVX > 0) {
6294     int xreg_enc = xreg->encoding();
6295     int nds_enc = nds->is_valid() ? nds->encoding() : 0;
6296     vex_prefix(adr, nds_enc, xreg_enc, pre, opc, attributes);
6297   } else {
6298     assert((nds == xreg) || (nds == xnoreg), "wrong sse encoding");
6299     rex_prefix(adr, xreg, pre, opc, attributes->is_rex_vex_w());
6300   }
6301 }
6302 
6303 int Assembler::simd_prefix_and_encode(XMMRegister dst, XMMRegister nds, XMMRegister src, VexSimdPrefix pre,
6304                                       VexOpcode opc, InstructionAttr *attributes) {
6305   int dst_enc = dst->encoding();
6306   int src_enc = src->encoding();
6307   if (UseAVX > 0) {
6308     int nds_enc = nds->is_valid() ? nds->encoding() : 0;
6309     return vex_prefix_and_encode(dst_enc, nds_enc, src_enc, pre, opc, attributes);
6310   } else {
6311     assert((nds == dst) || (nds == src) || (nds == xnoreg), "wrong sse encoding");
6312     return rex_prefix_and_encode(dst_enc, src_enc, pre, opc, attributes->is_rex_vex_w());
6313   }
6314 }
6315 
6316 int Assembler::kreg_prefix_and_encode(KRegister dst, KRegister nds, KRegister src, VexSimdPrefix pre,
6317                                       VexOpcode opc, InstructionAttr *attributes) {
6318   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
6319   return vex_prefix_and_encode(dst->encoding(), nds_enc, src->encoding(), pre, opc, attributes);
6320 }
6321 
6322 int Assembler::kreg_prefix_and_encode(KRegister dst, KRegister nds, Register src, VexSimdPrefix pre,
6323                                       VexOpcode opc, InstructionAttr *attributes) {
6324   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
6325   return vex_prefix_and_encode(dst->encoding(), nds_enc, src->encoding(), pre, opc, attributes);
6326 }
6327 
6328 void Assembler::cmppd(XMMRegister dst, XMMRegister nds, XMMRegister src, int cop, int vector_len) {
6329   assert(VM_Version::supports_avx(), "");
6330   assert(!VM_Version::supports_evex(), "");
6331   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
6332   int encode = simd_prefix_and_encode(dst, nds, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6333   emit_int8((unsigned char)0xC2);
6334   emit_int8((unsigned char)(0xC0 | encode));
6335   emit_int8((unsigned char)(0xF & cop));
6336 }
6337 
6338 void Assembler::vpblendd(XMMRegister dst, XMMRegister nds, XMMRegister src1, XMMRegister src2, int vector_len) {
6339   assert(VM_Version::supports_avx(), "");
6340   assert(!VM_Version::supports_evex(), "");
6341   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
6342   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
6343   int encode = vex_prefix_and_encode(dst->encoding(), nds_enc, src1->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6344   emit_int8((unsigned char)0x4B);
6345   emit_int8((unsigned char)(0xC0 | encode));
6346   int src2_enc = src2->encoding();
6347   emit_int8((unsigned char)(0xF0 & src2_enc<<4));
6348 }
6349 
6350 
6351 #ifndef _LP64
6352 
6353 void Assembler::incl(Register dst) {
6354   // Don't use it directly. Use MacroAssembler::incrementl() instead.
6355   emit_int8(0x40 | dst->encoding());
6356 }
6357 
6358 void Assembler::lea(Register dst, Address src) {
6359   leal(dst, src);
6360 }
6361 
6362 void Assembler::mov_literal32(Address dst, int32_t imm32, RelocationHolder const& rspec) {
6363   InstructionMark im(this);
6364   emit_int8((unsigned char)0xC7);
6365   emit_operand(rax, dst);
6366   emit_data((int)imm32, rspec, 0);
6367 }
6368 
6369 void Assembler::mov_literal32(Register dst, int32_t imm32, RelocationHolder const& rspec) {
6370   InstructionMark im(this);
6371   int encode = prefix_and_encode(dst->encoding());
6372   emit_int8((unsigned char)(0xB8 | encode));
6373   emit_data((int)imm32, rspec, 0);
6374 }
6375 
6376 void Assembler::popa() { // 32bit
6377   emit_int8(0x61);
6378 }
6379 
6380 void Assembler::push_literal32(int32_t imm32, RelocationHolder const& rspec) {
6381   InstructionMark im(this);
6382   emit_int8(0x68);
6383   emit_data(imm32, rspec, 0);
6384 }
6385 
6386 void Assembler::pusha() { // 32bit
6387   emit_int8(0x60);
6388 }
6389 
6390 void Assembler::set_byte_if_not_zero(Register dst) {
6391   emit_int8(0x0F);
6392   emit_int8((unsigned char)0x95);
6393   emit_int8((unsigned char)(0xE0 | dst->encoding()));
6394 }
6395 
6396 void Assembler::shldl(Register dst, Register src) {
6397   emit_int8(0x0F);
6398   emit_int8((unsigned char)0xA5);
6399   emit_int8((unsigned char)(0xC0 | src->encoding() << 3 | dst->encoding()));
6400 }
6401 
6402 // 0F A4 / r ib
6403 void Assembler::shldl(Register dst, Register src, int8_t imm8) {
6404   emit_int8(0x0F);
6405   emit_int8((unsigned char)0xA4);
6406   emit_int8((unsigned char)(0xC0 | src->encoding() << 3 | dst->encoding()));
6407   emit_int8(imm8);
6408 }
6409 
6410 void Assembler::shrdl(Register dst, Register src) {
6411   emit_int8(0x0F);
6412   emit_int8((unsigned char)0xAD);
6413   emit_int8((unsigned char)(0xC0 | src->encoding() << 3 | dst->encoding()));
6414 }
6415 
6416 #else // LP64
6417 
6418 void Assembler::set_byte_if_not_zero(Register dst) {
6419   int enc = prefix_and_encode(dst->encoding(), true);
6420   emit_int8(0x0F);
6421   emit_int8((unsigned char)0x95);
6422   emit_int8((unsigned char)(0xE0 | enc));
6423 }
6424 
6425 // 64bit only pieces of the assembler
6426 // This should only be used by 64bit instructions that can use rip-relative
6427 // it cannot be used by instructions that want an immediate value.
6428 
6429 bool Assembler::reachable(AddressLiteral adr) {
6430   int64_t disp;
6431   // None will force a 64bit literal to the code stream. Likely a placeholder
6432   // for something that will be patched later and we need to certain it will
6433   // always be reachable.
6434   if (adr.reloc() == relocInfo::none) {
6435     return false;
6436   }
6437   if (adr.reloc() == relocInfo::internal_word_type) {
6438     // This should be rip relative and easily reachable.
6439     return true;
6440   }
6441   if (adr.reloc() == relocInfo::virtual_call_type ||
6442       adr.reloc() == relocInfo::opt_virtual_call_type ||
6443       adr.reloc() == relocInfo::static_call_type ||
6444       adr.reloc() == relocInfo::static_stub_type ) {
6445     // This should be rip relative within the code cache and easily
6446     // reachable until we get huge code caches. (At which point
6447     // ic code is going to have issues).
6448     return true;
6449   }
6450   if (adr.reloc() != relocInfo::external_word_type &&
6451       adr.reloc() != relocInfo::poll_return_type &&  // these are really external_word but need special
6452       adr.reloc() != relocInfo::poll_type &&         // relocs to identify them
6453       adr.reloc() != relocInfo::runtime_call_type ) {
6454     return false;
6455   }
6456 
6457   // Stress the correction code
6458   if (ForceUnreachable) {
6459     // Must be runtimecall reloc, see if it is in the codecache
6460     // Flipping stuff in the codecache to be unreachable causes issues
6461     // with things like inline caches where the additional instructions
6462     // are not handled.
6463     if (CodeCache::find_blob(adr._target) == NULL) {
6464       return false;
6465     }
6466   }
6467   // For external_word_type/runtime_call_type if it is reachable from where we
6468   // are now (possibly a temp buffer) and where we might end up
6469   // anywhere in the codeCache then we are always reachable.
6470   // This would have to change if we ever save/restore shared code
6471   // to be more pessimistic.
6472   disp = (int64_t)adr._target - ((int64_t)CodeCache::low_bound() + sizeof(int));
6473   if (!is_simm32(disp)) return false;
6474   disp = (int64_t)adr._target - ((int64_t)CodeCache::high_bound() + sizeof(int));
6475   if (!is_simm32(disp)) return false;
6476 
6477   disp = (int64_t)adr._target - ((int64_t)pc() + sizeof(int));
6478 
6479   // Because rip relative is a disp + address_of_next_instruction and we
6480   // don't know the value of address_of_next_instruction we apply a fudge factor
6481   // to make sure we will be ok no matter the size of the instruction we get placed into.
6482   // We don't have to fudge the checks above here because they are already worst case.
6483 
6484   // 12 == override/rex byte, opcode byte, rm byte, sib byte, a 4-byte disp , 4-byte literal
6485   // + 4 because better safe than sorry.
6486   const int fudge = 12 + 4;
6487   if (disp < 0) {
6488     disp -= fudge;
6489   } else {
6490     disp += fudge;
6491   }
6492   return is_simm32(disp);
6493 }
6494 
6495 // Check if the polling page is not reachable from the code cache using rip-relative
6496 // addressing.
6497 bool Assembler::is_polling_page_far() {
6498   intptr_t addr = (intptr_t)os::get_polling_page();
6499   return ForceUnreachable ||
6500          !is_simm32(addr - (intptr_t)CodeCache::low_bound()) ||
6501          !is_simm32(addr - (intptr_t)CodeCache::high_bound());
6502 }
6503 
6504 void Assembler::emit_data64(jlong data,
6505                             relocInfo::relocType rtype,
6506                             int format) {
6507   if (rtype == relocInfo::none) {
6508     emit_int64(data);
6509   } else {
6510     emit_data64(data, Relocation::spec_simple(rtype), format);
6511   }
6512 }
6513 
6514 void Assembler::emit_data64(jlong data,
6515                             RelocationHolder const& rspec,
6516                             int format) {
6517   assert(imm_operand == 0, "default format must be immediate in this file");
6518   assert(imm_operand == format, "must be immediate");
6519   assert(inst_mark() != NULL, "must be inside InstructionMark");
6520   // Do not use AbstractAssembler::relocate, which is not intended for
6521   // embedded words.  Instead, relocate to the enclosing instruction.
6522   code_section()->relocate(inst_mark(), rspec, format);
6523 #ifdef ASSERT
6524   check_relocation(rspec, format);
6525 #endif
6526   emit_int64(data);
6527 }
6528 
6529 int Assembler::prefix_and_encode(int reg_enc, bool byteinst) {
6530   if (reg_enc >= 8) {
6531     prefix(REX_B);
6532     reg_enc -= 8;
6533   } else if (byteinst && reg_enc >= 4) {
6534     prefix(REX);
6535   }
6536   return reg_enc;
6537 }
6538 
6539 int Assembler::prefixq_and_encode(int reg_enc) {
6540   if (reg_enc < 8) {
6541     prefix(REX_W);
6542   } else {
6543     prefix(REX_WB);
6544     reg_enc -= 8;
6545   }
6546   return reg_enc;
6547 }
6548 
6549 int Assembler::prefix_and_encode(int dst_enc, bool dst_is_byte, int src_enc, bool src_is_byte) {
6550   if (dst_enc < 8) {
6551     if (src_enc >= 8) {
6552       prefix(REX_B);
6553       src_enc -= 8;
6554     } else if ((src_is_byte && src_enc >= 4) || (dst_is_byte && dst_enc >= 4)) {
6555       prefix(REX);
6556     }
6557   } else {
6558     if (src_enc < 8) {
6559       prefix(REX_R);
6560     } else {
6561       prefix(REX_RB);
6562       src_enc -= 8;
6563     }
6564     dst_enc -= 8;
6565   }
6566   return dst_enc << 3 | src_enc;
6567 }
6568 
6569 int Assembler::prefixq_and_encode(int dst_enc, int src_enc) {
6570   if (dst_enc < 8) {
6571     if (src_enc < 8) {
6572       prefix(REX_W);
6573     } else {
6574       prefix(REX_WB);
6575       src_enc -= 8;
6576     }
6577   } else {
6578     if (src_enc < 8) {
6579       prefix(REX_WR);
6580     } else {
6581       prefix(REX_WRB);
6582       src_enc -= 8;
6583     }
6584     dst_enc -= 8;
6585   }
6586   return dst_enc << 3 | src_enc;
6587 }
6588 
6589 void Assembler::prefix(Register reg) {
6590   if (reg->encoding() >= 8) {
6591     prefix(REX_B);
6592   }
6593 }
6594 
6595 void Assembler::prefix(Register dst, Register src, Prefix p) {
6596   if (src->encoding() >= 8) {
6597     p = (Prefix)(p | REX_B);
6598   }
6599   if (dst->encoding() >= 8) {
6600     p = (Prefix)( p | REX_R);
6601   }
6602   if (p != Prefix_EMPTY) {
6603     // do not generate an empty prefix
6604     prefix(p);
6605   }
6606 }
6607 
6608 void Assembler::prefix(Register dst, Address adr, Prefix p) {
6609   if (adr.base_needs_rex()) {
6610     if (adr.index_needs_rex()) {
6611       assert(false, "prefix(Register dst, Address adr, Prefix p) does not support handling of an X");
6612     } else {
6613       prefix(REX_B);
6614     }
6615   } else {
6616     if (adr.index_needs_rex()) {
6617       assert(false, "prefix(Register dst, Address adr, Prefix p) does not support handling of an X");
6618     }
6619   }
6620   if (dst->encoding() >= 8) {
6621     p = (Prefix)(p | REX_R);
6622   }
6623   if (p != Prefix_EMPTY) {
6624     // do not generate an empty prefix
6625     prefix(p);
6626   }
6627 }
6628 
6629 void Assembler::prefix(Address adr) {
6630   if (adr.base_needs_rex()) {
6631     if (adr.index_needs_rex()) {
6632       prefix(REX_XB);
6633     } else {
6634       prefix(REX_B);
6635     }
6636   } else {
6637     if (adr.index_needs_rex()) {
6638       prefix(REX_X);
6639     }
6640   }
6641 }
6642 
6643 void Assembler::prefixq(Address adr) {
6644   if (adr.base_needs_rex()) {
6645     if (adr.index_needs_rex()) {
6646       prefix(REX_WXB);
6647     } else {
6648       prefix(REX_WB);
6649     }
6650   } else {
6651     if (adr.index_needs_rex()) {
6652       prefix(REX_WX);
6653     } else {
6654       prefix(REX_W);
6655     }
6656   }
6657 }
6658 
6659 
6660 void Assembler::prefix(Address adr, Register reg, bool byteinst) {
6661   if (reg->encoding() < 8) {
6662     if (adr.base_needs_rex()) {
6663       if (adr.index_needs_rex()) {
6664         prefix(REX_XB);
6665       } else {
6666         prefix(REX_B);
6667       }
6668     } else {
6669       if (adr.index_needs_rex()) {
6670         prefix(REX_X);
6671       } else if (byteinst && reg->encoding() >= 4 ) {
6672         prefix(REX);
6673       }
6674     }
6675   } else {
6676     if (adr.base_needs_rex()) {
6677       if (adr.index_needs_rex()) {
6678         prefix(REX_RXB);
6679       } else {
6680         prefix(REX_RB);
6681       }
6682     } else {
6683       if (adr.index_needs_rex()) {
6684         prefix(REX_RX);
6685       } else {
6686         prefix(REX_R);
6687       }
6688     }
6689   }
6690 }
6691 
6692 void Assembler::prefixq(Address adr, Register src) {
6693   if (src->encoding() < 8) {
6694     if (adr.base_needs_rex()) {
6695       if (adr.index_needs_rex()) {
6696         prefix(REX_WXB);
6697       } else {
6698         prefix(REX_WB);
6699       }
6700     } else {
6701       if (adr.index_needs_rex()) {
6702         prefix(REX_WX);
6703       } else {
6704         prefix(REX_W);
6705       }
6706     }
6707   } else {
6708     if (adr.base_needs_rex()) {
6709       if (adr.index_needs_rex()) {
6710         prefix(REX_WRXB);
6711       } else {
6712         prefix(REX_WRB);
6713       }
6714     } else {
6715       if (adr.index_needs_rex()) {
6716         prefix(REX_WRX);
6717       } else {
6718         prefix(REX_WR);
6719       }
6720     }
6721   }
6722 }
6723 
6724 void Assembler::prefix(Address adr, XMMRegister reg) {
6725   if (reg->encoding() < 8) {
6726     if (adr.base_needs_rex()) {
6727       if (adr.index_needs_rex()) {
6728         prefix(REX_XB);
6729       } else {
6730         prefix(REX_B);
6731       }
6732     } else {
6733       if (adr.index_needs_rex()) {
6734         prefix(REX_X);
6735       }
6736     }
6737   } else {
6738     if (adr.base_needs_rex()) {
6739       if (adr.index_needs_rex()) {
6740         prefix(REX_RXB);
6741       } else {
6742         prefix(REX_RB);
6743       }
6744     } else {
6745       if (adr.index_needs_rex()) {
6746         prefix(REX_RX);
6747       } else {
6748         prefix(REX_R);
6749       }
6750     }
6751   }
6752 }
6753 
6754 void Assembler::prefixq(Address adr, XMMRegister src) {
6755   if (src->encoding() < 8) {
6756     if (adr.base_needs_rex()) {
6757       if (adr.index_needs_rex()) {
6758         prefix(REX_WXB);
6759       } else {
6760         prefix(REX_WB);
6761       }
6762     } else {
6763       if (adr.index_needs_rex()) {
6764         prefix(REX_WX);
6765       } else {
6766         prefix(REX_W);
6767       }
6768     }
6769   } else {
6770     if (adr.base_needs_rex()) {
6771       if (adr.index_needs_rex()) {
6772         prefix(REX_WRXB);
6773       } else {
6774         prefix(REX_WRB);
6775       }
6776     } else {
6777       if (adr.index_needs_rex()) {
6778         prefix(REX_WRX);
6779       } else {
6780         prefix(REX_WR);
6781       }
6782     }
6783   }
6784 }
6785 
6786 void Assembler::adcq(Register dst, int32_t imm32) {
6787   (void) prefixq_and_encode(dst->encoding());
6788   emit_arith(0x81, 0xD0, dst, imm32);
6789 }
6790 
6791 void Assembler::adcq(Register dst, Address src) {
6792   InstructionMark im(this);
6793   prefixq(src, dst);
6794   emit_int8(0x13);
6795   emit_operand(dst, src);
6796 }
6797 
6798 void Assembler::adcq(Register dst, Register src) {
6799   (void) prefixq_and_encode(dst->encoding(), src->encoding());
6800   emit_arith(0x13, 0xC0, dst, src);
6801 }
6802 
6803 void Assembler::addq(Address dst, int32_t imm32) {
6804   InstructionMark im(this);
6805   prefixq(dst);
6806   emit_arith_operand(0x81, rax, dst,imm32);
6807 }
6808 
6809 void Assembler::addq(Address dst, Register src) {
6810   InstructionMark im(this);
6811   prefixq(dst, src);
6812   emit_int8(0x01);
6813   emit_operand(src, dst);
6814 }
6815 
6816 void Assembler::addq(Register dst, int32_t imm32) {
6817   (void) prefixq_and_encode(dst->encoding());
6818   emit_arith(0x81, 0xC0, dst, imm32);
6819 }
6820 
6821 void Assembler::addq(Register dst, Address src) {
6822   InstructionMark im(this);
6823   prefixq(src, dst);
6824   emit_int8(0x03);
6825   emit_operand(dst, src);
6826 }
6827 
6828 void Assembler::addq(Register dst, Register src) {
6829   (void) prefixq_and_encode(dst->encoding(), src->encoding());
6830   emit_arith(0x03, 0xC0, dst, src);
6831 }
6832 
6833 void Assembler::adcxq(Register dst, Register src) {
6834   //assert(VM_Version::supports_adx(), "adx instructions not supported");
6835   emit_int8((unsigned char)0x66);
6836   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
6837   emit_int8(0x0F);
6838   emit_int8(0x38);
6839   emit_int8((unsigned char)0xF6);
6840   emit_int8((unsigned char)(0xC0 | encode));
6841 }
6842 
6843 void Assembler::adoxq(Register dst, Register src) {
6844   //assert(VM_Version::supports_adx(), "adx instructions not supported");
6845   emit_int8((unsigned char)0xF3);
6846   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
6847   emit_int8(0x0F);
6848   emit_int8(0x38);
6849   emit_int8((unsigned char)0xF6);
6850   emit_int8((unsigned char)(0xC0 | encode));
6851 }
6852 
6853 void Assembler::andq(Address dst, int32_t imm32) {
6854   InstructionMark im(this);
6855   prefixq(dst);
6856   emit_int8((unsigned char)0x81);
6857   emit_operand(rsp, dst, 4);
6858   emit_int32(imm32);
6859 }
6860 
6861 void Assembler::andq(Register dst, int32_t imm32) {
6862   (void) prefixq_and_encode(dst->encoding());
6863   emit_arith(0x81, 0xE0, dst, imm32);
6864 }
6865 
6866 void Assembler::andq(Register dst, Address src) {
6867   InstructionMark im(this);
6868   prefixq(src, dst);
6869   emit_int8(0x23);
6870   emit_operand(dst, src);
6871 }
6872 
6873 void Assembler::andq(Register dst, Register src) {
6874   (void) prefixq_and_encode(dst->encoding(), src->encoding());
6875   emit_arith(0x23, 0xC0, dst, src);
6876 }
6877 
6878 void Assembler::andnq(Register dst, Register src1, Register src2) {
6879   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
6880   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
6881   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
6882   emit_int8((unsigned char)0xF2);
6883   emit_int8((unsigned char)(0xC0 | encode));
6884 }
6885 
6886 void Assembler::andnq(Register dst, Register src1, Address src2) {
6887   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
6888   InstructionMark im(this);
6889   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
6890   vex_prefix(src2, src1->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
6891   emit_int8((unsigned char)0xF2);
6892   emit_operand(dst, src2);
6893 }
6894 
6895 void Assembler::bsfq(Register dst, Register src) {
6896   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
6897   emit_int8(0x0F);
6898   emit_int8((unsigned char)0xBC);
6899   emit_int8((unsigned char)(0xC0 | encode));
6900 }
6901 
6902 void Assembler::bsrq(Register dst, Register src) {
6903   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
6904   emit_int8(0x0F);
6905   emit_int8((unsigned char)0xBD);
6906   emit_int8((unsigned char)(0xC0 | encode));
6907 }
6908 
6909 void Assembler::bswapq(Register reg) {
6910   int encode = prefixq_and_encode(reg->encoding());
6911   emit_int8(0x0F);
6912   emit_int8((unsigned char)(0xC8 | encode));
6913 }
6914 
6915 void Assembler::blsiq(Register dst, Register src) {
6916   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
6917   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
6918   int encode = vex_prefix_and_encode(rbx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
6919   emit_int8((unsigned char)0xF3);
6920   emit_int8((unsigned char)(0xC0 | encode));
6921 }
6922 
6923 void Assembler::blsiq(Register dst, Address src) {
6924   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
6925   InstructionMark im(this);
6926   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
6927   vex_prefix(src, dst->encoding(), rbx->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
6928   emit_int8((unsigned char)0xF3);
6929   emit_operand(rbx, src);
6930 }
6931 
6932 void Assembler::blsmskq(Register dst, Register src) {
6933   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
6934   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
6935   int encode = vex_prefix_and_encode(rdx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
6936   emit_int8((unsigned char)0xF3);
6937   emit_int8((unsigned char)(0xC0 | encode));
6938 }
6939 
6940 void Assembler::blsmskq(Register dst, Address src) {
6941   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
6942   InstructionMark im(this);
6943   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
6944   vex_prefix(src, dst->encoding(), rdx->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
6945   emit_int8((unsigned char)0xF3);
6946   emit_operand(rdx, src);
6947 }
6948 
6949 void Assembler::blsrq(Register dst, Register src) {
6950   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
6951   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
6952   int encode = vex_prefix_and_encode(rcx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
6953   emit_int8((unsigned char)0xF3);
6954   emit_int8((unsigned char)(0xC0 | encode));
6955 }
6956 
6957 void Assembler::blsrq(Register dst, Address src) {
6958   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
6959   InstructionMark im(this);
6960   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
6961   vex_prefix(src, dst->encoding(), rcx->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
6962   emit_int8((unsigned char)0xF3);
6963   emit_operand(rcx, src);
6964 }
6965 
6966 void Assembler::cdqq() {
6967   prefix(REX_W);
6968   emit_int8((unsigned char)0x99);
6969 }
6970 
6971 void Assembler::clflush(Address adr) {
6972   prefix(adr);
6973   emit_int8(0x0F);
6974   emit_int8((unsigned char)0xAE);
6975   emit_operand(rdi, adr);
6976 }
6977 
6978 void Assembler::cmovq(Condition cc, Register dst, Register src) {
6979   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
6980   emit_int8(0x0F);
6981   emit_int8(0x40 | cc);
6982   emit_int8((unsigned char)(0xC0 | encode));
6983 }
6984 
6985 void Assembler::cmovq(Condition cc, Register dst, Address src) {
6986   InstructionMark im(this);
6987   prefixq(src, dst);
6988   emit_int8(0x0F);
6989   emit_int8(0x40 | cc);
6990   emit_operand(dst, src);
6991 }
6992 
6993 void Assembler::cmpq(Address dst, int32_t imm32) {
6994   InstructionMark im(this);
6995   prefixq(dst);
6996   emit_int8((unsigned char)0x81);
6997   emit_operand(rdi, dst, 4);
6998   emit_int32(imm32);
6999 }
7000 
7001 void Assembler::cmpq(Register dst, int32_t imm32) {
7002   (void) prefixq_and_encode(dst->encoding());
7003   emit_arith(0x81, 0xF8, dst, imm32);
7004 }
7005 
7006 void Assembler::cmpq(Address dst, Register src) {
7007   InstructionMark im(this);
7008   prefixq(dst, src);
7009   emit_int8(0x3B);
7010   emit_operand(src, dst);
7011 }
7012 
7013 void Assembler::cmpq(Register dst, Register src) {
7014   (void) prefixq_and_encode(dst->encoding(), src->encoding());
7015   emit_arith(0x3B, 0xC0, dst, src);
7016 }
7017 
7018 void Assembler::cmpq(Register dst, Address  src) {
7019   InstructionMark im(this);
7020   prefixq(src, dst);
7021   emit_int8(0x3B);
7022   emit_operand(dst, src);
7023 }
7024 
7025 void Assembler::cmpxchgq(Register reg, Address adr) {
7026   InstructionMark im(this);
7027   prefixq(adr, reg);
7028   emit_int8(0x0F);
7029   emit_int8((unsigned char)0xB1);
7030   emit_operand(reg, adr);
7031 }
7032 
7033 void Assembler::cvtsi2sdq(XMMRegister dst, Register src) {
7034   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
7035   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
7036   int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
7037   emit_int8(0x2A);
7038   emit_int8((unsigned char)(0xC0 | encode));
7039 }
7040 
7041 void Assembler::cvtsi2sdq(XMMRegister dst, Address src) {
7042   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
7043   InstructionMark im(this);
7044   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
7045   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
7046   simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
7047   emit_int8(0x2A);
7048   emit_operand(dst, src);
7049 }
7050 
7051 void Assembler::cvtsi2ssq(XMMRegister dst, Address src) {
7052   NOT_LP64(assert(VM_Version::supports_sse(), ""));
7053   InstructionMark im(this);
7054   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
7055   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
7056   simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
7057   emit_int8(0x2A);
7058   emit_operand(dst, src);
7059 }
7060 
7061 void Assembler::cvttsd2siq(Register dst, XMMRegister src) {
7062   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
7063   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
7064   int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
7065   emit_int8(0x2C);
7066   emit_int8((unsigned char)(0xC0 | encode));
7067 }
7068 
7069 void Assembler::cvttss2siq(Register dst, XMMRegister src) {
7070   NOT_LP64(assert(VM_Version::supports_sse(), ""));
7071   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
7072   int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
7073   emit_int8(0x2C);
7074   emit_int8((unsigned char)(0xC0 | encode));
7075 }
7076 
7077 void Assembler::decl(Register dst) {
7078   // Don't use it directly. Use MacroAssembler::decrementl() instead.
7079   // Use two-byte form (one-byte form is a REX prefix in 64-bit mode)
7080   int encode = prefix_and_encode(dst->encoding());
7081   emit_int8((unsigned char)0xFF);
7082   emit_int8((unsigned char)(0xC8 | encode));
7083 }
7084 
7085 void Assembler::decq(Register dst) {
7086   // Don't use it directly. Use MacroAssembler::decrementq() instead.
7087   // Use two-byte form (one-byte from is a REX prefix in 64-bit mode)
7088   int encode = prefixq_and_encode(dst->encoding());
7089   emit_int8((unsigned char)0xFF);
7090   emit_int8(0xC8 | encode);
7091 }
7092 
7093 void Assembler::decq(Address dst) {
7094   // Don't use it directly. Use MacroAssembler::decrementq() instead.
7095   InstructionMark im(this);
7096   prefixq(dst);
7097   emit_int8((unsigned char)0xFF);
7098   emit_operand(rcx, dst);
7099 }
7100 
7101 void Assembler::fxrstor(Address src) {
7102   prefixq(src);
7103   emit_int8(0x0F);
7104   emit_int8((unsigned char)0xAE);
7105   emit_operand(as_Register(1), src);
7106 }
7107 
7108 void Assembler::xrstor(Address src) {
7109   prefixq(src);
7110   emit_int8(0x0F);
7111   emit_int8((unsigned char)0xAE);
7112   emit_operand(as_Register(5), src);
7113 }
7114 
7115 void Assembler::fxsave(Address dst) {
7116   prefixq(dst);
7117   emit_int8(0x0F);
7118   emit_int8((unsigned char)0xAE);
7119   emit_operand(as_Register(0), dst);
7120 }
7121 
7122 void Assembler::xsave(Address dst) {
7123   prefixq(dst);
7124   emit_int8(0x0F);
7125   emit_int8((unsigned char)0xAE);
7126   emit_operand(as_Register(4), dst);
7127 }
7128 
7129 void Assembler::idivq(Register src) {
7130   int encode = prefixq_and_encode(src->encoding());
7131   emit_int8((unsigned char)0xF7);
7132   emit_int8((unsigned char)(0xF8 | encode));
7133 }
7134 
7135 void Assembler::imulq(Register dst, Register src) {
7136   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
7137   emit_int8(0x0F);
7138   emit_int8((unsigned char)0xAF);
7139   emit_int8((unsigned char)(0xC0 | encode));
7140 }
7141 
7142 void Assembler::imulq(Register dst, Register src, int value) {
7143   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
7144   if (is8bit(value)) {
7145     emit_int8(0x6B);
7146     emit_int8((unsigned char)(0xC0 | encode));
7147     emit_int8(value & 0xFF);
7148   } else {
7149     emit_int8(0x69);
7150     emit_int8((unsigned char)(0xC0 | encode));
7151     emit_int32(value);
7152   }
7153 }
7154 
7155 void Assembler::imulq(Register dst, Address src) {
7156   InstructionMark im(this);
7157   prefixq(src, dst);
7158   emit_int8(0x0F);
7159   emit_int8((unsigned char) 0xAF);
7160   emit_operand(dst, src);
7161 }
7162 
7163 void Assembler::incl(Register dst) {
7164   // Don't use it directly. Use MacroAssembler::incrementl() instead.
7165   // Use two-byte form (one-byte from is a REX prefix in 64-bit mode)
7166   int encode = prefix_and_encode(dst->encoding());
7167   emit_int8((unsigned char)0xFF);
7168   emit_int8((unsigned char)(0xC0 | encode));
7169 }
7170 
7171 void Assembler::incq(Register dst) {
7172   // Don't use it directly. Use MacroAssembler::incrementq() instead.
7173   // Use two-byte form (one-byte from is a REX prefix in 64-bit mode)
7174   int encode = prefixq_and_encode(dst->encoding());
7175   emit_int8((unsigned char)0xFF);
7176   emit_int8((unsigned char)(0xC0 | encode));
7177 }
7178 
7179 void Assembler::incq(Address dst) {
7180   // Don't use it directly. Use MacroAssembler::incrementq() instead.
7181   InstructionMark im(this);
7182   prefixq(dst);
7183   emit_int8((unsigned char)0xFF);
7184   emit_operand(rax, dst);
7185 }
7186 
7187 void Assembler::lea(Register dst, Address src) {
7188   leaq(dst, src);
7189 }
7190 
7191 void Assembler::leaq(Register dst, Address src) {
7192   InstructionMark im(this);
7193   prefixq(src, dst);
7194   emit_int8((unsigned char)0x8D);
7195   emit_operand(dst, src);
7196 }
7197 
7198 void Assembler::mov64(Register dst, int64_t imm64) {
7199   InstructionMark im(this);
7200   int encode = prefixq_and_encode(dst->encoding());
7201   emit_int8((unsigned char)(0xB8 | encode));
7202   emit_int64(imm64);
7203 }
7204 
7205 void Assembler::mov_literal64(Register dst, intptr_t imm64, RelocationHolder const& rspec) {
7206   InstructionMark im(this);
7207   int encode = prefixq_and_encode(dst->encoding());
7208   emit_int8(0xB8 | encode);
7209   emit_data64(imm64, rspec);
7210 }
7211 
7212 void Assembler::mov_narrow_oop(Register dst, int32_t imm32, RelocationHolder const& rspec) {
7213   InstructionMark im(this);
7214   int encode = prefix_and_encode(dst->encoding());
7215   emit_int8((unsigned char)(0xB8 | encode));
7216   emit_data((int)imm32, rspec, narrow_oop_operand);
7217 }
7218 
7219 void Assembler::mov_narrow_oop(Address dst, int32_t imm32,  RelocationHolder const& rspec) {
7220   InstructionMark im(this);
7221   prefix(dst);
7222   emit_int8((unsigned char)0xC7);
7223   emit_operand(rax, dst, 4);
7224   emit_data((int)imm32, rspec, narrow_oop_operand);
7225 }
7226 
7227 void Assembler::cmp_narrow_oop(Register src1, int32_t imm32, RelocationHolder const& rspec) {
7228   InstructionMark im(this);
7229   int encode = prefix_and_encode(src1->encoding());
7230   emit_int8((unsigned char)0x81);
7231   emit_int8((unsigned char)(0xF8 | encode));
7232   emit_data((int)imm32, rspec, narrow_oop_operand);
7233 }
7234 
7235 void Assembler::cmp_narrow_oop(Address src1, int32_t imm32, RelocationHolder const& rspec) {
7236   InstructionMark im(this);
7237   prefix(src1);
7238   emit_int8((unsigned char)0x81);
7239   emit_operand(rax, src1, 4);
7240   emit_data((int)imm32, rspec, narrow_oop_operand);
7241 }
7242 
7243 void Assembler::lzcntq(Register dst, Register src) {
7244   assert(VM_Version::supports_lzcnt(), "encoding is treated as BSR");
7245   emit_int8((unsigned char)0xF3);
7246   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
7247   emit_int8(0x0F);
7248   emit_int8((unsigned char)0xBD);
7249   emit_int8((unsigned char)(0xC0 | encode));
7250 }
7251 
7252 void Assembler::movdq(XMMRegister dst, Register src) {
7253   // table D-1 says MMX/SSE2
7254   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
7255   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
7256   int encode = simd_prefix_and_encode(dst, xnoreg, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7257   emit_int8(0x6E);
7258   emit_int8((unsigned char)(0xC0 | encode));
7259 }
7260 
7261 void Assembler::movdq(Register dst, XMMRegister src) {
7262   // table D-1 says MMX/SSE2
7263   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
7264   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
7265   // swap src/dst to get correct prefix
7266   int encode = simd_prefix_and_encode(src, xnoreg, as_XMMRegister(dst->encoding()), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7267   emit_int8(0x7E);
7268   emit_int8((unsigned char)(0xC0 | encode));
7269 }
7270 
7271 void Assembler::movq(Register dst, Register src) {
7272   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
7273   emit_int8((unsigned char)0x8B);
7274   emit_int8((unsigned char)(0xC0 | encode));
7275 }
7276 
7277 void Assembler::movq(Register dst, Address src) {
7278   InstructionMark im(this);
7279   prefixq(src, dst);
7280   emit_int8((unsigned char)0x8B);
7281   emit_operand(dst, src);
7282 }
7283 
7284 void Assembler::movq(Address dst, Register src) {
7285   InstructionMark im(this);
7286   prefixq(dst, src);
7287   emit_int8((unsigned char)0x89);
7288   emit_operand(src, dst);
7289 }
7290 
7291 void Assembler::movsbq(Register dst, Address src) {
7292   InstructionMark im(this);
7293   prefixq(src, dst);
7294   emit_int8(0x0F);
7295   emit_int8((unsigned char)0xBE);
7296   emit_operand(dst, src);
7297 }
7298 
7299 void Assembler::movsbq(Register dst, Register src) {
7300   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
7301   emit_int8(0x0F);
7302   emit_int8((unsigned char)0xBE);
7303   emit_int8((unsigned char)(0xC0 | encode));
7304 }
7305 
7306 void Assembler::movslq(Register dst, int32_t imm32) {
7307   // dbx shows movslq(rcx, 3) as movq     $0x0000000049000000,(%rbx)
7308   // and movslq(r8, 3); as movl     $0x0000000048000000,(%rbx)
7309   // as a result we shouldn't use until tested at runtime...
7310   ShouldNotReachHere();
7311   InstructionMark im(this);
7312   int encode = prefixq_and_encode(dst->encoding());
7313   emit_int8((unsigned char)(0xC7 | encode));
7314   emit_int32(imm32);
7315 }
7316 
7317 void Assembler::movslq(Address dst, int32_t imm32) {
7318   assert(is_simm32(imm32), "lost bits");
7319   InstructionMark im(this);
7320   prefixq(dst);
7321   emit_int8((unsigned char)0xC7);
7322   emit_operand(rax, dst, 4);
7323   emit_int32(imm32);
7324 }
7325 
7326 void Assembler::movslq(Register dst, Address src) {
7327   InstructionMark im(this);
7328   prefixq(src, dst);
7329   emit_int8(0x63);
7330   emit_operand(dst, src);
7331 }
7332 
7333 void Assembler::movslq(Register dst, Register src) {
7334   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
7335   emit_int8(0x63);
7336   emit_int8((unsigned char)(0xC0 | encode));
7337 }
7338 
7339 void Assembler::movswq(Register dst, Address src) {
7340   InstructionMark im(this);
7341   prefixq(src, dst);
7342   emit_int8(0x0F);
7343   emit_int8((unsigned char)0xBF);
7344   emit_operand(dst, src);
7345 }
7346 
7347 void Assembler::movswq(Register dst, Register src) {
7348   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
7349   emit_int8((unsigned char)0x0F);
7350   emit_int8((unsigned char)0xBF);
7351   emit_int8((unsigned char)(0xC0 | encode));
7352 }
7353 
7354 void Assembler::movzbq(Register dst, Address src) {
7355   InstructionMark im(this);
7356   prefixq(src, dst);
7357   emit_int8((unsigned char)0x0F);
7358   emit_int8((unsigned char)0xB6);
7359   emit_operand(dst, src);
7360 }
7361 
7362 void Assembler::movzbq(Register dst, Register src) {
7363   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
7364   emit_int8(0x0F);
7365   emit_int8((unsigned char)0xB6);
7366   emit_int8(0xC0 | encode);
7367 }
7368 
7369 void Assembler::movzwq(Register dst, Address src) {
7370   InstructionMark im(this);
7371   prefixq(src, dst);
7372   emit_int8((unsigned char)0x0F);
7373   emit_int8((unsigned char)0xB7);
7374   emit_operand(dst, src);
7375 }
7376 
7377 void Assembler::movzwq(Register dst, Register src) {
7378   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
7379   emit_int8((unsigned char)0x0F);
7380   emit_int8((unsigned char)0xB7);
7381   emit_int8((unsigned char)(0xC0 | encode));
7382 }
7383 
7384 void Assembler::mulq(Address src) {
7385   InstructionMark im(this);
7386   prefixq(src);
7387   emit_int8((unsigned char)0xF7);
7388   emit_operand(rsp, src);
7389 }
7390 
7391 void Assembler::mulq(Register src) {
7392   int encode = prefixq_and_encode(src->encoding());
7393   emit_int8((unsigned char)0xF7);
7394   emit_int8((unsigned char)(0xE0 | encode));
7395 }
7396 
7397 void Assembler::mulxq(Register dst1, Register dst2, Register src) {
7398   assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
7399   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
7400   int encode = vex_prefix_and_encode(dst1->encoding(), dst2->encoding(), src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_38, &attributes);
7401   emit_int8((unsigned char)0xF6);
7402   emit_int8((unsigned char)(0xC0 | encode));
7403 }
7404 
7405 void Assembler::negq(Register dst) {
7406   int encode = prefixq_and_encode(dst->encoding());
7407   emit_int8((unsigned char)0xF7);
7408   emit_int8((unsigned char)(0xD8 | encode));
7409 }
7410 
7411 void Assembler::notq(Register dst) {
7412   int encode = prefixq_and_encode(dst->encoding());
7413   emit_int8((unsigned char)0xF7);
7414   emit_int8((unsigned char)(0xD0 | encode));
7415 }
7416 
7417 void Assembler::orq(Address dst, int32_t imm32) {
7418   InstructionMark im(this);
7419   prefixq(dst);
7420   emit_int8((unsigned char)0x81);
7421   emit_operand(rcx, dst, 4);
7422   emit_int32(imm32);
7423 }
7424 
7425 void Assembler::orq(Register dst, int32_t imm32) {
7426   (void) prefixq_and_encode(dst->encoding());
7427   emit_arith(0x81, 0xC8, dst, imm32);
7428 }
7429 
7430 void Assembler::orq(Register dst, Address src) {
7431   InstructionMark im(this);
7432   prefixq(src, dst);
7433   emit_int8(0x0B);
7434   emit_operand(dst, src);
7435 }
7436 
7437 void Assembler::orq(Register dst, Register src) {
7438   (void) prefixq_and_encode(dst->encoding(), src->encoding());
7439   emit_arith(0x0B, 0xC0, dst, src);
7440 }
7441 
7442 void Assembler::popa() { // 64bit
7443   movq(r15, Address(rsp, 0));
7444   movq(r14, Address(rsp, wordSize));
7445   movq(r13, Address(rsp, 2 * wordSize));
7446   movq(r12, Address(rsp, 3 * wordSize));
7447   movq(r11, Address(rsp, 4 * wordSize));
7448   movq(r10, Address(rsp, 5 * wordSize));
7449   movq(r9,  Address(rsp, 6 * wordSize));
7450   movq(r8,  Address(rsp, 7 * wordSize));
7451   movq(rdi, Address(rsp, 8 * wordSize));
7452   movq(rsi, Address(rsp, 9 * wordSize));
7453   movq(rbp, Address(rsp, 10 * wordSize));
7454   // skip rsp
7455   movq(rbx, Address(rsp, 12 * wordSize));
7456   movq(rdx, Address(rsp, 13 * wordSize));
7457   movq(rcx, Address(rsp, 14 * wordSize));
7458   movq(rax, Address(rsp, 15 * wordSize));
7459 
7460   addq(rsp, 16 * wordSize);
7461 }
7462 
7463 void Assembler::popcntq(Register dst, Address src) {
7464   assert(VM_Version::supports_popcnt(), "must support");
7465   InstructionMark im(this);
7466   emit_int8((unsigned char)0xF3);
7467   prefixq(src, dst);
7468   emit_int8((unsigned char)0x0F);
7469   emit_int8((unsigned char)0xB8);
7470   emit_operand(dst, src);
7471 }
7472 
7473 void Assembler::popcntq(Register dst, Register src) {
7474   assert(VM_Version::supports_popcnt(), "must support");
7475   emit_int8((unsigned char)0xF3);
7476   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
7477   emit_int8((unsigned char)0x0F);
7478   emit_int8((unsigned char)0xB8);
7479   emit_int8((unsigned char)(0xC0 | encode));
7480 }
7481 
7482 void Assembler::popq(Address dst) {
7483   InstructionMark im(this);
7484   prefixq(dst);
7485   emit_int8((unsigned char)0x8F);
7486   emit_operand(rax, dst);
7487 }
7488 
7489 void Assembler::pusha() { // 64bit
7490   // we have to store original rsp.  ABI says that 128 bytes
7491   // below rsp are local scratch.
7492   movq(Address(rsp, -5 * wordSize), rsp);
7493 
7494   subq(rsp, 16 * wordSize);
7495 
7496   movq(Address(rsp, 15 * wordSize), rax);
7497   movq(Address(rsp, 14 * wordSize), rcx);
7498   movq(Address(rsp, 13 * wordSize), rdx);
7499   movq(Address(rsp, 12 * wordSize), rbx);
7500   // skip rsp
7501   movq(Address(rsp, 10 * wordSize), rbp);
7502   movq(Address(rsp, 9 * wordSize), rsi);
7503   movq(Address(rsp, 8 * wordSize), rdi);
7504   movq(Address(rsp, 7 * wordSize), r8);
7505   movq(Address(rsp, 6 * wordSize), r9);
7506   movq(Address(rsp, 5 * wordSize), r10);
7507   movq(Address(rsp, 4 * wordSize), r11);
7508   movq(Address(rsp, 3 * wordSize), r12);
7509   movq(Address(rsp, 2 * wordSize), r13);
7510   movq(Address(rsp, wordSize), r14);
7511   movq(Address(rsp, 0), r15);
7512 }
7513 
7514 void Assembler::pushq(Address src) {
7515   InstructionMark im(this);
7516   prefixq(src);
7517   emit_int8((unsigned char)0xFF);
7518   emit_operand(rsi, src);
7519 }
7520 
7521 void Assembler::rclq(Register dst, int imm8) {
7522   assert(isShiftCount(imm8 >> 1), "illegal shift count");
7523   int encode = prefixq_and_encode(dst->encoding());
7524   if (imm8 == 1) {
7525     emit_int8((unsigned char)0xD1);
7526     emit_int8((unsigned char)(0xD0 | encode));
7527   } else {
7528     emit_int8((unsigned char)0xC1);
7529     emit_int8((unsigned char)(0xD0 | encode));
7530     emit_int8(imm8);
7531   }
7532 }
7533 
7534 void Assembler::rcrq(Register dst, int imm8) {
7535   assert(isShiftCount(imm8 >> 1), "illegal shift count");
7536   int encode = prefixq_and_encode(dst->encoding());
7537   if (imm8 == 1) {
7538     emit_int8((unsigned char)0xD1);
7539     emit_int8((unsigned char)(0xD8 | encode));
7540   } else {
7541     emit_int8((unsigned char)0xC1);
7542     emit_int8((unsigned char)(0xD8 | encode));
7543     emit_int8(imm8);
7544   }
7545 }
7546 
7547 void Assembler::rorq(Register dst, int imm8) {
7548   assert(isShiftCount(imm8 >> 1), "illegal shift count");
7549   int encode = prefixq_and_encode(dst->encoding());
7550   if (imm8 == 1) {
7551     emit_int8((unsigned char)0xD1);
7552     emit_int8((unsigned char)(0xC8 | encode));
7553   } else {
7554     emit_int8((unsigned char)0xC1);
7555     emit_int8((unsigned char)(0xc8 | encode));
7556     emit_int8(imm8);
7557   }
7558 }
7559 
7560 void Assembler::rorxq(Register dst, Register src, int imm8) {
7561   assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
7562   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
7563   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_3A, &attributes);
7564   emit_int8((unsigned char)0xF0);
7565   emit_int8((unsigned char)(0xC0 | encode));
7566   emit_int8(imm8);
7567 }
7568 
7569 void Assembler::sarq(Register dst, int imm8) {
7570   assert(isShiftCount(imm8 >> 1), "illegal shift count");
7571   int encode = prefixq_and_encode(dst->encoding());
7572   if (imm8 == 1) {
7573     emit_int8((unsigned char)0xD1);
7574     emit_int8((unsigned char)(0xF8 | encode));
7575   } else {
7576     emit_int8((unsigned char)0xC1);
7577     emit_int8((unsigned char)(0xF8 | encode));
7578     emit_int8(imm8);
7579   }
7580 }
7581 
7582 void Assembler::sarq(Register dst) {
7583   int encode = prefixq_and_encode(dst->encoding());
7584   emit_int8((unsigned char)0xD3);
7585   emit_int8((unsigned char)(0xF8 | encode));
7586 }
7587 
7588 void Assembler::sbbq(Address dst, int32_t imm32) {
7589   InstructionMark im(this);
7590   prefixq(dst);
7591   emit_arith_operand(0x81, rbx, dst, imm32);
7592 }
7593 
7594 void Assembler::sbbq(Register dst, int32_t imm32) {
7595   (void) prefixq_and_encode(dst->encoding());
7596   emit_arith(0x81, 0xD8, dst, imm32);
7597 }
7598 
7599 void Assembler::sbbq(Register dst, Address src) {
7600   InstructionMark im(this);
7601   prefixq(src, dst);
7602   emit_int8(0x1B);
7603   emit_operand(dst, src);
7604 }
7605 
7606 void Assembler::sbbq(Register dst, Register src) {
7607   (void) prefixq_and_encode(dst->encoding(), src->encoding());
7608   emit_arith(0x1B, 0xC0, dst, src);
7609 }
7610 
7611 void Assembler::shlq(Register dst, int imm8) {
7612   assert(isShiftCount(imm8 >> 1), "illegal shift count");
7613   int encode = prefixq_and_encode(dst->encoding());
7614   if (imm8 == 1) {
7615     emit_int8((unsigned char)0xD1);
7616     emit_int8((unsigned char)(0xE0 | encode));
7617   } else {
7618     emit_int8((unsigned char)0xC1);
7619     emit_int8((unsigned char)(0xE0 | encode));
7620     emit_int8(imm8);
7621   }
7622 }
7623 
7624 void Assembler::shlq(Register dst) {
7625   int encode = prefixq_and_encode(dst->encoding());
7626   emit_int8((unsigned char)0xD3);
7627   emit_int8((unsigned char)(0xE0 | encode));
7628 }
7629 
7630 void Assembler::shrq(Register dst, int imm8) {
7631   assert(isShiftCount(imm8 >> 1), "illegal shift count");
7632   int encode = prefixq_and_encode(dst->encoding());
7633   emit_int8((unsigned char)0xC1);
7634   emit_int8((unsigned char)(0xE8 | encode));
7635   emit_int8(imm8);
7636 }
7637 
7638 void Assembler::shrq(Register dst) {
7639   int encode = prefixq_and_encode(dst->encoding());
7640   emit_int8((unsigned char)0xD3);
7641   emit_int8(0xE8 | encode);
7642 }
7643 
7644 void Assembler::subq(Address dst, int32_t imm32) {
7645   InstructionMark im(this);
7646   prefixq(dst);
7647   emit_arith_operand(0x81, rbp, dst, imm32);
7648 }
7649 
7650 void Assembler::subq(Address dst, Register src) {
7651   InstructionMark im(this);
7652   prefixq(dst, src);
7653   emit_int8(0x29);
7654   emit_operand(src, dst);
7655 }
7656 
7657 void Assembler::subq(Register dst, int32_t imm32) {
7658   (void) prefixq_and_encode(dst->encoding());
7659   emit_arith(0x81, 0xE8, dst, imm32);
7660 }
7661 
7662 // Force generation of a 4 byte immediate value even if it fits into 8bit
7663 void Assembler::subq_imm32(Register dst, int32_t imm32) {
7664   (void) prefixq_and_encode(dst->encoding());
7665   emit_arith_imm32(0x81, 0xE8, dst, imm32);
7666 }
7667 
7668 void Assembler::subq(Register dst, Address src) {
7669   InstructionMark im(this);
7670   prefixq(src, dst);
7671   emit_int8(0x2B);
7672   emit_operand(dst, src);
7673 }
7674 
7675 void Assembler::subq(Register dst, Register src) {
7676   (void) prefixq_and_encode(dst->encoding(), src->encoding());
7677   emit_arith(0x2B, 0xC0, dst, src);
7678 }
7679 
7680 void Assembler::testq(Register dst, int32_t imm32) {
7681   // not using emit_arith because test
7682   // doesn't support sign-extension of
7683   // 8bit operands
7684   int encode = dst->encoding();
7685   if (encode == 0) {
7686     prefix(REX_W);
7687     emit_int8((unsigned char)0xA9);
7688   } else {
7689     encode = prefixq_and_encode(encode);
7690     emit_int8((unsigned char)0xF7);
7691     emit_int8((unsigned char)(0xC0 | encode));
7692   }
7693   emit_int32(imm32);
7694 }
7695 
7696 void Assembler::testq(Register dst, Register src) {
7697   (void) prefixq_and_encode(dst->encoding(), src->encoding());
7698   emit_arith(0x85, 0xC0, dst, src);
7699 }
7700 
7701 void Assembler::xaddq(Address dst, Register src) {
7702   InstructionMark im(this);
7703   prefixq(dst, src);
7704   emit_int8(0x0F);
7705   emit_int8((unsigned char)0xC1);
7706   emit_operand(src, dst);
7707 }
7708 
7709 void Assembler::xchgq(Register dst, Address src) {
7710   InstructionMark im(this);
7711   prefixq(src, dst);
7712   emit_int8((unsigned char)0x87);
7713   emit_operand(dst, src);
7714 }
7715 
7716 void Assembler::xchgq(Register dst, Register src) {
7717   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
7718   emit_int8((unsigned char)0x87);
7719   emit_int8((unsigned char)(0xc0 | encode));
7720 }
7721 
7722 void Assembler::xorq(Register dst, Register src) {
7723   (void) prefixq_and_encode(dst->encoding(), src->encoding());
7724   emit_arith(0x33, 0xC0, dst, src);
7725 }
7726 
7727 void Assembler::xorq(Register dst, Address src) {
7728   InstructionMark im(this);
7729   prefixq(src, dst);
7730   emit_int8(0x33);
7731   emit_operand(dst, src);
7732 }
7733 
7734 #endif // !LP64